@sanity/ailf 0.1.0

package/dist/pipeline/grader-sensitivity-runner.js

@@ -0,0 +1,282 @@
+/**
+ * pipeline/grader-sensitivity-runner.ts
+ *
+ * Orchestration module for grader sensitivity (discrimination power) testing
+ * (Phase 4).
+ *
+ * Discovers canonical reference solutions, applies programmatic degradations,
+ * grades each original/degraded pair, and calls `analyzeSensitivity()` from
+ * the pure computation module.
+ *
+ * Migrated from lib/grader-sensitivity.ts — no process.argv, no process.exit(),
+ * no module-level constants. Accepts rootDir as parameter.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
+ */
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, } from "fs";
+import { basename, join } from "path";
+import { DEGRADATION_STRATEGIES } from "./degradations.js";
+import { gradeOnce, loadGraderModel } from "./grader-api.js";
+import { analyzeSensitivity, } from "./grader-sensitivity.js";
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Discover canonical reference solutions.
+ * Scans canonical/reference-solutions/<area>/ for source files.
+ */
+function discoverReferenceSolutions(rootDir, areaFilter) {
+    const refsDir = join(rootDir, "canonical", "reference-solutions");
+    if (!existsSync(refsDir)) {
+        // Reference solutions now live in the Content Lake (ailf.referenceSolution).
+        // Return empty array when local files don't exist.
+        return [];
+    }
+    const areas = readdirSync(refsDir, { withFileTypes: true })
+        .filter((d) => d.isDirectory())
+        .map((d) => d.name)
+        .filter((name) => !areaFilter || name === areaFilter)
+        .sort();
+    if (areas.length === 0) {
+        throw new Error(areaFilter
+            ? `No reference solutions found for area "${areaFilter}".`
+            : "No reference solution directories found.");
+    }
+    const solutions = [];
+    for (const area of areas) {
+        const areaDir = join(refsDir, area);
+        const files = readdirSync(areaDir)
+            .filter((f) => f.endsWith(".ts") ||
+            f.endsWith(".tsx") ||
+            f.endsWith(".js") ||
+            f.endsWith(".jsx"))
+            .sort();
+        for (const file of files) {
+            const filePath = join(areaDir, file);
+            const content = readFileSync(filePath, "utf-8");
+            solutions.push({
+                area,
+                content,
+                sourcePath: `canonical/reference-solutions/${area}/${file}`,
+            });
+        }
+    }
+    return solutions;
+}
+/**
+ * Generate all degraded pairs from reference solutions.
+ * Each solution × each degradation strategy = one pair.
+ */
+function generateDegradedPairs(solutions) {
+    const pairs = [];
+    for (const solution of solutions) {
+        for (const strategy of DEGRADATION_STRATEGIES) {
+            const degraded = strategy.apply(solution.content);
+            // Only include if degradation actually changed the code
+            if (degraded !== solution.content) {
+                pairs.push({
+                    degradation: strategy,
+                    degraded,
+                    original: solution.content,
+                    sourcePath: solution.sourcePath,
+                });
+            }
+        }
+    }
+    return pairs;
+}
+// ---------------------------------------------------------------------------
+// Report formatting (pure)
+// ---------------------------------------------------------------------------
+/**
+ * Format a GraderSensitivityResult as a human-readable table report.
+ * Returns a string — does NOT print to console.
+ */
+export function formatSensitivityReport(result) {
+    const lines = [];
+    lines.push("=".repeat(80));
+    lines.push("                    GRADER SENSITIVITY REPORT");
+    lines.push("=".repeat(80));
+    lines.push("");
+    lines.push(`  Grader:       ${result.graderModel}`);
+    lines.push(`  Total pairs:  ${result.totalPairs}`);
+    lines.push("");
+    // Overall metrics
+    lines.push("-".repeat(80));
+    lines.push("OVERALL");
+    lines.push("-".repeat(80));
+    lines.push("");
+    lines.push(`  Concordance:  ${result.concordanceRate}%`);
+    lines.push(`  Avg separation: ${result.avgSeparation} points`);
+    lines.push("");
+    // Per-dimension table
+    lines.push("-".repeat(80));
+    lines.push("PER-DIMENSION SENSITIVITY");
+    lines.push("-".repeat(80));
+    lines.push("");
+    const h = "| Dimension        | Concordance | Avg Sep | Tied  | Pairs |";
+    const sep = "|------------------|-------------|---------|-------|-------|";
+    lines.push(h);
+    lines.push(sep);
+    const dims = [
+        { data: result.perDimension.taskCompletion, name: "Task Completion" },
+        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
+        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
+    ];
+    for (const { data, name } of dims) {
+        lines.push(`| ${name.padEnd(16)} | ${String(data.concordanceRate + "%").padStart(11)} | ${String(data.avgSeparation).padStart(7)} | ${String(data.tiedRate + "%").padStart(5)} | ${String(data.pairCount).padStart(5)} |`);
+    }
+    lines.push("");
+    // Cross-dimension analysis
+    lines.push("-".repeat(80));
+    lines.push("CROSS-DIMENSION SENSITIVITY");
+    lines.push("-".repeat(80));
+    lines.push("");
+    lines.push(`  On-target:   ${result.crossDimension.onTarget.concordanceRate}% concordance, ${result.crossDimension.onTarget.avgSeparation} avg sep (${result.crossDimension.onTarget.pairCount} pairs)`);
+    lines.push(`  Off-target:  ${result.crossDimension.offTarget.concordanceRate}% concordance, ${result.crossDimension.offTarget.avgSeparation} avg sep (${result.crossDimension.offTarget.pairCount} pairs)`);
+    lines.push("");
+    // Per-degradation breakdown
+    if (result.byDegradation.length > 0) {
+        lines.push("-".repeat(80));
+        lines.push("PER-DEGRADATION TYPE");
+        lines.push("-".repeat(80));
+        lines.push("");
+        const dh = "| Degradation                                  | Concordance | Avg Sep | Pairs |";
+        const ds = "|----------------------------------------------|-------------|---------|-------|";
+        lines.push(dh);
+        lines.push(ds);
+        for (const deg of result.byDegradation) {
+            const desc = deg.description.slice(0, 44).padEnd(44);
+            lines.push(`| ${desc} | ${String(deg.concordanceRate + "%").padStart(11)} | ${String(deg.avgSeparation).padStart(7)} | ${String(deg.pairCount).padStart(5)} |`);
+        }
+        lines.push("");
+    }
+    // Failed pairs (worst failures)
+    const topN = Math.min(5, result.failedPairs.length);
+    if (topN > 0) {
+        lines.push("-".repeat(80));
+        lines.push(`TOP ${topN} REVERSED PAIRS (grader ranked degraded higher)`);
+        lines.push("-".repeat(80));
+        lines.push("");
+        for (let i = 0; i < topN; i++) {
+            const p = result.failedPairs[i];
+            const delta = p.degradedScore - p.originalScore;
+            lines.push(`  ${i + 1}. ${basename(p.sourcePath)} — ${p.dimension}`);
+            lines.push(`     Original=${p.originalScore}, Degraded=${p.degradedScore} (Δ=+${delta})`);
+            lines.push(`     Degradation: ${p.degradationDescription}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+/**
+ * Run grader sensitivity analysis.
+ *
+ * Discovers reference solutions, applies degradations, grades each pair
+ * with the configured grader, and analyzes discrimination power.
+ *
+ * @throws Error if no reference solutions found or no pairs generated
+ */
+export async function runGraderSensitivity(options) {
+    const { rootDir, areaFilter, format = "table" } = options;
+    console.log("=== Grader Sensitivity Analysis ===\n");
+    // Resolve grader model
+    const grader = loadGraderModel(rootDir);
+    console.log(`  Grader:       ${grader.label} (${grader.id})`);
+    // Discover reference solutions
+    const solutions = discoverReferenceSolutions(rootDir, areaFilter);
+    console.log(`  Solutions:    ${solutions.length} reference files`);
+    if (areaFilter) {
+        console.log(`  Area filter:  ${areaFilter}`);
+    }
+    // Generate degraded pairs
+    const degradedPairs = generateDegradedPairs(solutions);
+    console.log(`  Pairs:        ${degradedPairs.length} (solutions × degradations)`);
+    if (degradedPairs.length === 0) {
+        throw new Error("No degraded pairs generated. Check reference solutions.");
+    }
+    // Each pair needs 2 grades (original + degraded) × 3 dimensions = 6 calls
+    const dimensions = [
+        "taskCompletion",
+        "codeCorrectness",
+        "docCoverage",
+    ];
+    const totalCalls = degradedPairs.length * dimensions.length * 2;
+    const estimatedCost = totalCalls * 0.005;
+    console.log(`  API calls:    ${totalCalls}`);
+    console.log(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
+    console.log();
+    // Grade each pair
+    console.log("  Grading original and degraded pairs...");
+    const rubricTemplate = (dim) => {
+        const labels = {
+            codeCorrectness: "Evaluate code correctness: Does the code follow best practices, use correct APIs, and avoid anti-patterns? Score 0–100.",
+            docCoverage: "Evaluate documentation coverage: Does the response demonstrate knowledge from official documentation? Are imports, API names, and configuration correct per the docs? Score 0–100.",
+            taskCompletion: "Evaluate task completion: Does the response fully implement the requested feature? Are all requirements addressed? Score 0–100.",
+        };
+        return labels[dim] ?? `Evaluate ${dim}. Score 0–100.`;
+    };
+    const sensitivityPairs = [];
+    let completed = 0;
+    let failed = 0;
+    for (const pair of degradedPairs) {
+        const area = pair.sourcePath
+            .split("/")
+            .find((_s, i, arr) => arr[i - 1] === "reference-solutions") ?? "unknown";
+        for (const dimension of dimensions) {
+            const rubric = rubricTemplate(dimension);
+            // Grade original
+            const originalScore = await gradeOnce(grader.id, pair.original, rubric);
+            // Grade degraded
+            const degradedScore = await gradeOnce(grader.id, pair.degraded, rubric);
+            completed++;
+            if (completed % 10 === 0 ||
+                completed === degradedPairs.length * dimensions.length) {
+                const total = degradedPairs.length * dimensions.length;
+                const pct = Math.round((completed / total) * 100);
+                process.stdout.write(`\r    Progress: ${completed}/${total} (${pct}%)`);
+            }
+            if (originalScore === null || degradedScore === null) {
+                failed++;
+                continue;
+            }
+            sensitivityPairs.push({
+                area,
+                degradationDescription: pair.degradation.description,
+                degradedScore,
+                dimension,
+                originalScore,
+                sourcePath: pair.sourcePath,
+                targetDimension: pair.degradation.targetDimension,
+            });
+        }
+    }
+    console.log(); // newline after progress
+    if (failed > 0) {
+        console.log(`    ⚠ ${failed} grading pairs failed (excluded)`);
+    }
+    console.log();
+    if (sensitivityPairs.length === 0) {
+        throw new Error("No sensitivity pairs to analyze. All grading calls failed.");
+    }
+    // Analyze
+    const result = analyzeSensitivity(sensitivityPairs, grader.id);
+    // Output
+    if (format === "table") {
+        console.log(formatSensitivityReport(result));
+    }
+    else {
+        console.log(JSON.stringify(result, null, 2));
+    }
+    // Write output
+    const outPath = options.outputPath ??
+        join(rootDir, "results", "latest", "grader-sensitivity.json");
+    const outDir = join(outPath, "..");
+    mkdirSync(outDir, { recursive: true });
+    writeFileSync(outPath, JSON.stringify(result, null, 2));
+    console.log(`\n  📄 Results written to ${outPath}`);
+    return result;
+}

package/dist/pipeline/grader-sensitivity.d.ts

@@ -0,0 +1,94 @@
+/**
+ * pipeline/grader-sensitivity.ts
+ *
+ * Pure computation module for measuring grader sensitivity (discrimination power).
+ *
+ * Takes paired grading results — original (good) and degraded (bad) versions
+ * of the same response graded by the same grader — and computes:
+ * - Concordance rate: % of pairs where the grader ranked the original higher
+ * - Score separation: average score difference between good and bad
+ * - Per-dimension sensitivity
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
+ */
+/** Sensitivity broken down by degradation type */
+export interface DegradationSensitivity {
+    /** Average score separation for this degradation type */
+    avgSeparation: number;
+    /** Concordance rate for this degradation type */
+    concordanceRate: number;
+    /** Description of the degradation */
+    description: string;
+    /** Number of pairs using this degradation */
+    pairCount: number;
+    /** Which dimension this degradation targeted */
+    targetDimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+}
+/** Per-dimension sensitivity metrics */
+export interface DimensionSensitivity {
+    /** Average score separation (original - degraded) */
+    avgSeparation: number;
+    /** % of pairs where grader ranked original higher */
+    concordanceRate: number;
+    /** Number of paired comparisons */
+    pairCount: number;
+    /** % of pairs where scores were exactly equal */
+    tiedRate: number;
+}
+/** Full sensitivity analysis result */
+export interface GraderSensitivityResult {
+    /** Average score separation across all pairs */
+    avgSeparation: number;
+    /** Per-degradation-type sensitivity */
+    byDegradation: DegradationSensitivity[];
+    /** Overall concordance rate (% of correctly ranked pairs) */
+    concordanceRate: number;
+    /** Cross-dimension sensitivity: does degradation in one dimension affect others? */
+    crossDimension: {
+        /** When targetDimension != dimension, does the score still drop? */
+        onTarget: DimensionSensitivity;
+        /** Scores for non-targeted dimensions */
+        offTarget: DimensionSensitivity;
+    };
+    /** Number of pairs where the grader gave a HIGHER score to the degraded version */
+    failedPairs: SensitivityPair[];
+    /** When this analysis was generated */
+    generatedAt: string;
+    /** Grader model used */
+    graderModel: string;
+    /** Per-dimension sensitivity metrics */
+    perDimension: {
+        taskCompletion: DimensionSensitivity;
+        codeCorrectness: DimensionSensitivity;
+        docCoverage: DimensionSensitivity;
+    };
+    /** Total paired comparisons analyzed */
+    totalPairs: number;
+}
+/** A single paired comparison: original vs. degraded response */
+export interface SensitivityPair {
+    /** Feature area (e.g., "groq") */
+    area: string;
+    /** What degradation was applied */
+    degradationDescription: string;
+    /** Score assigned to the degraded version (0–100) */
+    degradedScore: number;
+    /** Which dimension this judgment measures */
+    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Score assigned to the original version (0–100) */
+    originalScore: number;
+    /** Source file of the reference solution */
+    sourcePath: string;
+    /** Which dimension the degradation targeted */
+    targetDimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+}
+/**
+ * Analyze sensitivity from a set of paired grading results.
+ *
+ * @param pairs Array of paired comparisons (original vs. degraded)
+ * @param graderModel Grader model ID for the report
+ * @returns Full sensitivity analysis with concordance, separation, and per-dimension metrics
+ */
+export declare function analyzeSensitivity(pairs: SensitivityPair[], graderModel: string): GraderSensitivityResult;

package/dist/pipeline/grader-sensitivity.js

@@ -0,0 +1,144 @@
+/**
+ * pipeline/grader-sensitivity.ts
+ *
+ * Pure computation module for measuring grader sensitivity (discrimination power).
+ *
+ * Takes paired grading results — original (good) and degraded (bad) versions
+ * of the same response graded by the same grader — and computes:
+ * - Concordance rate: % of pairs where the grader ranked the original higher
+ * - Score separation: average score difference between good and bad
+ * - Per-dimension sensitivity
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
+ */
+// ---------------------------------------------------------------------------
+// Pure computation
+// ---------------------------------------------------------------------------
+/**
+ * Analyze sensitivity from a set of paired grading results.
+ *
+ * @param pairs Array of paired comparisons (original vs. degraded)
+ * @param graderModel Grader model ID for the report
+ * @returns Full sensitivity analysis with concordance, separation, and per-dimension metrics
+ */
+export function analyzeSensitivity(pairs, graderModel) {
+    if (pairs.length === 0) {
+        return emptyResult(graderModel);
+    }
+    // Overall concordance and separation
+    const { avgSeparation, concordanceRate, tiedRate: _tiedRate, } = computeMetrics(pairs);
+    // Per-dimension (based on the grading dimension, not the target dimension)
+    const perDimension = {
+        codeCorrectness: computeMetrics(pairs.filter((p) => p.dimension === "codeCorrectness")),
+        docCoverage: computeMetrics(pairs.filter((p) => p.dimension === "docCoverage")),
+        taskCompletion: computeMetrics(pairs.filter((p) => p.dimension === "taskCompletion")),
+    };
+    // Cross-dimension: on-target (dimension matches targetDimension) vs off-target
+    const onTargetPairs = pairs.filter((p) => p.dimension === p.targetDimension);
+    const offTargetPairs = pairs.filter((p) => p.dimension !== p.targetDimension);
+    const crossDimension = {
+        offTarget: computeMetrics(offTargetPairs),
+        onTarget: computeMetrics(onTargetPairs),
+    };
+    // Per-degradation type
+    const byDegradation = computeByDegradation(pairs);
+    // Failed pairs (grader ranked degraded higher)
+    const failedPairs = pairs
+        .filter((p) => p.degradedScore > p.originalScore)
+        .sort((a, b) => {
+        const sepA = a.degradedScore - a.originalScore;
+        const sepB = b.degradedScore - b.originalScore;
+        return sepB - sepA; // worst failures first
+    });
+    return {
+        avgSeparation,
+        byDegradation,
+        concordanceRate,
+        crossDimension,
+        failedPairs,
+        generatedAt: new Date().toISOString(),
+        graderModel,
+        perDimension,
+        totalPairs: pairs.length,
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/** Group pairs by degradation description and compute per-group metrics */
+function computeByDegradation(pairs) {
+    const groups = new Map();
+    for (const p of pairs) {
+        const key = p.degradationDescription;
+        const group = groups.get(key) ?? [];
+        group.push(p);
+        groups.set(key, group);
+    }
+    const results = [];
+    for (const [description, group] of groups) {
+        const metrics = computeMetrics(group);
+        results.push({
+            avgSeparation: metrics.avgSeparation,
+            concordanceRate: metrics.concordanceRate,
+            description,
+            pairCount: metrics.pairCount,
+            targetDimension: group[0].targetDimension,
+        });
+    }
+    // Sort by concordance rate ascending (worst discrimination first)
+    results.sort((a, b) => a.concordanceRate - b.concordanceRate);
+    return results;
+}
+/** Compute concordance, separation, and tied rate from a set of pairs */
+function computeMetrics(pairs) {
+    if (pairs.length === 0) {
+        return { avgSeparation: 0, concordanceRate: 0, pairCount: 0, tiedRate: 0 };
+    }
+    let concordant = 0;
+    let tied = 0;
+    let totalSeparation = 0;
+    for (const p of pairs) {
+        const sep = p.originalScore - p.degradedScore;
+        totalSeparation += sep;
+        if (p.originalScore > p.degradedScore)
+            concordant++;
+        if (p.originalScore === p.degradedScore)
+            tied++;
+    }
+    return {
+        avgSeparation: round(totalSeparation / pairs.length),
+        concordanceRate: round((concordant / pairs.length) * 100),
+        pairCount: pairs.length,
+        tiedRate: round((tied / pairs.length) * 100),
+    };
+}
+/** Empty result for when there are no pairs */
+function emptyResult(graderModel) {
+    const emptyDim = {
+        avgSeparation: 0,
+        concordanceRate: 0,
+        pairCount: 0,
+        tiedRate: 0,
+    };
+    return {
+        avgSeparation: 0,
+        byDegradation: [],
+        concordanceRate: 0,
+        crossDimension: { offTarget: emptyDim, onTarget: emptyDim },
+        failedPairs: [],
+        generatedAt: new Date().toISOString(),
+        graderModel,
+        perDimension: {
+            codeCorrectness: emptyDim,
+            docCoverage: emptyDim,
+            taskCompletion: emptyDim,
+        },
+        totalPairs: 0,
+    };
+}
+/** Round to 1 decimal place */
+function round(n) {
+    return Math.round(n * 10) / 10;
+}

package/dist/pipeline/grader-validate-runner.d.ts

@@ -0,0 +1,38 @@
+/**
+ * pipeline/grader-validate-runner.ts
+ *
+ * Orchestration module for grader validation against human reference grades
+ * (Phase 2).
+ *
+ * Loads human-graded reference samples from canonical/grader-references/,
+ * runs the grader model on each sample, and compares against human scores
+ * using `validateGrader()` from the pure computation module.
+ *
+ * Migrated from lib/grader-validate.ts — no process.argv, no process.exit(),
+ * no module-level constants. Accepts rootDir as parameter.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 2
+ */
+import { type GraderValidation } from "./grader-validation.js";
+export interface GraderValidateRunnerOptions {
+    /** Grader model to validate (defaults to loadGraderModel(rootDir).id) */
+    graderModel?: string;
+    /** MAE threshold for pass/fail (default: 10) */
+    maeThreshold?: number;
+    /** Root directory of the eval package */
+    rootDir: string;
+}
+/**
+ * Format a GraderValidation result as a human-readable table report.
+ * Returns a string — does NOT print to console.
+ */
+export declare function formatValidationReport(result: GraderValidation): string;
+/**
+ * Run grader validation against human reference grades.
+ *
+ * Loads human-graded samples, grades each with the grader model,
+ * and computes validation metrics (MAE, correlation, bias).
+ *
+ * @throws Error if no reference grades found, or no grades to analyze
+ */
+export declare function runGraderValidate(options: GraderValidateRunnerOptions): Promise<GraderValidation>;