@sanity/ailf 0.1.0

package/dist/pipeline/compare.js

@@ -0,0 +1,208 @@
+/**
+ * pipeline/compare.ts
+ *
+ * Core comparison primitive for the evaluation framework.
+ *
+ * Takes two ScoreSummary objects (baseline and experiment) and produces a
+ * structured ComparisonReport with overall, per-area, and per-dimension
+ * deltas, plus improved/regressed/unchanged classification.
+ *
+ * This is the single function that backs all comparison scenarios:
+ * doc improvement, model comparison, branch validation, etc.
+ * What varies is what produced each ScoreSummary — the comparison
+ * logic is always the same.
+ *
+ * @see docs/ideas/evaluation-roadmap.md — BP5: Make comparison a primitive
+ * @see docs/ideas/metrics-design.md — Tier 4: Comparison results
+ */
+import { DEFAULT_NOISE_THRESHOLD, } from "./types.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Classify a delta as improved, regressed, or unchanged given a threshold */
+export function classifyChange(delta, threshold) {
+    if (delta > threshold)
+        return "improved";
+    if (delta < -threshold)
+        return "regressed";
+    return "unchanged";
+}
+/**
+ * Compare two evaluation score summaries and produce a structured report.
+ *
+ * This is a pure function — no side effects, no file I/O.
+ *
+ * @param baseline  The "before" or "control" score summary
+ * @param experiment  The "after" or "treatment" score summary
+ * @param options  Optional configuration (noise threshold, etc.)
+ * @returns A ComparisonReport with deltas, classifications, and breakdowns
+ */
+export function compare(baseline, experiment, options) {
+    // Use grader consistency data for empirical threshold when available
+    const graderConsistency = options?.graderConsistency;
+    const empirical = graderConsistency !== undefined;
+    const threshold = empirical
+        ? graderConsistency.recommendedThreshold
+        : (options?.noiseThreshold ?? DEFAULT_NOISE_THRESHOLD);
+    // Collect all unique area names from both summaries
+    const baselineAreas = new Set(baseline.scores.map((s) => s.feature));
+    const experimentAreas = new Set(experiment.scores.map((s) => s.feature));
+    const allAreas = new Set([...baselineAreas, ...experimentAreas]);
+    // Identify mismatched areas
+    const onlyInBaseline = [...baselineAreas].filter((a) => !experimentAreas.has(a));
+    const onlyInExperiment = [...experimentAreas].filter((a) => !baselineAreas.has(a));
+    // Build per-area deltas
+    const areas = [...allAreas]
+        .sort()
+        .map((area) => buildAreaDelta(area, findScore(baseline.scores, area), findScore(experiment.scores, area), threshold));
+    // Classify areas
+    const improved = areas
+        .filter((a) => a.change === "improved")
+        .map((a) => a.area);
+    const regressed = areas
+        .filter((a) => a.change === "regressed")
+        .map((a) => a.area);
+    const unchanged = areas
+        .filter((a) => a.change === "unchanged")
+        .map((a) => a.area);
+    // Per-area deltas as a record
+    const perArea = {};
+    for (const a of areas) {
+        perArea[a.area] = a.delta;
+    }
+    // Per-dimension average deltas (only for areas present in both summaries)
+    const commonAreas = areas.filter((a) => baselineAreas.has(a.area) && experimentAreas.has(a.area));
+    const commonCount = commonAreas.length || 1;
+    const perDimension = {
+        codeCorrectness: commonAreas.reduce((s, a) => s + a.dimensions.codeCorrectness.delta, 0) /
+            commonCount,
+        docCoverage: commonAreas.reduce((s, a) => s + a.dimensions.docCoverage.delta, 0) /
+            commonCount,
+        taskCompletion: commonAreas.reduce((s, a) => s + a.dimensions.taskCompletion.delta, 0) /
+            commonCount,
+    };
+    // Doc Lift average delta (common areas only)
+    const docLift = commonAreas.reduce((s, a) => s + a.docLiftDelta, 0) / commonCount;
+    // Cost delta (if both summaries have cost data)
+    const baselineCost = baseline.overall.cost?.total;
+    const experimentCost = experiment.overall.cost?.total;
+    const costDelta = baselineCost !== undefined && experimentCost !== undefined
+        ? experimentCost - baselineCost
+        : undefined;
+    // Overall score delta
+    const overallDelta = experiment.overall.avgScore - baseline.overall.avgScore;
+    // Per-model deltas (when both summaries have per-model data)
+    // Guard: older reports may have perModel as a Record instead of an array
+    let perModelDeltas;
+    if (Array.isArray(baseline.perModel) && Array.isArray(experiment.perModel)) {
+        const allModelIds = new Set([
+            ...baseline.perModel.map((m) => m.modelId),
+            ...experiment.perModel.map((m) => m.modelId),
+        ]);
+        perModelDeltas = [];
+        for (const modelId of allModelIds) {
+            const bScore = baseline.perModel.find((m) => m.modelId === modelId)?.overall
+                .avgScore ?? 0;
+            const eScore = experiment.perModel.find((m) => m.modelId === modelId)?.overall
+                .avgScore ?? 0;
+            perModelDeltas.push({ delta: eScore - bScore, modelId });
+        }
+    }
+    return {
+        areas,
+        baseline,
+        deltas: {
+            docLift,
+            overall: overallDelta,
+            perArea,
+            perDimension,
+            ...(costDelta !== undefined && { cost: costDelta }),
+            ...(perModelDeltas && { perModel: perModelDeltas }),
+        },
+        experiment,
+        generatedAt: new Date().toISOString(),
+        improved,
+        mismatched: {
+            onlyInBaseline,
+            onlyInExperiment,
+        },
+        noiseThreshold: threshold,
+        noiseThresholdEmpirical: empirical,
+        regressed,
+        unchanged,
+    };
+}
+/** Build an AreaDelta from baseline and experiment scores for a single area */
+function buildAreaDelta(area, baselineScore, experimentScore, threshold) {
+    const b = baselineScore;
+    const e = experimentScore;
+    const bTotal = b?.totalScore ?? 0;
+    const eTotal = e?.totalScore ?? 0;
+    const delta = eTotal - bTotal;
+    const bTask = b?.taskCompletion ?? 0;
+    const eTask = e?.taskCompletion ?? 0;
+    const bCode = b?.codeCorrectness ?? 0;
+    const eCode = e?.codeCorrectness ?? 0;
+    const bDoc = b?.docCoverage ?? 0;
+    const eDoc = e?.docCoverage ?? 0;
+    // Support both new field names and legacy data (old baselines/Sanity docs)
+    const bRaw = b;
+    const eRaw = e;
+    const bLift = b?.docLift ?? bRaw?.["liftFromDocs"] ?? 0;
+    const eLift = e?.docLift ?? eRaw?.["liftFromDocs"] ?? 0;
+    const bCeiling = b?.ceilingScore ?? bRaw?.["withDocsScore"] ?? 0;
+    const eCeiling = e?.ceilingScore ?? eRaw?.["withDocsScore"] ?? 0;
+    const bFloor = b?.floorScore ?? bRaw?.["withoutDocsScore"] ?? 0;
+    const eFloor = e?.floorScore ?? eRaw?.["withoutDocsScore"] ?? 0;
+    const bCost = b?.totalCost ?? 0;
+    const eCost = e?.totalCost ?? 0;
+    const hasCost = bCost > 0 || eCost > 0;
+    // Actual (agentic) score deltas — only when both runs have actual data
+    const bActual = b?.actualScore;
+    const eActual = e?.actualScore;
+    const hasActual = bActual !== undefined && eActual !== undefined;
+    // Retrieval gap deltas — only when both runs have retrieval gap data
+    const bRetGap = b?.retrievalGap;
+    const eRetGap = e?.retrievalGap;
+    const hasRetGap = bRetGap !== undefined && eRetGap !== undefined;
+    // Infrastructure efficiency deltas
+    const bInfra = b?.infrastructureEfficiency;
+    const eInfra = e?.infrastructureEfficiency;
+    const hasInfra = bInfra != null && eInfra != null;
+    return {
+        ...(hasActual && { actualDelta: eActual - bActual }),
+        area,
+        baseline: bTotal,
+        ceilingDelta: eCeiling - bCeiling,
+        change: classifyChange(delta, threshold),
+        delta,
+        dimensions: {
+            codeCorrectness: {
+                baseline: bCode,
+                delta: eCode - bCode,
+                experiment: eCode,
+            },
+            docCoverage: { baseline: bDoc, delta: eDoc - bDoc, experiment: eDoc },
+            taskCompletion: {
+                baseline: bTask,
+                delta: eTask - bTask,
+                experiment: eTask,
+            },
+        },
+        docLiftDelta: eLift - bLift,
+        experiment: eTotal,
+        floorDelta: eFloor - bFloor,
+        ...(hasInfra && {
+            infrastructureEfficiencyDelta: eInfra - bInfra,
+        }),
+        ...(hasRetGap && { retrievalGapDelta: eRetGap - bRetGap }),
+        ...(hasCost && { costDelta: eCost - bCost }),
+    };
+}
+// ---------------------------------------------------------------------------
+// Main compare function
+// ---------------------------------------------------------------------------
+/** Safely get a feature score from a scores array */
+function findScore(scores, area) {
+    return scores.find((s) => s.feature === area);
+}

package/dist/pipeline/coverage-audit.d.ts

@@ -0,0 +1,39 @@
+/**
+ * coverage-audit.ts
+ *
+ * Pure computation functions for cross-referencing the product feature registry
+ * (config/features.yaml) against actual task files (tasks/*.yaml)
+ * to produce a documentation coverage audit.
+ *
+ * Phase 3c of the Scenario Matrix implementation.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+import type { CoverageAuditReport, ProductFeature } from "./types.js";
+/**
+ * Count unique document slugs referenced across all tasks.
+ */
+export declare function countReferencedDocs(rootDir: string): {
+    slugs: string[];
+    total: number;
+};
+/**
+ * Count actual tasks per area from task YAML files.
+ */
+export declare function countTasksByArea(rootDir: string): Record<string, number>;
+/**
+ * Format coverage audit for console output.
+ */
+export declare function formatCoverageConsole(report: CoverageAuditReport): string;
+/**
+ * Format coverage audit as markdown.
+ */
+export declare function formatCoverageMarkdown(report: CoverageAuditReport): string;
+/**
+ * Load and validate the feature registry from config/features.yaml.
+ */
+export declare function loadFeatureRegistry(rootDir: string): null | ProductFeature[];
+/**
+ * Run the coverage audit and produce a structured report.
+ */
+export declare function runCoverageAudit(rootDir: string): CoverageAuditReport | null;

package/dist/pipeline/coverage-audit.js

@@ -0,0 +1,165 @@
+/**
+ * coverage-audit.ts
+ *
+ * Pure computation functions for cross-referencing the product feature registry
+ * (config/features.yaml) against actual task files (tasks/*.yaml)
+ * to produce a documentation coverage audit.
+ *
+ * Phase 3c of the Scenario Matrix implementation.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+import { load } from "js-yaml";
+import { FeatureRegistrySchema } from "./schemas.js";
+import { resolveMappings } from "./resolve-mappings.js";
+// ---------------------------------------------------------------------------
+// Core logic (exported for testing)
+// ---------------------------------------------------------------------------
+/**
+ * Count unique document slugs referenced across all tasks.
+ */
+export function countReferencedDocs(rootDir) {
+    const mappings = resolveMappings(rootDir);
+    const allSlugs = new Set();
+    for (const config of Object.values(mappings.feature_areas)) {
+        for (const task of config.tasks) {
+            for (const doc of task.canonical_docs) {
+                allSlugs.add(doc.slug);
+            }
+        }
+    }
+    const slugs = [...allSlugs].sort();
+    return { slugs, total: slugs.length };
+}
+/**
+ * Count actual tasks per area from task YAML files.
+ */
+export function countTasksByArea(rootDir) {
+    const mappings = resolveMappings(rootDir);
+    const counts = {};
+    for (const [area, config] of Object.entries(mappings.feature_areas)) {
+        counts[area] = config.tasks.length;
+    }
+    return counts;
+}
+/**
+ * Format coverage audit for console output.
+ */
+export function formatCoverageConsole(report) {
+    const lines = [];
+    lines.push("═══════════════════════════════════════════════════════════════");
+    lines.push(" DOCUMENTATION COVERAGE AUDIT");
+    lines.push("═══════════════════════════════════════════════════════════════");
+    lines.push("");
+    lines.push(`Coverage: ${report.covered.length}/${report.totalFeatures} features (${report.coveragePercent}%)`);
+    lines.push("");
+    // Covered features
+    if (report.covered.length > 0) {
+        lines.push("COVERED FEATURES:");
+        for (const f of report.covered) {
+            const taskLabel = f.actualTaskCount === 1 ? "1 task" : `${f.actualTaskCount} tasks`;
+            const sections = f.sections.join(", ");
+            lines.push(`  ✅ ${f.id.padEnd(20)} ${taskLabel.padEnd(10)} ${f.priority.padEnd(10)} ${sections}`);
+        }
+        lines.push("");
+    }
+    // Uncovered features
+    if (report.uncovered.length > 0) {
+        lines.push("UNCOVERED FEATURES (by priority):");
+        for (const f of report.uncovered) {
+            const sections = f.sections.join(", ");
+            lines.push(`  ❌ ${f.id.padEnd(20)} ${f.priority.padEnd(10)} ${sections}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Format coverage audit as markdown.
+ */
+export function formatCoverageMarkdown(report) {
+    const lines = [];
+    lines.push("### 📊 Documentation Coverage Audit");
+    lines.push("");
+    lines.push(`**Coverage: ${report.covered.length}/${report.totalFeatures} features (${report.coveragePercent}%)**`);
+    lines.push("");
+    if (report.covered.length > 0) {
+        lines.push("#### Covered Features");
+        lines.push("");
+        lines.push("| Feature | Tasks | Priority | Sections |");
+        lines.push("|---------|-------|----------|----------|");
+        for (const f of report.covered) {
+            lines.push(`| ✅ ${f.name} | ${f.actualTaskCount} | ${f.priority} | ${f.sections.join(", ")} |`);
+        }
+        lines.push("");
+    }
+    if (report.uncovered.length > 0) {
+        lines.push("#### Uncovered Features");
+        lines.push("");
+        lines.push("| Feature | Priority | Sections |");
+        lines.push("|---------|----------|----------|");
+        for (const f of report.uncovered) {
+            lines.push(`| ❌ ${f.name} | ${f.priority} | ${f.sections.join(", ")} |`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Load and validate the feature registry from config/features.yaml.
+ */
+export function loadFeatureRegistry(rootDir) {
+    const filePath = join(rootDir, "config", "features.yaml");
+    if (!existsSync(filePath)) {
+        return null;
+    }
+    const raw = readFileSync(filePath, "utf-8");
+    const parsed = load(raw);
+    const result = FeatureRegistrySchema.safeParse(parsed);
+    if (!result.success) {
+        console.error("❌ config/features.yaml validation failed:");
+        for (const issue of result.error.issues) {
+            console.error(`  ${issue.path.join(".")}: ${issue.message}`);
+        }
+        return null;
+    }
+    return result.data.features;
+}
+/**
+ * Run the coverage audit and produce a structured report.
+ */
+export function runCoverageAudit(rootDir) {
+    const features = loadFeatureRegistry(rootDir);
+    if (!features)
+        return null;
+    const taskCounts = countTasksByArea(rootDir);
+    const totalFeatures = features.length;
+    const covered = [];
+    const uncovered = [];
+    for (const feature of features) {
+        if (feature.status === "covered" && feature.area) {
+            const actualTaskCount = taskCounts[feature.area] ?? 0;
+            covered.push({ ...feature, actualTaskCount });
+        }
+        else if (feature.status === "uncovered" || feature.status === "planned") {
+            uncovered.push(feature);
+        }
+    }
+    // Sort uncovered by priority
+    const priorityOrder = { critical: 0, high: 1, low: 3, medium: 2 };
+    uncovered.sort((a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]);
+    const coveredCount = covered.length;
+    const coveragePercent = totalFeatures > 0 ? (coveredCount / totalFeatures) * 100 : 0;
+    return {
+        coveragePercent: Math.round(coveragePercent * 10) / 10,
+        covered,
+        generatedAt: new Date().toISOString(),
+        totalFeatures,
+        uncovered,
+    };
+}

package/dist/pipeline/degradations.d.ts

@@ -0,0 +1,85 @@
+/**
+ * pipeline/degradations.ts
+ *
+ * Programmatic code degradation strategies for sensitivity testing.
+ *
+ * Each strategy takes a "good" reference solution (string) and returns a
+ * "bad" version that should score lower on a specific dimension:
+ *
+ * - Task Completion: remove key functional sections
+ * - Code Correctness: introduce anti-patterns and deprecated APIs
+ * - Doc Coverage: strip documentation references, add hallucinated details
+ *
+ * These are deterministic, pure functions — no randomness, no side effects.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
+ */
+/** A degradation targeting a specific scoring dimension */
+export interface Degradation {
+    /** Apply the degradation to source code */
+    apply: (source: string) => string;
+    /** Human-readable description of what this degradation does */
+    description: string;
+    /** Which dimension should score lower after degradation */
+    targetDimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+}
+/** Result of applying a degradation to a reference solution */
+export interface DegradedPair {
+    /** The degradation that was applied */
+    degradation: Degradation;
+    /** The degraded ("bad") version */
+    degraded: string;
+    /** The original ("good") reference solution */
+    original: string;
+    /** Reference solution file path (relative) */
+    sourcePath: string;
+}
+/**
+ * Remove the bottom half of the code (functions, exports, etc).
+ * A response missing half its functionality should score lower on Task Completion.
+ */
+export declare const removeBottomHalf: Degradation;
+/**
+ * Remove all export statements and exported functions.
+ * Missing exports = incomplete API surface → lower Task Completion.
+ */
+export declare const removeExports: Degradation;
+/**
+ * Replace function bodies with TODO comments.
+ * Skeleton code that doesn't actually implement anything.
+ */
+export declare const stubFunctions: Degradation;
+/**
+ * Replace modern API calls with deprecated/incorrect patterns.
+ * Targets Sanity-specific patterns that the grader should catch.
+ */
+export declare const introduceDeprecatedAPIs: Degradation;
+/**
+ * Introduce common GROQ syntax errors.
+ * Invalid queries that look plausible but wouldn't work.
+ */
+export declare const introduceGroqErrors: Degradation;
+/**
+ * Remove all TypeScript types and use any everywhere.
+ * Technically works but is an anti-pattern.
+ */
+export declare const removeTypes: Degradation;
+/**
+ * Remove all comments and documentation.
+ * The response shows no evidence of using documentation.
+ */
+export declare const stripComments: Degradation;
+/**
+ * Add hallucinated API calls and made-up configuration options.
+ * Response looks confident but uses APIs that don't exist.
+ */
+export declare const addHallucinations: Degradation;
+/**
+ * Replace correct import paths with plausible but wrong ones.
+ * Shows the response was guessing at the API surface.
+ */
+export declare const wrongImports: Degradation;
+/** All available degradation strategies, grouped by target dimension */
+export declare const DEGRADATION_STRATEGIES: Degradation[];
+/** Get degradation strategies targeting a specific dimension */
+export declare function getStrategiesForDimension(dimension: "codeCorrectness" | "docCoverage" | "taskCompletion"): Degradation[];