@sanity/ailf 0.1.0

package/dist/pipeline/failure-modes.js

@@ -0,0 +1,348 @@
+/**
+ * pipeline/failure-modes.ts
+ *
+ * Keyword-based failure mode classifier for grader reasoning text,
+ * cross-referenced with ceiling decomposition data.
+ *
+ * Phase 3a of the Scenario Matrix implementation.
+ *
+ * The classifier uses two signal sources:
+ * 1. Keyword matching on grader reason text (primary)
+ * 2. Ceiling decomposition structural signals (supplementary)
+ *
+ * When both sources agree, confidence is boosted. When only ceiling
+ * signals are available, they serve as a fallback for unclassified cases.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+import { detectFeatureArea } from "../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Only classify judgments with scores below this threshold */
+const CLASSIFICATION_THRESHOLD = 60;
+/** All failure mode types for initializing empty counts */
+const ALL_MODES = [
+    "incorrect-docs",
+    "missing-docs",
+    "model-limitation",
+    "outdated-docs",
+    "poor-structure",
+    "unclassified",
+];
+// ---------------------------------------------------------------------------
+// Keyword patterns
+// ---------------------------------------------------------------------------
+const OUTDATED_PATTERN = /deprecated|old api|v[0-9]+ syntax|no longer supported|legacy|previous version|outdated|superseded|replaced by/i;
+const MISSING_PATTERN = /no documentation|not covered|had to guess|not found|missing.*doc|no.*information|undocumented|couldn't find|without.*documentation/i;
+const INCORRECT_PATTERN = /contradicts|incorrect.*doc|doc.*incorrect|wrong.*doc|doc.*wrong|documentation says.*but|factual error|inaccurate|misleading.*doc/i;
+const POOR_STRUCTURE_PATTERN = /unclear|ambiguous|couldn't determine|conflicting|confusing|hard to follow|poorly organized|scattered|fragmented/i;
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Build a complete failure mode report from grader judgments and scores.
+ *
+ * @param judgments - All grader judgments from the evaluation
+ * @param scores - Per-area feature scores (for ceiling decomposition)
+ * @returns Failure mode report with per-area breakdowns
+ */
+export function buildFailureModeReport(judgments, scores) {
+    // Build a lookup from area → scores
+    const scoreByArea = new Map();
+    for (const score of scores) {
+        scoreByArea.set(score.feature, score);
+    }
+    const classifiedJudgments = [];
+    const summary = initModeCounts();
+    const byArea = {};
+    for (const judgment of judgments) {
+        // Extract area from taskId description (e.g., "GROQ - Blog queries..." → "groq")
+        const area = resolveArea(judgment.taskId, scoreByArea);
+        const areaScore = area ? scoreByArea.get(area) : undefined;
+        const ceilingScore = areaScore?.ceilingScore ?? 100;
+        const floorScore = areaScore?.floorScore ?? 0;
+        const classification = classifyFailureMode(judgment, ceilingScore, floorScore);
+        classifiedJudgments.push({ classification, judgment });
+        summary[classification.mode]++;
+        // Per-area tracking
+        if (area) {
+            if (!byArea[area]) {
+                byArea[area] = {
+                    area,
+                    modes: initModeCounts(),
+                    topMode: "unclassified",
+                    totalJudgments: 0,
+                };
+            }
+            byArea[area].modes[classification.mode]++;
+            byArea[area].totalJudgments++;
+        }
+    }
+    // Compute top mode per area
+    for (const areaData of Object.values(byArea)) {
+        areaData.topMode = findTopMode(areaData.modes);
+    }
+    const totalJudgments = judgments.length;
+    const classified = totalJudgments - (summary["unclassified"] ?? 0);
+    const classificationRate = totalJudgments > 0 ? (classified / totalJudgments) * 100 : 0;
+    return {
+        byArea,
+        classificationRate,
+        classifiedJudgments,
+        summary,
+        totalJudgments,
+    };
+}
+/**
+ * Classify the failure mode of a low-scoring grader judgment.
+ *
+ * Uses keyword matching on the reason text, then cross-references with
+ * ceiling decomposition data for structural confirmation.
+ *
+ * @param judgment - The grader judgment to classify
+ * @param ceilingScore - The area's ceiling score (with-docs best case)
+ * @param floorScore - The area's floor score (no-docs baseline)
+ * @returns Classified failure mode with confidence level
+ */
+export function classifyFailureMode(judgment, ceilingScore, floorScore) {
+    // Passing scores don't need failure mode analysis
+    if (judgment.score >= CLASSIFICATION_THRESHOLD) {
+        return { confidence: "low", mode: "unclassified", source: "keyword" };
+    }
+    const reason = judgment.reason.toLowerCase();
+    // Step 1: Keyword-based classification
+    const keywordMode = classifyByKeyword(reason);
+    // Step 2: Ceiling-based structural classification
+    const ceilingMode = classifyByCeiling(judgment.score, ceilingScore, floorScore);
+    // Step 3: Combine signals
+    return combineClassifications(keywordMode, ceilingMode);
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Format a failure mode report for console output.
+ */
+export function formatFailureModesConsole(report) {
+    const lines = [];
+    lines.push("🔍 FAILURE MODE ANALYSIS");
+    lines.push("");
+    lines.push(`  ${report.totalJudgments} judgments analyzed, ${report.classificationRate.toFixed(0)}% classified`);
+    lines.push("");
+    // Summary table
+    lines.push("  Mode                Count");
+    lines.push("  ──────────────────  ─────");
+    for (const mode of ALL_MODES) {
+        const count = report.summary[mode] ?? 0;
+        if (count > 0) {
+            const icon = modeIcon(mode);
+            lines.push(`  ${icon} ${mode.padEnd(18)} ${count}`);
+        }
+    }
+    lines.push("");
+    // Per-area breakdown
+    if (Object.keys(report.byArea).length > 0) {
+        lines.push("  Per-area top failure modes:");
+        for (const [area, data] of Object.entries(report.byArea).sort(([a], [b]) => a.localeCompare(b))) {
+            const icon = modeIcon(data.topMode);
+            lines.push(`    ${area}: ${icon} ${data.topMode} (${data.totalJudgments} judgments)`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Format a failure mode report as markdown for PR comments.
+ */
+export function formatFailureModesMarkdown(report) {
+    const lines = [];
+    lines.push("### 🔍 Failure Mode Analysis");
+    lines.push("");
+    if (report.totalJudgments === 0) {
+        lines.push("No judgments analyzed.");
+        return lines.join("\n");
+    }
+    lines.push(`**${report.totalJudgments} judgments** analyzed, **${report.classificationRate.toFixed(0)}%** classified`);
+    lines.push("");
+    // Summary table
+    lines.push("| Mode | Count | % |");
+    lines.push("|------|-------|---|");
+    for (const mode of ALL_MODES) {
+        const count = report.summary[mode] ?? 0;
+        if (count > 0) {
+            const pct = report.totalJudgments > 0
+                ? ((count / report.totalJudgments) * 100).toFixed(0)
+                : "0";
+            const icon = modeIcon(mode);
+            lines.push(`| ${icon} ${mode} | ${count} | ${pct}% |`);
+        }
+    }
+    lines.push("");
+    // Per-area table
+    if (Object.keys(report.byArea).length > 0) {
+        lines.push("<details>");
+        lines.push("<summary>Per-area breakdown</summary>");
+        lines.push("");
+        lines.push("| Area | Top Mode | Judgments |");
+        lines.push("|------|----------|----------|");
+        for (const [area, data] of Object.entries(report.byArea).sort(([a], [b]) => a.localeCompare(b))) {
+            const icon = modeIcon(data.topMode);
+            lines.push(`| ${area} | ${icon} ${data.topMode} | ${data.totalJudgments} |`);
+        }
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/** Classify by ceiling decomposition structural signals */
+function classifyByCeiling(score, ceilingScore, floorScore) {
+    const docLift = ceilingScore - floorScore;
+    // Negative Doc Lift: docs are actively harmful
+    if (docLift < 0) {
+        return { confidence: "medium", mode: "outdated-docs", source: "ceiling" };
+    }
+    // High ceiling, low actual: model can't use good docs → model limitation
+    if (ceilingScore > 70 && score < 40) {
+        return {
+            confidence: "medium",
+            mode: "model-limitation",
+            source: "ceiling",
+        };
+    }
+    // High floor, low ceiling: model knows better than docs → outdated
+    if (floorScore > 50 && ceilingScore < floorScore + 10) {
+        return { confidence: "medium", mode: "outdated-docs", source: "ceiling" };
+    }
+    // Low ceiling, low floor: both docs and model knowledge insufficient
+    if (ceilingScore < 40 && floorScore < 30) {
+        return { confidence: "low", mode: "missing-docs", source: "ceiling" };
+    }
+    return null;
+}
+/** Classify by keyword matching on the reason text */
+function classifyByKeyword(reason) {
+    if (OUTDATED_PATTERN.test(reason)) {
+        return { confidence: "high", mode: "outdated-docs", source: "keyword" };
+    }
+    if (MISSING_PATTERN.test(reason)) {
+        return { confidence: "high", mode: "missing-docs", source: "keyword" };
+    }
+    if (INCORRECT_PATTERN.test(reason)) {
+        return { confidence: "medium", mode: "incorrect-docs", source: "keyword" };
+    }
+    if (POOR_STRUCTURE_PATTERN.test(reason)) {
+        return { confidence: "medium", mode: "poor-structure", source: "keyword" };
+    }
+    return null;
+}
+/**
+ * Combine keyword and ceiling classifications.
+ *
+ * Priority:
+ * 1. If both agree on mode → high confidence, source = "keyword+ceiling"
+ * 2. If keyword matched → use keyword result
+ * 3. If only ceiling matched → use ceiling result (lower confidence)
+ * 4. If neither matched → unclassified
+ */
+function combineClassifications(keyword, ceiling) {
+    if (keyword && ceiling) {
+        if (keyword.mode === ceiling.mode) {
+            // Both agree — boost confidence
+            return {
+                confidence: "high",
+                mode: keyword.mode,
+                source: "keyword+ceiling",
+            };
+        }
+        // Disagree — prefer keyword (it has more signal)
+        return keyword;
+    }
+    if (keyword)
+        return keyword;
+    if (ceiling)
+        return ceiling;
+    return { confidence: "low", mode: "unclassified", source: "keyword" };
+}
+/**
+ * Resolve area name from a task ID or description.
+ *
+ * Task IDs in grader judgments use human-readable descriptions
+ * (e.g., "GROQ - Blog queries with filtering and pagination (gold)")
+ * while score areas use kebab-case slugs (e.g., "groq").
+ *
+ * Strategy:
+ * 1. Use detectFeatureArea() which handles human-readable descriptions
+ * 2. Fall back to prefix matching for kebab-case task IDs
+ * 3. Return undefined if no match is found
+ */
+function resolveArea(taskId, scoreByArea) {
+    // Strategy 1: Use the shared feature area detector (handles descriptions)
+    const detected = detectFeatureArea(taskId);
+    if (detected !== "other" && scoreByArea.has(detected))
+        return detected;
+    // Strategy 2: Direct prefix match (kebab-case task IDs like "groq-blog-queries")
+    for (const area of scoreByArea.keys()) {
+        if (taskId.startsWith(area))
+            return area;
+    }
+    // Strategy 3: Progressive prefix matching on hyphens
+    const parts = taskId.split("-");
+    if (parts.length > 1) {
+        for (let i = parts.length - 1; i >= 1; i--) {
+            const candidate = parts.slice(0, i).join("-");
+            if (scoreByArea.has(candidate))
+                return candidate;
+        }
+    }
+    return undefined;
+}
+/** Find the most common failure mode */
+function findTopMode(modes) {
+    let topMode = "unclassified";
+    let topCount = 0;
+    for (const mode of ALL_MODES) {
+        if (mode === "unclassified")
+            continue; // Prefer classified modes
+        if ((modes[mode] ?? 0) > topCount) {
+            topCount = modes[mode];
+            topMode = mode;
+        }
+    }
+    // If nothing classified, return unclassified
+    if (topCount === 0)
+        return "unclassified";
+    return topMode;
+}
+/** Initialize mode counts to zero */
+function initModeCounts() {
+    return {
+        "incorrect-docs": 0,
+        "missing-docs": 0,
+        "model-limitation": 0,
+        "outdated-docs": 0,
+        "poor-structure": 0,
+        unclassified: 0,
+    };
+}
+/** Get icon for a failure mode */
+function modeIcon(mode) {
+    switch (mode) {
+        case "incorrect-docs":
+            return "❌";
+        case "missing-docs":
+            return "📭";
+        case "model-limitation":
+            return "🤖";
+        case "outdated-docs":
+            return "📅";
+        case "poor-structure":
+            return "🏗️";
+        case "unclassified":
+            return "❓";
+    }
+}

package/dist/pipeline/fetch-url-content.d.ts

@@ -0,0 +1,44 @@
+/**
+ * fetch-url-content.ts
+ *
+ * Fetches documentation content from a URL. Tries the .md endpoint first
+ * (Sanity's agent-friendly format), then content-type negotiation.
+ * Does NOT attempt HTML-to-Markdown conversion — if the endpoint
+ * doesn't serve markdown, the fetch fails cleanly.
+ *
+ * Part of Phase 4: Modular doc fetching for baseline mode.
+ */
+/** Metadata for a single URL fetch, suitable for JSON serialization */
+export interface UrlFetchMetadata {
+    contentLength?: number;
+    error?: string;
+    method: UrlFetchResult["method"];
+    status?: number;
+    url: string;
+}
+/** Result of fetching documentation content from a URL */
+export interface UrlFetchResult {
+    /** The markdown content, or undefined if fetch failed */
+    content?: string;
+    /** Error message if fetch failed */
+    error?: string;
+    /** How the content was obtained */
+    method: "content-negotiation" | "failed" | "md-endpoint";
+    /** HTTP status code */
+    status?: number;
+    /** The URL that was fetched */
+    url: string;
+}
+/**
+ * Fetch documentation content from a URL.
+ *
+ * Strategy:
+ *   1. Try `.md` endpoint with `Accept: text/markdown`
+ *   2. Try content-type negotiation (ask for markdown)
+ *   3. Fail cleanly if only HTML is available (no conversion)
+ *
+ * @param url - The documentation URL to fetch
+ * @param headers - Optional custom headers to merge with defaults
+ * @returns A `UrlFetchResult` with success or failure metadata
+ */
+export declare function fetchUrlContent(url: string, headers?: Record<string, string>): Promise<UrlFetchResult>;

package/dist/pipeline/fetch-url-content.js

@@ -0,0 +1,93 @@
+/**
+ * fetch-url-content.ts
+ *
+ * Fetches documentation content from a URL. Tries the .md endpoint first
+ * (Sanity's agent-friendly format), then content-type negotiation.
+ * Does NOT attempt HTML-to-Markdown conversion — if the endpoint
+ * doesn't serve markdown, the fetch fails cleanly.
+ *
+ * Part of Phase 4: Modular doc fetching for baseline mode.
+ */
+/**
+ * Fetch documentation content from a URL.
+ *
+ * Strategy:
+ *   1. Try `.md` endpoint with `Accept: text/markdown`
+ *   2. Try content-type negotiation (ask for markdown)
+ *   3. Fail cleanly if only HTML is available (no conversion)
+ *
+ * @param url - The documentation URL to fetch
+ * @param headers - Optional custom headers to merge with defaults
+ * @returns A `UrlFetchResult` with success or failure metadata
+ */
+export async function fetchUrlContent(url, headers) {
+    const cleanUrl = url.replace(/\/$/, "");
+    const mergedHeaders = {
+        ...headers,
+        "User-Agent": "SanityEvalBot/1.0",
+    };
+    // Strategy 1: Try .md endpoint
+    const mdUrl = cleanUrl.endsWith(".md") ? cleanUrl : `${cleanUrl}.md`;
+    try {
+        const response = await fetch(mdUrl, {
+            headers: { ...mergedHeaders, Accept: "text/markdown, text/plain" },
+        });
+        if (response.ok) {
+            const contentType = response.headers.get("content-type") ?? "";
+            const text = await response.text();
+            // Accept if content-type says markdown OR the content doesn't look like HTML
+            if (contentType.includes("markdown") ||
+                !text.trimStart().startsWith("<!DOCTYPE")) {
+                return {
+                    content: text,
+                    method: "md-endpoint",
+                    status: response.status,
+                    url,
+                };
+            }
+        }
+    }
+    catch {
+        // .md endpoint unavailable — try next strategy
+    }
+    // Strategy 2: Content-type negotiation (ask for markdown)
+    try {
+        const response = await fetch(cleanUrl, {
+            headers: { ...mergedHeaders, Accept: "text/markdown, text/plain" },
+        });
+        if (response.ok) {
+            const contentType = response.headers.get("content-type") ?? "";
+            const text = await response.text();
+            if (contentType.includes("markdown") ||
+                contentType.includes("text/plain")) {
+                return {
+                    content: text,
+                    method: "content-negotiation",
+                    status: response.status,
+                    url,
+                };
+            }
+            // Got HTML or something else — don't try to convert it
+            return {
+                error: `Endpoint returned ${contentType} instead of markdown. ` +
+                    "The URL does not appear to support markdown content negotiation.",
+                method: "failed",
+                status: response.status,
+                url,
+            };
+        }
+        return {
+            error: `HTTP ${response.status}`,
+            method: "failed",
+            status: response.status,
+            url,
+        };
+    }
+    catch (err) {
+        return {
+            error: `Network error: ${err.message}`,
+            method: "failed",
+            url,
+        };
+    }
+}

package/dist/pipeline/gap-analysis.d.ts

@@ -0,0 +1,48 @@
+/**
+ * pipeline/gap-analysis.ts
+ *
+ * Expected impact estimation for identified gaps.
+ *
+ * Phase 3b of the Scenario Matrix implementation.
+ *
+ * Given failure modes and scores, estimates the score lift that fixing
+ * each gap would produce. Gaps are prioritized by estimated lift × task count
+ * to produce an actionable remediation plan.
+ *
+ * The estimation model is conservative: it assumes fixing a gap raises the
+ * bottleneck dimension to the median of non-bottlenecked dimensions (not 100).
+ * This produces realistic estimates rather than theoretical maximums.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+import type { FailureModeReport, FeatureScore, GapAnalysisReport, GapEstimate } from "./types.js";
+/**
+ * Build a complete gap analysis report.
+ *
+ * @param failureModeReport - Classified failure modes from Phase 3a
+ * @param scores - Per-area feature scores
+ * @param weights - Dimension weights
+ * @returns Gap analysis report with prioritized remediation plan
+ */
+export declare function buildGapAnalysisReport(failureModeReport: FailureModeReport, scores: FeatureScore[], weights?: Record<string, number>): GapAnalysisReport;
+/**
+ * Estimate the impact of fixing each identified gap.
+ *
+ * For each area with failure modes, calculates the potential score lift
+ * from fixing the identified issues. Uses a conservative estimation model
+ * that targets the median of non-bottlenecked dimensions rather than 100.
+ *
+ * @param failureModeReport - Classified failure modes from Phase 3a
+ * @param scores - Per-area feature scores
+ * @param weights - Dimension weights (defaults to rubrics.yaml weights)
+ * @returns Gap estimates sorted by priority (highest first)
+ */
+export declare function estimateImpact(failureModeReport: FailureModeReport, scores: FeatureScore[], weights?: Record<string, number>): GapEstimate[];
+/**
+ * Format a gap analysis report for console output.
+ */
+export declare function formatGapAnalysisConsole(report: GapAnalysisReport): string;
+/**
+ * Format a gap analysis report as markdown for PR comments.
+ */
+export declare function formatGapAnalysisMarkdown(report: GapAnalysisReport): string;