@sanity/ailf 0.1.0

package/dist/pipeline/pr-comment.d.ts

@@ -0,0 +1,19 @@
+/**
+ * pipeline/pr-comment.ts — Generates a markdown PR comment from eval score-summary.json.
+ *
+ * All functions accept rootDir as a parameter — no module-level constants.
+ * No process.argv parsing. No env var fallbacks.
+ *
+ * Reads: results/latest/score-summary.json
+ * Writes: markdown to stdout or --output file
+ */
+/** Options for the generatePrComment() function. */
+export interface PrCommentOptions {
+    /** Path to write the comment (default: stdout) */
+    outputPath?: string;
+    /** Promptfoo share URL to include in the comment */
+    promptfooUrl?: string;
+    /** Root directory of the eval package */
+    rootDir: string;
+}
+export declare function generatePrComment(options: PrCommentOptions): void;

package/dist/pipeline/pr-comment.js

@@ -0,0 +1,502 @@
+/**
+ * pipeline/pr-comment.ts — Generates a markdown PR comment from eval score-summary.json.
+ *
+ * All functions accept rootDir as a parameter — no module-level constants.
+ * No process.argv parsing. No env var fallbacks.
+ *
+ * Reads: results/latest/score-summary.json
+ * Writes: markdown to stdout or --output file
+ */
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+function formatCost(cost) {
+    if (cost === 0) {
+        return "$0.00";
+    }
+    if (cost < 0.01) {
+        return `$${cost.toFixed(4)}`;
+    }
+    return `$${cost.toFixed(2)}`;
+}
+function generateComment(summary, options = {}) {
+    const { belowCritical, overall, scores, source, timestamp } = summary;
+    const sorted = [...scores].sort((a, b) => b.totalScore - a.totalScore);
+    const lines = [];
+    // Header
+    lines.push(`## ${overallEmoji(overall.avgScore)} AI Literacy Score Report`);
+    lines.push("");
+    const totalCost = scores.reduce((sum, s) => sum + (s.totalCost ?? 0), 0);
+    const graderCostValue = overall.cost?.graderTotal ?? 0;
+    const combinedCostValue = totalCost + graderCostValue;
+    const costStr = combinedCostValue > 0 ? ` · Cost: ${formatCost(combinedCostValue)}` : "";
+    const actualStr = overall.avgActualScore !== undefined
+        ? ` · Actual: ${Math.round(overall.avgActualScore)}/100`
+        : "";
+    const retGapStr = overall.avgRetrievalGap !== undefined
+        ? ` · Ret. Gap: ${Math.round(overall.avgRetrievalGap)}`
+        : "";
+    lines.push(`**Overall: ${Math.round(overall.avgScore)}/100** · Doc Lift: +${Math.round(overall.avgDocLift)}${actualStr}${retGapStr} · ${scores.reduce((sum, s) => sum + s.testCount, 0)} tests across ${scores.length} areas${costStr}`);
+    lines.push("");
+    // Critical warnings
+    if (belowCritical.length > 0) {
+        lines.push(`> ⚠️ **Below critical threshold:** ${belowCritical.map((a) => `\`${a}\``).join(", ")}`);
+        lines.push("");
+    }
+    // Environment info
+    if (source) {
+        lines.push("<details>");
+        lines.push("<summary>🔧 Environment</summary>");
+        lines.push("");
+        lines.push("| Setting | Value |");
+        lines.push("|---------|-------|");
+        lines.push(`| **Source** | ${source.name} |`);
+        lines.push(`| **Docs URL** | ${source.baseUrl} |`);
+        if (source.dataset) {
+            lines.push(`| **Dataset** | ${source.dataset} |`);
+        }
+        lines.push(`| **Project** | ${source.projectId} |`);
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    // Source verification
+    const { sourceIsolation, sourceVerification } = summary;
+    if (sourceVerification || sourceIsolation) {
+        lines.push("<details>");
+        lines.push("<summary>🔍 Source verification</summary>");
+        lines.push("");
+        lines.push("| Setting | Value |");
+        lines.push("|---------|-------|");
+        if (sourceVerification) {
+            lines.push(`| **Source** | ${sourceVerification.source} |`);
+            lines.push(`| **Mode** | ${sourceVerification.mode} |`);
+            if (sourceVerification.allowedOrigins) {
+                lines.push(`| **Sandbox** | ${sourceVerification.allowedOrigins.join(", ")} |`);
+            }
+            if (sourceVerification.searchMode) {
+                lines.push(`| **Search** | ${sourceVerification.searchMode} |`);
+            }
+        }
+        if (sourceIsolation) {
+            const pct = Math.round(sourceIsolation.isolationScore * 100);
+            const icon = sourceIsolation.offOrigin === 0 ? "✅" : "⚠️";
+            lines.push(`| **Agent isolation** | ${icon} ${pct}% (${sourceIsolation.onOrigin}/${sourceIsolation.total} on-origin) |`);
+            if (sourceIsolation.offOrigin > 0) {
+                lines.push(`| **Off-origin fetches** | ${sourceIsolation.offOriginUrls.slice(0, 5).join(", ")} |`);
+            }
+        }
+        if (sourceVerification?.urlFetch) {
+            const uf = sourceVerification.urlFetch;
+            lines.push(`| **URL fetch** | ${uf.totalFetched} fetched, ${uf.totalFailed} failed |`);
+            for (const f of uf.fetchedUrls) {
+                lines.push(`| | ✅ ${f.url} (via ${f.method}) |`);
+            }
+            for (const f of uf.failures) {
+                // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty error string should show "unknown"
+                lines.push(`| | ⚠️ ${f.url}: ${f.error || "unknown"} |`);
+            }
+        }
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    // Score table
+    lines.push("### Scores by Feature Area");
+    lines.push("");
+    lines.push("| Feature | Score | Grade | Task | Code | Docs | Doc Lift | Tests |");
+    lines.push("|---------|-------|-------|------|------|------|----------|-------|");
+    for (const s of sorted) {
+        lines.push(`| ${s.feature} | **${Math.round(s.totalScore)}** | ${gradeEmoji(s.totalScore)} ${gradeLetter(s.totalScore)} | ${Math.round(s.taskCompletion)} | ${Math.round(s.codeCorrectness)} | ${Math.round(s.docCoverage)} | ${liftArrow(s.docLift)} | ${s.testCount} |`);
+    }
+    lines.push("");
+    // Breakdown legend
+    lines.push("<details>");
+    lines.push("<summary>📊 Score breakdown legend</summary>");
+    lines.push("");
+    lines.push("| Column | Description |");
+    lines.push("|--------|-------------|");
+    lines.push("| **Score** | Overall AI literacy score (0–100) — weighted sum of Task + Code + Docs |");
+    lines.push("| **Task** | Task completion — does the LLM understand what to build? |");
+    lines.push("| **Code** | Code correctness — does the generated code actually work? |");
+    lines.push("| **Docs** | Documentation coverage — are the right APIs/patterns referenced? |");
+    lines.push("| **Doc Lift** | Score improvement when docs are provided vs baseline (no docs) |");
+    lines.push("| **Grade** | ✅ A (≥80) · 🟡 B (≥70) · 🟠 C (≥50) · 🔴 D (<50) |");
+    lines.push("");
+    lines.push("</details>");
+    lines.push("");
+    // Negative Doc Lift warning (always visible, not collapsible)
+    const negDocLiftAreas = sorted.filter((s) => s.negativeDocLift);
+    if (negDocLiftAreas.length > 0) {
+        for (const s of negDocLiftAreas) {
+            lines.push(`> 🚨 **Negative Doc Lift:** \`${s.feature}\` (${s.docLift}) — docs hurt performance. Floor: ${s.floorScore}, Ceiling: ${s.ceilingScore}`);
+        }
+        lines.push("");
+    }
+    // Three-layer decomposition (when actual scores are present)
+    const hasActualScores = sorted.some((s) => s.actualScore !== undefined);
+    if (hasActualScores) {
+        // Full decomposition — show all three layers prominently
+        lines.push("### 🔬 Three-Layer Decomposition");
+        lines.push("");
+        lines.push("| Feature | Floor | Ceiling | Actual | Doc Lift | Retr. Gap | Infra % |");
+        lines.push("|---------|-------|---------|--------|----------|-----------|---------|");
+        for (const s of sorted) {
+            const actualStr = s.actualScore !== undefined ? String(s.actualScore) : "—";
+            const gapStr = s.retrievalGap !== undefined
+                ? s.retrievalGap >= 0
+                    ? `+${s.retrievalGap}`
+                    : String(s.retrievalGap)
+                : "—";
+            const infraStr = s.infrastructureEfficiency != null
+                ? `${Math.round(s.infrastructureEfficiency * 100)}%`
+                : "—";
+            const flag = s.invertedRetrievalGap ? " 🔄" : "";
+            lines.push(`| ${s.feature} | ${s.floorScore} | ${s.ceilingScore} | ${actualStr} | ${liftArrow(s.docLift)} | ${gapStr}${flag} | ${infraStr} |`);
+        }
+        lines.push("");
+        // Decomposition guide
+        lines.push("<details>");
+        lines.push("<summary>📖 What do these numbers mean?</summary>");
+        lines.push("");
+        lines.push("- **Floor:** How well models do without any documentation (just training data)");
+        lines.push("- **Ceiling:** How well models do with perfect documentation (hand-picked, injected)");
+        lines.push("- **Actual:** How well models do when they have to find docs themselves (like real users)");
+        lines.push("- **Doc Lift:** How much docs help (Ceiling − Floor). Negative = docs hurt");
+        lines.push("- **Retr. Gap:** How much quality is lost in discovery (Ceiling − Actual)");
+        lines.push("- **Infra %:** What fraction of doc quality reaches agents (Actual ÷ Ceiling)");
+        if (sorted.some((s) => s.invertedRetrievalGap)) {
+            lines.push("- **🔄:** Inverted retrieval gap — agents avoid bad docs, scoring higher than ceiling");
+        }
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    else {
+        // Baseline-only — show the existing ceiling decomposition
+        lines.push("<details>");
+        lines.push("<summary>📊 Ceiling decomposition</summary>");
+        lines.push("");
+        lines.push("| Feature | Floor | Ceiling | Doc Lift | Quality Gap |");
+        lines.push("|---------|-------|---------|----------|-------------|");
+        for (const s of sorted) {
+            lines.push(`| ${s.feature} | ${s.floorScore} | ${s.ceilingScore} | ${liftArrow(s.docLift)} | ${s.docQualityGap} |`);
+        }
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+        // With vs without docs comparison (uses ceiling/floor model)
+        lines.push("<details>");
+        lines.push("<summary>📄 With docs vs without docs</summary>");
+        lines.push("");
+        lines.push("| Feature | With Docs | Without Docs | Lift |");
+        lines.push("|---------|-----------|--------------|------|");
+        for (const s of sorted) {
+            lines.push(`| ${s.feature} | ${s.ceilingScore} | ${s.floorScore} | ${liftArrow(s.docLift)} |`);
+        }
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    // Cost breakdown (only when cost data is available)
+    if (totalCost > 0 || overall.cost) {
+        const graderCost = overall.cost?.graderTotal ?? 0;
+        const combinedCost = totalCost + graderCost;
+        lines.push("<details>");
+        lines.push("<summary>💰 Eval cost breakdown</summary>");
+        lines.push("");
+        lines.push("| Category | Cost |");
+        lines.push("|----------|------|");
+        lines.push(`| Provider (model inference) | ${formatCost(totalCost)} |`);
+        if (graderCost > 0) {
+            const graderLabel = overall.cost?.graderModel ?? "unknown";
+            lines.push(`| Grader (${graderLabel}) | ${formatCost(graderCost)} |`);
+        }
+        lines.push(`| **Total** | **${formatCost(combinedCost)}** |`);
+        lines.push("");
+        // Per-feature provider cost breakdown
+        lines.push("**Provider cost by feature area:**");
+        lines.push("");
+        lines.push("| Feature | Tests | Cost | Avg/Test |");
+        lines.push("|---------|-------|------|----------|");
+        for (const s of sorted) {
+            const avgCost = s.testCount > 0 ? s.totalCost / s.testCount : 0;
+            lines.push(`| ${s.feature} | ${s.testCount} | ${formatCost(s.totalCost)} | ${formatCost(avgCost)} |`);
+        }
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    // Per-model breakdown (when multiple models were evaluated)
+    if (summary.perModel && summary.perModel.length > 1) {
+        const sorted = [...summary.perModel].sort((a, b) => b.overall.avgScore - a.overall.avgScore);
+        lines.push("<details>");
+        lines.push("<summary>🤖 Per-model scores</summary>");
+        lines.push("");
+        lines.push("| Model | Score | Doc Lift | Tests | Cost |");
+        lines.push("|-------|-------|----------|-------|------|");
+        for (const entry of sorted) {
+            const displayName = entry.label || entry.modelId;
+            const costStr = entry.overall.cost ? formatCost(entry.overall.cost) : "—";
+            const liftStr = entry.overall.avgDocLift >= 0
+                ? `+${Math.round(entry.overall.avgDocLift)}`
+                : String(Math.round(entry.overall.avgDocLift));
+            lines.push(`| ${displayName} | **${Math.round(entry.overall.avgScore)}** | ${liftStr} | ${entry.overall.testCount} | ${costStr} |`);
+        }
+        lines.push("");
+        // Per-model × per-area table
+        for (const entry of sorted) {
+            const displayName = entry.label || entry.modelId;
+            lines.push(`**${displayName}** (${entry.modelId}):`);
+            lines.push("");
+            lines.push("| Feature | Score | Task | Code | Docs | Lift |");
+            lines.push("|---------|-------|------|------|------|------|");
+            for (const s of entry.scores) {
+                const lift = s.docLift >= 0 ? `+${s.docLift}` : String(s.docLift);
+                lines.push(`| ${s.feature} | **${s.totalScore}** | ${s.taskCompletion} | ${s.codeCorrectness} | ${s.docCoverage} | ${lift} |`);
+            }
+            lines.push("");
+        }
+        // Cost per quality point
+        const withCost = sorted.filter((e) => e.overall.cost && e.overall.cost > 0);
+        if (withCost.length > 0) {
+            lines.push("**Cost efficiency:**");
+            lines.push("");
+            lines.push("| Model | $/point | Score | Cost |");
+            lines.push("|-------|---------|-------|------|");
+            for (const entry of withCost) {
+                const costPerPoint = entry.overall.avgScore > 0
+                    ? (entry.overall.cost ?? 0) / entry.overall.avgScore
+                    : 0;
+                lines.push(`| ${entry.label} | ${formatCost(costPerPoint)} | ${Math.round(entry.overall.avgScore)} | ${formatCost(entry.overall.cost ?? 0)} |`);
+            }
+            lines.push("");
+        }
+        lines.push("</details>");
+        lines.push("");
+    }
+    // Comparison section (when --compare was used)
+    if (options.comparisonReport) {
+        const report = options.comparisonReport;
+        const overallDelta = report.deltas.overall;
+        const overallDeltaStr = overallDelta > 0
+            ? `+${Math.round(overallDelta)}`
+            : String(Math.round(overallDelta));
+        const overallChangeIcon = overallDelta > report.noiseThreshold
+            ? "📈"
+            : overallDelta < -report.noiseThreshold
+                ? "📉"
+                : "➡️";
+        lines.push("### 📊 Score Comparison");
+        lines.push("");
+        lines.push(`**Overall: ${Math.round(report.baseline.overall.avgScore)} → ${Math.round(report.experiment.overall.avgScore)}** (${overallChangeIcon} ${overallDeltaStr})`);
+        lines.push("");
+        // Check if comparison data includes actual/retrieval gap deltas
+        const hasActualDeltas = report.areas.some((a) => a.actualDelta !== undefined);
+        if (hasActualDeltas) {
+            lines.push("| Feature | Baseline | Current | Delta | Actual Δ | Ret. Gap Δ | Infra Δ |");
+            lines.push("|---------|----------|---------|-------|----------|------------|---------|");
+            for (const a of report.areas) {
+                const icon = a.change === "improved"
+                    ? "📈"
+                    : a.change === "regressed"
+                        ? "📉"
+                        : "➡️";
+                const d = (n) => n > 0 ? `+${Math.round(n)}` : String(Math.round(n));
+                const actualStr = a.actualDelta !== undefined ? d(a.actualDelta) : "—";
+                const retGapStr = a.retrievalGapDelta !== undefined ? d(a.retrievalGapDelta) : "—";
+                const infraStr = a.infrastructureEfficiencyDelta !== undefined
+                    ? `${a.infrastructureEfficiencyDelta > 0 ? "+" : ""}${Math.round(a.infrastructureEfficiencyDelta * 100)}pp`
+                    : "—";
+                lines.push(`| ${a.area} | ${a.baseline} | ${a.experiment} | ${icon} ${d(a.delta)} | ${actualStr} | ${retGapStr} | ${infraStr} |`);
+            }
+        }
+        else {
+            lines.push("| Feature | Baseline | Current | Delta | Task | Code | Docs |");
+            lines.push("|---------|----------|---------|-------|------|------|------|");
+            for (const a of report.areas) {
+                const icon = a.change === "improved"
+                    ? "📈"
+                    : a.change === "regressed"
+                        ? "📉"
+                        : "➡️";
+                const d = (n) => n > 0 ? `+${Math.round(n)}` : String(Math.round(n));
+                lines.push(`| ${a.area} | ${a.baseline} | ${a.experiment} | ${icon} ${d(a.delta)} | ${d(a.dimensions.taskCompletion.delta)} | ${d(a.dimensions.codeCorrectness.delta)} | ${d(a.dimensions.docCoverage.delta)} |`);
+            }
+        }
+        lines.push("");
+        const parts = [];
+        if (report.improved.length > 0) {
+            parts.push(`📈 ${report.improved.length} improved`);
+        }
+        if (report.regressed.length > 0) {
+            parts.push(`📉 ${report.regressed.length} regressed`);
+        }
+        if (report.unchanged.length > 0) {
+            parts.push(`➡️ ${report.unchanged.length} unchanged`);
+        }
+        if (parts.length > 0) {
+            const isEmpirical = "noiseThresholdEmpirical" in report &&
+                report.noiseThresholdEmpirical === true;
+            const thresholdNote = isEmpirical
+                ? ` (empirical threshold: ±${report.noiseThreshold.toFixed(1)})`
+                : ` (threshold: ±${report.noiseThreshold})`;
+            lines.push(parts.join(" · ") + thresholdNote);
+            lines.push("");
+        }
+        // Dimension averages in collapsible
+        lines.push("<details>");
+        lines.push("<summary>Dimension averages</summary>");
+        lines.push("");
+        const dim = report.deltas.perDimension;
+        const dd = (n) => n > 0 ? `+${Math.round(n)}` : String(Math.round(n));
+        lines.push("| Dimension | Delta |");
+        lines.push("|-----------|-------|");
+        lines.push(`| Task Completion | ${dd(dim.taskCompletion)} |`);
+        lines.push(`| Code Correctness | ${dd(dim.codeCorrectness)} |`);
+        lines.push(`| Doc Coverage | ${dd(dim.docCoverage)} |`);
+        lines.push(`| Doc Lift | ${dd(report.deltas.docLift)} |`);
+        lines.push("");
+        lines.push("</details>");
+        lines.push("");
+    }
+    // Recommendations
+    if (belowCritical.length > 0 || sorted.some((s) => s.totalScore < 70)) {
+        lines.push("### 💡 Recommendations");
+        lines.push("");
+        for (const s of sorted) {
+            if (s.totalScore < 50) {
+                lines.push(`- 🔴 **${s.feature}** (score: ${Math.round(s.totalScore)}) — needs significant doc improvements. `);
+                if (s.codeCorrectness < 10) {
+                    lines.push(`  Code correctness is very low (${Math.round(s.codeCorrectness)}) — add more complete code examples.`);
+                }
+                if (s.docCoverage < 10) {
+                    lines.push(`  Doc coverage is very low (${Math.round(s.docCoverage)}) — key APIs/patterns may be missing from docs.`);
+                }
+            }
+            else if (s.totalScore < 70) {
+                lines.push(`- 🟠 **${s.feature}** (score: ${Math.round(s.totalScore)}) — has room for improvement.`);
+                if (s.codeCorrectness < 15) {
+                    lines.push(`  Code correctness (${Math.round(s.codeCorrectness)}) could improve with better code examples.`);
+                }
+            }
+        }
+        lines.push("");
+    }
+    // Footer
+    const date = new Date(timestamp).toLocaleString("en-US", {
+        day: "numeric",
+        hour: "numeric",
+        minute: "2-digit",
+        month: "short",
+        timeZone: "UTC",
+        timeZoneName: "short",
+        year: "numeric",
+    });
+    lines.push("---");
+    const runUrl = process.env.GITHUB_RUN_URL ?? "";
+    const runLink = runUrl ? ` · <a href="${runUrl}">view run</a>` : "";
+    const promptfooLink = options.promptfooUrl
+        ? ` · <a href="${options.promptfooUrl}">view detailed results</a>`
+        : "";
+    lines.push(`<sub>🤖 Generated by AI Literacy Framework · ${date}${runLink}${promptfooLink} · <a href="https://github.com/sanity-labs/ai-literacy-framework">docs</a></sub>`);
+    return lines.join("\n");
+}
+function gradeEmoji(score) {
+    if (score >= 80) {
+        return "✅";
+    }
+    if (score >= 70) {
+        return "🟡";
+    }
+    if (score >= 50) {
+        return "🟠";
+    }
+    return "🔴";
+}
+function gradeLetter(score) {
+    if (score >= 80) {
+        return "A";
+    }
+    if (score >= 70) {
+        return "B";
+    }
+    if (score >= 50) {
+        return "C";
+    }
+    return "D";
+}
+// ── Main ───────────────────────────────────────────────────────────────
+function liftArrow(lift) {
+    const rounded = Math.round(lift);
+    if (rounded > 0) {
+        return `📈 +${rounded}`;
+    }
+    if (rounded < 0) {
+        return `📉 ${rounded}`;
+    }
+    return "➡️ 0";
+}
+function overallEmoji(avg) {
+    if (avg >= 75) {
+        return "🟢";
+    }
+    if (avg >= 60) {
+        return "🟡";
+    }
+    if (avg >= 45) {
+        return "🟠";
+    }
+    return "🔴";
+}
+/** Normalize legacy field names in a score object */
+function normalizeScore(s) {
+    const ceiling = s.ceilingScore ??
+        s.withDocsScore ??
+        s.totalScore ??
+        0;
+    const floor = s.floorScore ??
+        s.withoutDocsScore ??
+        0;
+    const lift = s.docLift ??
+        s.liftFromDocs ??
+        ceiling - floor;
+    return {
+        ...s,
+        ceilingScore: ceiling,
+        docLift: lift,
+        docQualityGap: s.docQualityGap ?? 100 - ceiling,
+        floorScore: floor,
+        negativeDocLift: s.negativeDocLift ?? lift < 0,
+    };
+}
+// ── Main ──────────────────────────────────────────────────────────────
+export function generatePrComment(options) {
+    const { outputPath, promptfooUrl, rootDir } = options;
+    const summaryPath = resolve(rootDir, "results/latest/score-summary.json");
+    const comparisonPath = resolve(rootDir, "results/latest/comparison-report.json");
+    const raw = readFileSync(summaryPath, "utf-8");
+    const parsed = JSON.parse(raw);
+    // Normalize legacy field names in scores
+    const summary = {
+        ...parsed,
+        scores: parsed.scores.map((s) => normalizeScore(s)),
+    };
+    // Load comparison report if it exists (produced by --compare flag)
+    let comparisonReport;
+    if (existsSync(comparisonPath)) {
+        try {
+            const compRaw = readFileSync(comparisonPath, "utf-8");
+            comparisonReport = JSON.parse(compRaw);
+        }
+        catch {
+            // Non-fatal — comparison report is optional
+        }
+    }
+    const comment = generateComment(summary, { comparisonReport, promptfooUrl });
+    if (outputPath) {
+        writeFileSync(outputPath, comment);
+    }
+    else {
+        process.stdout.write(comment);
+    }
+}

package/dist/pipeline/probe.d.ts

@@ -0,0 +1,52 @@
+/**
+ * pipeline/probe.ts
+ *
+ * Tier B probes for new features without evaluation tasks.
+ *
+ * Phase 4b of the Scenario Matrix implementation.
+ *
+ * When a content release adds documents for a genuinely new feature (no
+ * evaluation tasks exist), a probe provides a directional "usability"
+ * signal by analyzing the document content and, optionally, evaluating
+ * a generic implementation prompt.
+ *
+ * Probes are NOT scored evaluations — they answer "are these docs usable?"
+ * not "are these docs good enough?" The output is always labeled as
+ * directional and never displayed on the same scale as scored evaluations.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-4-content-release-integration.md
+ */
+import type { ProbeResult } from "./types.js";
+/** Generic probe prompt template */
+export declare const PROBE_PROMPT = "Given the following documentation about a Sanity feature, write a TypeScript\nimplementation that demonstrates the feature's core functionality.\n\nUse only APIs and patterns described in the documentation. If the documentation\nis insufficient to implement something, say so explicitly rather than guessing.\n\nDocumentation:\n{{docs}}";
+/**
+ * Analyze a model's probe output against the source documentation.
+ *
+ * Extracts API names from both the documentation and the model output,
+ * identifies hallucinations (APIs used but not documented), and classifies
+ * the overall usability.
+ *
+ * This is a pure function — it doesn't call any LLM. The model output
+ * is provided as input (from a prior evaluation step or manual run).
+ *
+ * @param documentContent - The concatenated documentation text
+ * @param modelOutput - The model's response to the probe prompt
+ * @param documentSlugs - The slugs of the documents that were probed
+ * @returns Probe result with usability classification
+ */
+export declare function analyzeProbeOutput(documentContent: string, modelOutput: string, documentSlugs: string[]): ProbeResult;
+/**
+ * Format a probe result for console output.
+ */
+export declare function formatProbeConsole(probe: ProbeResult): string;
+/**
+ * Format a probe result as markdown.
+ */
+export declare function formatProbeMarkdown(probe: ProbeResult): string;
+/**
+ * Generate task scaffolding suggestions from document content analysis.
+ *
+ * Extracts function names, configuration patterns, and common operations
+ * from code blocks in the documentation to suggest evaluation tasks.
+ */
+export declare function generateTaskSuggestions(documentContent: string, documentSlugs: string[]): string[];