@sanity/ailf 4.0.6 → 4.1.0

package/dist/pipeline/pr-comment.js

@@ -1,459 +1,70 @@
 /**
- * pipeline/pr-comment.ts — Generates a markdown PR comment from eval score-summary.json.
+ * pipeline/pr-comment.ts — Generate a markdown PR comment from
+ * `results/latest/score-summary.json` (and an optional comparison-report).
  *
- * All functions accept rootDir as a parameter — no module-level constants.
- * No process.argv parsing. No env var fallbacks.
+ * Thin wrapper around `@sanity/ailf-core`'s unified renderer (W0150).
+ * Reads the local JSON files, applies legacy-field normalization on the
+ * scores, builds a `RenderableReport` envelope (so the CLI's
+ * `--promptfoo-url` flag flows through `provenance.promptfooUrls[0]`),
+ * then delegates rendering.
  *
- * Reads: results/latest/score-summary.json
- * Writes: markdown to stdout or --output file
+ * All functions accept `rootDir` as a parameter — no module-level
+ * constants. No `process.argv` parsing. No env-var fallbacks.
  */
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
-function formatCost(cost) {
-    if (cost === 0) {
-        return "$0.00";
-    }
-    if (cost < 0.01) {
-        return `$${cost.toFixed(4)}`;
-    }
-    return `$${cost.toFixed(2)}`;
-}
-function generateComment(summary, options = {}) {
-    const { belowCritical, overall, scores, source, timestamp } = summary;
-    const sorted = [...scores].sort((a, b) => b.totalScore - a.totalScore);
-    const lines = [];
-    // Header
-    lines.push(`## ${overallEmoji(overall.avgScore)} AI Literacy Score Report`);
-    lines.push("");
-    const totalCost = scores.reduce((sum, s) => sum + (s.totalCost ?? 0), 0);
-    const graderCostValue = overall.cost?.graderTotal ?? 0;
-    const combinedCostValue = totalCost + graderCostValue;
-    const costStr = combinedCostValue > 0 ? ` · Cost: ${formatCost(combinedCostValue)}` : "";
-    const actualStr = overall.avgActualScore !== undefined
-        ? ` · Actual: ${Math.round(overall.avgActualScore)}/100`
-        : "";
-    const retGapStr = overall.avgRetrievalGap !== undefined
-        ? ` · Ret. Gap: ${Math.round(overall.avgRetrievalGap)}`
-        : "";
-    lines.push(`**Overall: ${Math.round(overall.avgScore)}/100** · Doc Lift: +${Math.round(overall.avgDocLift)}${actualStr}${retGapStr} · ${scores.reduce((sum, s) => sum + s.testCount, 0)} tests across ${scores.length} areas${costStr}`);
-    lines.push("");
-    // Critical warnings
-    if (belowCritical.length > 0) {
-        lines.push(`> ⚠️ **Below critical threshold:** ${belowCritical.map((a) => `\`${a}\``).join(", ")}`);
-        lines.push("");
-    }
-    // Environment info
-    if (source) {
-        lines.push("<details>");
-        lines.push("<summary>🔧 Environment</summary>");
-        lines.push("");
-        lines.push("| Setting | Value |");
-        lines.push("|---------|-------|");
-        lines.push(`| **Source** | ${source.name} |`);
-        lines.push(`| **Docs URL** | ${source.baseUrl} |`);
-        if (source.dataset) {
-            lines.push(`| **Dataset** | ${source.dataset} |`);
-        }
-        lines.push(`| **Project** | ${source.projectId} |`);
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-    }
-    // Source verification
-    const { sourceIsolation, sourceVerification } = summary;
-    if (sourceVerification || sourceIsolation) {
-        lines.push("<details>");
-        lines.push("<summary>🔍 Source verification</summary>");
-        lines.push("");
-        lines.push("| Setting | Value |");
-        lines.push("|---------|-------|");
-        if (sourceVerification) {
-            lines.push(`| **Source** | ${sourceVerification.source} |`);
-            lines.push(`| **Mode** | ${sourceVerification.mode} |`);
-            if (sourceVerification.allowedOrigins) {
-                lines.push(`| **Sandbox** | ${sourceVerification.allowedOrigins.join(", ")} |`);
-            }
-            if (sourceVerification.searchMode) {
-                lines.push(`| **Search** | ${sourceVerification.searchMode} |`);
-            }
-        }
-        if (sourceIsolation) {
-            const pct = Math.round(sourceIsolation.isolationScore * 100);
-            const icon = sourceIsolation.offOrigin === 0 ? "✅" : "⚠️";
-            lines.push(`| **Agent isolation** | ${icon} ${pct}% (${sourceIsolation.onOrigin}/${sourceIsolation.total} on-origin) |`);
-            if (sourceIsolation.offOrigin > 0) {
-                lines.push(`| **Off-origin fetches** | ${sourceIsolation.offOriginUrls.slice(0, 5).join(", ")} |`);
-            }
-        }
-        if (sourceVerification?.urlFetch) {
-            const uf = sourceVerification.urlFetch;
-            lines.push(`| **URL fetch** | ${uf.totalFetched} fetched, ${uf.totalFailed} failed |`);
-            for (const f of uf.fetchedUrls) {
-                lines.push(`| | ✅ ${f.url} (via ${f.method}) |`);
-            }
-            for (const f of uf.failures) {
-                // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty error string should show "unknown"
-                lines.push(`| | ⚠️ ${f.url}: ${f.error || "unknown"} |`);
-            }
-        }
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-    }
-    // Score table
-    lines.push("### Scores by Feature Area");
-    lines.push("");
-    lines.push("| Feature | Score | Grade | Task | Code | Docs | Doc Lift | Tests |");
-    lines.push("|---------|-------|-------|------|------|------|----------|-------|");
-    for (const s of sorted) {
-        lines.push(`| ${s.feature} | **${Math.round(s.totalScore)}** | ${gradeEmoji(s.totalScore)} ${gradeLetter(s.totalScore)} | ${Math.round(s.taskCompletion)} | ${Math.round(s.codeCorrectness)} | ${Math.round(s.docCoverage)} | ${liftArrow(s.docLift)} | ${s.testCount} |`);
-    }
-    lines.push("");
-    // Breakdown legend
-    lines.push("<details>");
-    lines.push("<summary>📊 Score breakdown legend</summary>");
-    lines.push("");
-    lines.push("| Column | Description |");
-    lines.push("|--------|-------------|");
-    lines.push("| **Score** | Overall AI literacy score (0–100) — weighted sum of Task + Code + Docs |");
-    lines.push("| **Task** | Task completion — does the LLM understand what to build? |");
-    lines.push("| **Code** | Code correctness — does the generated code actually work? |");
-    lines.push("| **Docs** | Documentation coverage — are the right APIs/patterns referenced? |");
-    lines.push("| **Doc Lift** | Score improvement when docs are provided vs baseline (no docs) |");
-    lines.push("| **Grade** | ✅ A (≥80) · 🟡 B (≥70) · 🟠 C (≥50) · 🔴 D (<50) |");
-    lines.push("");
-    lines.push("</details>");
-    lines.push("");
-    // Negative Doc Lift warning (always visible, not collapsible)
-    const negDocLiftAreas = sorted.filter((s) => s.negativeDocLift);
-    if (negDocLiftAreas.length > 0) {
-        for (const s of negDocLiftAreas) {
-            lines.push(`> 🚨 **Negative Doc Lift:** \`${s.feature}\` (${s.docLift}) — docs hurt performance. Floor: ${s.floorScore}, Ceiling: ${s.ceilingScore}`);
-        }
-        lines.push("");
-    }
-    // Three-layer decomposition (when actual scores are present)
-    const hasActualScores = sorted.some((s) => s.actualScore !== undefined);
-    if (hasActualScores) {
-        // Full decomposition — show all three layers prominently
-        lines.push("### 🔬 Three-Layer Decomposition");
-        lines.push("");
-        lines.push("| Feature | Floor | Ceiling | Actual | Doc Lift | Retr. Gap | Infra % |");
-        lines.push("|---------|-------|---------|--------|----------|-----------|---------|");
-        for (const s of sorted) {
-            const actualStr = s.actualScore !== undefined ? String(s.actualScore) : "—";
-            const gapStr = s.retrievalGap !== undefined
-                ? s.retrievalGap >= 0
-                    ? `+${s.retrievalGap}`
-                    : String(s.retrievalGap)
-                : "—";
-            const infraStr = s.infrastructureEfficiency != null
-                ? `${Math.round(s.infrastructureEfficiency * 100)}%`
-                : "—";
-            const flag = s.invertedRetrievalGap ? " 🔄" : "";
-            lines.push(`| ${s.feature} | ${s.floorScore} | ${s.ceilingScore} | ${actualStr} | ${liftArrow(s.docLift)} | ${gapStr}${flag} | ${infraStr} |`);
-        }
-        lines.push("");
-        // Decomposition guide
-        lines.push("<details>");
-        lines.push("<summary>📖 What do these numbers mean?</summary>");
-        lines.push("");
-        lines.push("- **Floor:** How well models do without any documentation (just training data)");
-        lines.push("- **Ceiling:** How well models do with perfect documentation (hand-picked, injected)");
-        lines.push("- **Actual:** How well models do when they have to find docs themselves (like real users)");
-        lines.push("- **Doc Lift:** How much docs help (Ceiling − Floor). Negative = docs hurt");
-        lines.push("- **Retr. Gap:** How much quality is lost in discovery (Ceiling − Actual)");
-        lines.push("- **Infra %:** What fraction of doc quality reaches agents (Actual ÷ Ceiling)");
-        if (sorted.some((s) => s.invertedRetrievalGap)) {
-            lines.push("- **🔄:** Inverted retrieval gap — agents avoid bad docs, scoring higher than ceiling");
-        }
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-    }
-    else {
-        // Baseline-only — show the existing ceiling decomposition
-        lines.push("<details>");
-        lines.push("<summary>📊 Ceiling decomposition</summary>");
-        lines.push("");
-        lines.push("| Feature | Floor | Ceiling | Doc Lift | Quality Gap |");
-        lines.push("|---------|-------|---------|----------|-------------|");
-        for (const s of sorted) {
-            lines.push(`| ${s.feature} | ${s.floorScore} | ${s.ceilingScore} | ${liftArrow(s.docLift)} | ${s.docQualityGap} |`);
-        }
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-        // With vs without docs comparison (uses ceiling/floor model)
-        lines.push("<details>");
-        lines.push("<summary>📄 With docs vs without docs</summary>");
-        lines.push("");
-        lines.push("| Feature | With Docs | Without Docs | Lift |");
-        lines.push("|---------|-----------|--------------|------|");
-        for (const s of sorted) {
-            lines.push(`| ${s.feature} | ${s.ceilingScore} | ${s.floorScore} | ${liftArrow(s.docLift)} |`);
-        }
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-    }
-    // Cost breakdown (only when cost data is available)
-    if (totalCost > 0 || overall.cost) {
-        const graderCost = overall.cost?.graderTotal ?? 0;
-        const combinedCost = totalCost + graderCost;
-        lines.push("<details>");
-        lines.push("<summary>💰 Eval cost breakdown</summary>");
-        lines.push("");
-        lines.push("| Category | Cost |");
-        lines.push("|----------|------|");
-        lines.push(`| Provider (model inference) | ${formatCost(totalCost)} |`);
-        if (graderCost > 0) {
-            const graderLabel = overall.cost?.graderModel ?? "unknown";
-            lines.push(`| Grader (${graderLabel}) | ${formatCost(graderCost)} |`);
-        }
-        lines.push(`| **Total** | **${formatCost(combinedCost)}** |`);
-        lines.push("");
-        // Per-feature provider cost breakdown
-        lines.push("**Provider cost by feature area:**");
-        lines.push("");
-        lines.push("| Feature | Tests | Cost | Avg/Test |");
-        lines.push("|---------|-------|------|----------|");
-        for (const s of sorted) {
-            const avgCost = s.testCount > 0 ? s.totalCost / s.testCount : 0;
-            lines.push(`| ${s.feature} | ${s.testCount} | ${formatCost(s.totalCost)} | ${formatCost(avgCost)} |`);
-        }
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-    }
-    // Per-model breakdown (when multiple models were evaluated)
-    if (summary.perModel && summary.perModel.length > 1) {
-        const sorted = [...summary.perModel].sort((a, b) => b.overall.avgScore - a.overall.avgScore);
-        lines.push("<details>");
-        lines.push("<summary>🤖 Per-model scores</summary>");
-        lines.push("");
-        lines.push("| Model | Score | Doc Lift | Tests | Cost |");
-        lines.push("|-------|-------|----------|-------|------|");
-        for (const entry of sorted) {
-            const displayName = entry.label || entry.modelId;
-            const costStr = entry.overall.cost ? formatCost(entry.overall.cost) : "—";
-            const liftStr = entry.overall.avgDocLift >= 0
-                ? `+${Math.round(entry.overall.avgDocLift)}`
-                : String(Math.round(entry.overall.avgDocLift));
-            lines.push(`| ${displayName} | **${Math.round(entry.overall.avgScore)}** | ${liftStr} | ${entry.overall.testCount} | ${costStr} |`);
-        }
-        lines.push("");
-        // Per-model × per-area table
-        for (const entry of sorted) {
-            const displayName = entry.label || entry.modelId;
-            lines.push(`**${displayName}** (${entry.modelId}):`);
-            lines.push("");
-            lines.push("| Feature | Score | Task | Code | Docs | Lift |");
-            lines.push("|---------|-------|------|------|------|------|");
-            for (const s of entry.scores) {
-                const lift = s.docLift >= 0 ? `+${s.docLift}` : String(s.docLift);
-                lines.push(`| ${s.feature} | **${s.totalScore}** | ${s.taskCompletion} | ${s.codeCorrectness} | ${s.docCoverage} | ${lift} |`);
-            }
-            lines.push("");
-        }
-        // Cost per quality point
-        const withCost = sorted.filter((e) => e.overall.cost && e.overall.cost > 0);
-        if (withCost.length > 0) {
-            lines.push("**Cost efficiency:**");
-            lines.push("");
-            lines.push("| Model | $/point | Score | Cost |");
-            lines.push("|-------|---------|-------|------|");
-            for (const entry of withCost) {
-                const costPerPoint = entry.overall.avgScore > 0
-                    ? (entry.overall.cost ?? 0) / entry.overall.avgScore
-                    : 0;
-                lines.push(`| ${entry.label} | ${formatCost(costPerPoint)} | ${Math.round(entry.overall.avgScore)} | ${formatCost(entry.overall.cost ?? 0)} |`);
-            }
-            lines.push("");
-        }
-        lines.push("</details>");
-        lines.push("");
-    }
-    // Comparison section (when --compare was used)
-    if (options.comparisonReport) {
-        const report = options.comparisonReport;
-        const overallDelta = report.deltas.overall;
-        const overallDeltaStr = overallDelta > 0
-            ? `+${Math.round(overallDelta)}`
-            : String(Math.round(overallDelta));
-        const overallChangeIcon = overallDelta > report.noiseThreshold
-            ? "📈"
-            : overallDelta < -report.noiseThreshold
-                ? "📉"
-                : "➡️";
-        lines.push("### 📊 Score Comparison");
-        lines.push("");
-        lines.push(`**Overall: ${Math.round(report.baseline.overall.avgScore)} → ${Math.round(report.experiment.overall.avgScore)}** (${overallChangeIcon} ${overallDeltaStr})`);
-        lines.push("");
-        // Check if comparison data includes actual/retrieval gap deltas
-        const hasActualDeltas = report.areas.some((a) => a.actualDelta !== undefined);
-        if (hasActualDeltas) {
-            lines.push("| Feature | Baseline | Current | Delta | Actual Δ | Ret. Gap Δ | Infra Δ |");
-            lines.push("|---------|----------|---------|-------|----------|------------|---------|");
-            for (const a of report.areas.filter((a) => a.change !== "not-evaluated")) {
-                const icon = a.change === "improved"
-                    ? "📈"
-                    : a.change === "regressed"
-                        ? "📉"
-                        : "➡️";
-                const d = (n) => n > 0 ? `+${Math.round(n)}` : String(Math.round(n));
-                const actualStr = a.actualDelta !== undefined ? d(a.actualDelta) : "—";
-                const retGapStr = a.retrievalGapDelta !== undefined ? d(a.retrievalGapDelta) : "—";
-                const infraStr = a.infrastructureEfficiencyDelta !== undefined
-                    ? `${a.infrastructureEfficiencyDelta > 0 ? "+" : ""}${Math.round(a.infrastructureEfficiencyDelta * 100)}pp`
-                    : "—";
-                lines.push(`| ${a.area} | ${a.baseline} | ${a.experiment} | ${icon} ${d(a.delta)} | ${actualStr} | ${retGapStr} | ${infraStr} |`);
-            }
-        }
-        else {
-            lines.push("| Feature | Baseline | Current | Delta | Task | Code | Docs |");
-            lines.push("|---------|----------|---------|-------|------|------|------|");
-            for (const a of report.areas.filter((a) => a.change !== "not-evaluated")) {
-                const icon = a.change === "improved"
-                    ? "📈"
-                    : a.change === "regressed"
-                        ? "📉"
-                        : "➡️";
-                const d = (n) => n > 0 ? `+${Math.round(n)}` : String(Math.round(n));
-                // TODO(multi-mode): These dimension keys are literacy-specific.
-                // For other modes, iterate Object.entries(a.dimensions) dynamically.
-                lines.push(`| ${a.area} | ${a.baseline} | ${a.experiment} | ${icon} ${d(a.delta)} | ${d(a.dimensions.taskCompletion.delta)} | ${d(a.dimensions.codeCorrectness.delta)} | ${d(a.dimensions.docCoverage.delta)} |`);
-            }
-        }
-        lines.push("");
-        const parts = [];
-        if (report.improved.length > 0) {
-            parts.push(`📈 ${report.improved.length} improved`);
-        }
-        if (report.regressed.length > 0) {
-            parts.push(`📉 ${report.regressed.length} regressed`);
-        }
-        if (report.unchanged.length > 0) {
-            parts.push(`➡️ ${report.unchanged.length} unchanged`);
-        }
-        if (report.notEvaluated?.length > 0) {
-            parts.push(`⏭️ ${report.notEvaluated.length} not evaluated`);
-        }
-        if (parts.length > 0) {
-            const isEmpirical = "noiseThresholdEmpirical" in report &&
-                report.noiseThresholdEmpirical === true;
-            const thresholdNote = isEmpirical
-                ? ` (empirical threshold: ±${report.noiseThreshold.toFixed(1)})`
-                : ` (threshold: ±${report.noiseThreshold})`;
-            lines.push(parts.join(" · ") + thresholdNote);
-            lines.push("");
+import { reportToMarkdown } from "../_vendor/ailf-core/index.js";
+export function generatePrComment(options) {
+    const { outputPath, promptfooUrl, rootDir } = options;
+    const summaryPath = resolve(rootDir, "results/latest/score-summary.json");
+    const comparisonPath = resolve(rootDir, "results/latest/comparison-report.json");
+    const raw = readFileSync(summaryPath, "utf-8");
+    const parsed = JSON.parse(raw);
+    const summary = {
+        ...parsed,
+        scores: parsed.scores.map((s) => normalizeScore(s)),
+    };
+    let comparison;
+    if (existsSync(comparisonPath)) {
+        try {
+            comparison = JSON.parse(readFileSync(comparisonPath, "utf-8"));
         }
-        // Dimension averages in collapsible
-        lines.push("<details>");
-        lines.push("<summary>Dimension averages</summary>");
-        lines.push("");
-        const dim = report.deltas.perDimension;
-        const dd = (n) => n > 0 ? `+${Math.round(n)}` : String(Math.round(n));
-        lines.push("| Dimension | Delta |");
-        lines.push("|-----------|-------|");
-        lines.push(`| Task Completion | ${dd(dim.taskCompletion)} |`);
-        lines.push(`| Code Correctness | ${dd(dim.codeCorrectness)} |`);
-        lines.push(`| Doc Coverage | ${dd(dim.docCoverage)} |`);
-        lines.push(`| Doc Lift | ${dd(report.deltas.docLift)} |`);
-        lines.push("");
-        lines.push("</details>");
-        lines.push("");
-    }
-    // Recommendations
-    if (belowCritical.length > 0 || sorted.some((s) => s.totalScore < 70)) {
-        lines.push("### 💡 Recommendations");
-        lines.push("");
-        for (const s of sorted) {
-            if (s.totalScore < 50) {
-                lines.push(`- 🔴 **${s.feature}** (score: ${Math.round(s.totalScore)}) — needs significant doc improvements. `);
-                if (s.codeCorrectness < 10) {
-                    lines.push(`  Code correctness is very low (${Math.round(s.codeCorrectness)}) — add more complete code examples.`);
-                }
-                if (s.docCoverage < 10) {
-                    lines.push(`  Doc coverage is very low (${Math.round(s.docCoverage)}) — key APIs/patterns may be missing from docs.`);
-                }
-            }
-            else if (s.totalScore < 70) {
-                lines.push(`- 🟠 **${s.feature}** (score: ${Math.round(s.totalScore)}) — has room for improvement.`);
-                if (s.codeCorrectness < 15) {
-                    lines.push(`  Code correctness (${Math.round(s.codeCorrectness)}) could improve with better code examples.`);
-                }
-            }
+        catch {
+            // Non-fatal — comparison report is optional.
         }
-        lines.push("");
     }
-    // Footer
-    const date = new Date(timestamp).toLocaleString("en-US", {
-        day: "numeric",
-        hour: "numeric",
-        minute: "2-digit",
-        month: "short",
-        timeZone: "UTC",
-        timeZoneName: "short",
-        year: "numeric",
+    const report = scoreSummaryToRenderableReport(summary, comparison, {
+        promptfooUrl,
     });
-    lines.push("---");
-    const runUrl = process.env.GITHUB_RUN_URL ?? "";
-    const runLink = runUrl ? ` · <a href="${runUrl}">view run</a>` : "";
-    const promptfooLink = options.promptfooUrl
-        ? ` · <a href="${options.promptfooUrl}">view detailed results</a>`
-        : "";
-    lines.push(`<sub>🤖 Generated by AI Literacy Framework · ${date}${runLink}${promptfooLink} · <a href="https://github.com/sanity-labs/ai-literacy-framework">docs</a></sub>`);
-    return lines.join("\n");
-}
-function gradeEmoji(score) {
-    if (score >= 80) {
-        return "✅";
-    }
-    if (score >= 70) {
-        return "🟡";
-    }
-    if (score >= 50) {
-        return "🟠";
-    }
-    return "🔴";
-}
-function gradeLetter(score) {
-    if (score >= 80) {
-        return "A";
-    }
-    if (score >= 70) {
-        return "B";
-    }
-    if (score >= 50) {
-        return "C";
-    }
-    return "D";
-}
-// ── Main ───────────────────────────────────────────────────────────────
-function liftArrow(lift) {
-    const rounded = Math.round(lift);
-    if (rounded > 0) {
-        return `📈 +${rounded}`;
+    const markdown = reportToMarkdown(report);
+    if (outputPath) {
+        writeFileSync(outputPath, markdown);
     }
-    if (rounded < 0) {
-        return `📉 ${rounded}`;
+    else {
+        process.stdout.write(markdown);
     }
-    return "➡️ 0";
 }
-function overallEmoji(avg) {
-    if (avg >= 75) {
-        return "🟢";
-    }
-    if (avg >= 60) {
-        return "🟡";
-    }
-    if (avg >= 45) {
-        return "🟠";
+/**
+ * Adapter: build a {@link RenderableReport} from the in-memory pipeline
+ * artifacts. Exposed for the cross-renderer byte-equality contract test.
+ */
+export function scoreSummaryToRenderableReport(summary, comparison, options = {}) {
+    const provenance = {};
+    if (options.promptfooUrl) {
+        provenance.promptfooUrls = [{ mode: "unknown", url: options.promptfooUrl }];
     }
-    return "🔴";
+    return {
+        completedAt: summary.timestamp,
+        summary,
+        comparison,
+        provenance,
+    };
 }
-/** Normalize legacy field names in a score object */
+// ---------------------------------------------------------------------------
+// Legacy field normalization — pre-W0051 score summaries used different
+// field names. Keep these here so old cached score-summary.json files
+// still render correctly.
+// ---------------------------------------------------------------------------
 function normalizeScore(s) {
     const ceiling = s.ceilingScore ??
         s.withDocsScore ??
@@ -474,34 +85,3 @@ function normalizeScore(s) {
         negativeDocLift: s.negativeDocLift ?? lift < 0,
     };
 }
-// ── Main ──────────────────────────────────────────────────────────────
-export function generatePrComment(options) {
-    const { outputPath, promptfooUrl, rootDir } = options;
-    const summaryPath = resolve(rootDir, "results/latest/score-summary.json");
-    const comparisonPath = resolve(rootDir, "results/latest/comparison-report.json");
-    const raw = readFileSync(summaryPath, "utf-8");
-    const parsed = JSON.parse(raw);
-    // Normalize legacy field names in scores
-    const summary = {
-        ...parsed,
-        scores: parsed.scores.map((s) => normalizeScore(s)),
-    };
-    // Load comparison report if it exists (produced by --compare flag)
-    let comparisonReport;
-    if (existsSync(comparisonPath)) {
-        try {
-            const compRaw = readFileSync(comparisonPath, "utf-8");
-            comparisonReport = JSON.parse(compRaw);
-        }
-        catch {
-            // Non-fatal — comparison report is optional
-        }
-    }
-    const comment = generateComment(summary, { comparisonReport, promptfooUrl });
-    if (outputPath) {
-        writeFileSync(outputPath, comment);
-    }
-    else {
-        process.stdout.write(comment);
-    }
-}

package/dist/pipeline/resolve-mappings.d.ts

@@ -8,10 +8,15 @@
  * The output shape matches what downstream consumers expect so
  * fetch-docs, validate, and calculate-scores work without changes.
  */
-export interface CanonicalDoc {
-    reason: string;
+import type { CanonicalDoc as CoreCanonicalDoc } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Slug-keyed variant of the canonical doc reference. Resolve-mappings only
+ * deals with `{ slug, reason }` pairs (path/id/perspective variants are
+ * filtered upstream); deriving via `Extract` keeps drift impossible.
+ */
+export type CanonicalDoc = Extract<CoreCanonicalDoc, {
     slug: string;
-}
+}>;
 export interface CanonicalTask {
     canonical_docs: CanonicalDoc[];
     description: string;

package/dist/promptfoo-providers/mock-path.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Absolute filesystem path to the AILF mock Promptfoo provider
+ * (`mock-provider.cjs`, this directory's sibling).
+ *
+ * Promptfoo's `file://` provider loader requires an absolute path. Both
+ * the top-level promptfoo compiler (`promptfoo-compiler.ts:buildProviders`)
+ * and the literacy provider assembler (`provider-assembler.ts`) swap real
+ * provider IDs to `file://${MOCK_PROVIDER_ABSPATH}` when
+ * `AILF_REPLAY_LLMS=1` is set (W0110). This module is the single source
+ * of truth for that path so the two call sites can't drift apart.
+ */
+export declare const MOCK_PROVIDER_ABSPATH: string;

package/dist/promptfoo-providers/mock-path.js

@@ -0,0 +1,15 @@
+/**
+ * Absolute filesystem path to the AILF mock Promptfoo provider
+ * (`mock-provider.cjs`, this directory's sibling).
+ *
+ * Promptfoo's `file://` provider loader requires an absolute path. Both
+ * the top-level promptfoo compiler (`promptfoo-compiler.ts:buildProviders`)
+ * and the literacy provider assembler (`provider-assembler.ts`) swap real
+ * provider IDs to `file://${MOCK_PROVIDER_ABSPATH}` when
+ * `AILF_REPLAY_LLMS=1` is set (W0110). This module is the single source
+ * of truth for that path so the two call sites can't drift apart.
+ */
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const MOCK_PROVIDER_ABSPATH = resolve(__dirname, "mock-provider.cjs");