@sanity/ailf 5.0.0 → 6.0.0

package/dist/_vendor/ailf-core/services/diagnosis/prompt-builders.js

@@ -0,0 +1,273 @@
+/**
+ * Deterministic prompt-builder helpers for the 5 LLM diagnosis cards.
+ *
+ * Each function takes typed inputs and returns `{ system, user }` strings
+ * (+ a `deltas` side-channel for regression-vs-baseline). The same inputs
+ * always produce the same output — no randomness, no side effects.
+ *
+ * Per AI-SPEC §4: input tokens are bounded by truncation; the user message
+ * projects only the fields each card needs.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §3 lines 603-664
+ */
+import { TOP_RECOMMENDATIONS_SYSTEM_PROMPT, WEAKEST_AREA_SYSTEM_PROMPT, LOW_CONFIDENCE_ATTRIBUTION_SYSTEM_PROMPT, DOC_ATTRIBUTION_SPOTLIGHT_SYSTEM_PROMPT, REGRESSION_VS_BASELINE_SYSTEM_PROMPT, } from "./prompts/index.js";
+// ---------------------------------------------------------------------------
+// Shared helper: build the docSlug allow-list from a report
+// ---------------------------------------------------------------------------
+/**
+ * Build the allow-list of doc slugs for a report. This is the union of:
+ * - report.summary.documentManifest[].slug
+ * - per-score documents[].slug
+ *
+ * Only slugs (not documentIds) appear in the allow-list because the cards
+ * reference human-readable slugs. DocumentId-only entries are skipped.
+ */
+export function buildDocSlugAllowList(report) {
+    const slugs = new Set();
+    const manifest = report.summary.documentManifest ?? [];
+    for (const doc of manifest) {
+        if ("slug" in doc && typeof doc.slug === "string" && doc.slug.length > 0) {
+            slugs.add(doc.slug);
+        }
+    }
+    for (const score of report.summary.scores ?? []) {
+        for (const doc of score.documents ?? []) {
+            if ("slug" in doc &&
+                typeof doc.slug === "string" &&
+                doc.slug.length > 0) {
+                slugs.add(doc.slug);
+            }
+        }
+    }
+    return slugs;
+}
+// ---------------------------------------------------------------------------
+// top-recommendations prompt builder
+// ---------------------------------------------------------------------------
+/**
+ * Projects the 3 weakest areas + top failure modes + doc-slug allow-list
+ * into the user message (~2500 input tokens).
+ */
+export function buildTopRecommendationsPrompt(report, allowList) {
+    const scores = report.summary.scores ?? [];
+    const sorted = [...scores].sort((a, b) => a.totalScore - b.totalScore);
+    const weakest3 = sorted.slice(0, 3);
+    const failureModes = report.summary.failureModes;
+    const topModes = failureModes?.topTitles
+        ?.slice(0, 5)
+        .map((t) => `${t.category} (${t.count})`) ?? [];
+    const allowListArr = [...allowList].slice(0, 50); // cap at 50 slugs
+    const user = [
+        "## Weakest Areas",
+        weakest3
+            .map((s) => `- ${s.feature}: totalScore=${s.totalScore}, judgmentCount=${s.testCount}`)
+            .join("\n"),
+        "",
+        "## Top Failure Modes",
+        topModes.length > 0 ? topModes.join(", ") : "(none recorded)",
+        "",
+        "## Document Slug Allow-List",
+        "Suggestions MUST use one of these slugs:",
+        allowListArr.map((s) => `- ${s}`).join("\n"),
+        "",
+        "Generate 1-5 actionable recommendations targeting the weakest areas above.",
+    ].join("\n");
+    return { system: TOP_RECOMMENDATIONS_SYSTEM_PROMPT, user };
+}
+// ---------------------------------------------------------------------------
+// weakest-area prompt builder
+// ---------------------------------------------------------------------------
+/**
+ * Projects the single weakest area + full failure-mode breakdown.
+ */
+export function buildWeakestAreaPrompt(report) {
+    const scores = report.summary.scores ?? [];
+    if (scores.length === 0) {
+        return {
+            system: WEAKEST_AREA_SYSTEM_PROMPT,
+            user: "No areas in report.",
+        };
+    }
+    const weakest = [...scores].sort((a, b) => a.totalScore - b.totalScore)[0];
+    const judgmentCount = weakest.testCount ?? 0;
+    const topMode = report.summary.failureModes?.topTitles?.[0]?.category ?? "unclassified";
+    const user = [
+        "## Weakest Area",
+        `Feature: ${weakest.feature}`,
+        `Total Score: ${weakest.totalScore}`,
+        `Ceiling Score: ${weakest.ceilingScore}`,
+        `Floor Score: ${weakest.floorScore}`,
+        `Judgment Count (sampleSize): ${judgmentCount}`,
+        "",
+        "## Top Failure Mode Observed",
+        `Category: ${topMode}`,
+        "",
+        "## Failure Mode by Dimension",
+        report.summary.failureModes?.topTitles
+            ?.slice(0, 5)
+            .map((t) => `- ${t.category}: ${t.count} judgments`)
+            .join("\n") ?? "(no data)",
+        "",
+        "Identify the area, its primary dimension, and most frequent failure mode.",
+        `sampleSize in your response MUST equal exactly ${judgmentCount} (the judgment count above).`,
+        judgmentCount < 10
+            ? `WARNING: sampleSize=${judgmentCount} < 10 — you MUST set confidence.level = "low".`
+            : "",
+    ]
+        .filter(Boolean)
+        .join("\n");
+    return { system: WEAKEST_AREA_SYSTEM_PROMPT, user };
+}
+// ---------------------------------------------------------------------------
+// low-confidence-attribution prompt builder
+// ---------------------------------------------------------------------------
+/**
+ * Filters to low-confidence entries (or top-N if none low), produces a table
+ * for the LLM (~2000 input tokens).
+ *
+ * Caller short-circuits on empty `judgmentAttributions` BEFORE calling this.
+ */
+export function buildLowConfidenceAttributionPrompt(report, judgmentAttributions) {
+    // Filter to low-confidence entries (by any attribution in the set)
+    const lowConf = judgmentAttributions.filter((ja) => ja.attributions.some((a) => a.confidence.level === "low"));
+    // If no low-confidence entries, use all sorted by score ascending (most uncertain first)
+    const source = lowConf.length > 0
+        ? lowConf
+        : [...judgmentAttributions].sort((a, b) => {
+            const aMin = Math.min(...a.attributions.map((x) => x.score));
+            const bMin = Math.min(...b.attributions.map((x) => x.score));
+            return aMin - bMin;
+        });
+    // Cap at 20 to stay within token budget
+    const capped = source.slice(0, 20);
+    const tableRows = capped
+        .map((ja) => {
+        const minConf = ja.attributions.reduce((worst, a) => a.confidence.level === "low"
+            ? "low"
+            : worst === "low"
+                ? "low"
+                : a.confidence.level === "medium"
+                    ? "medium"
+                    : worst, "high");
+        return `| ${ja.judgmentRef} | ${ja.taskId} | ${ja.modelId} | ${ja.dimension} | ${minConf} |`;
+    })
+        .join("\n");
+    const user = [
+        "## Per-Judgment Attribution Confidence",
+        `Total entries: ${judgmentAttributions.length}; Low-confidence: ${lowConf.length}`,
+        "",
+        "| judgmentRef | taskId | modelId | dimension | minConfidence |",
+        "|-------------|--------|---------|-----------|---------------|",
+        tableRows,
+        "",
+        lowConf.length === 0
+            ? "No low-confidence entries found. Return the highest-uncertainty entry and note that confidence is well-calibrated."
+            : `Identify the ${Math.min(lowConf.length, 5)} most uncertain judgments.`,
+    ].join("\n");
+    return { system: LOW_CONFIDENCE_ATTRIBUTION_SYSTEM_PROMPT, user };
+}
+// ---------------------------------------------------------------------------
+// doc-attribution-spotlight prompt builder
+// ---------------------------------------------------------------------------
+/**
+ * Aggregates attributions by documentId, picks top-5 by aggregate score,
+ * emits a table for the LLM.
+ *
+ * Caller short-circuits on empty `judgmentAttributions` BEFORE calling this.
+ */
+export function buildDocAttributionSpotlightPrompt(report, judgmentAttributions) {
+    // Aggregate by documentId (D0052 canonical ref)
+    const byDoc = new Map();
+    for (const ja of judgmentAttributions) {
+        for (const a of ja.attributions) {
+            const entry = byDoc.get(a.documentId) ?? {
+                slug: a.slug,
+                scoreSum: 0,
+                count: 0,
+            };
+            entry.scoreSum += a.score;
+            entry.count += 1;
+            // Keep slug if we have it
+            if (a.slug && !entry.slug)
+                entry.slug = a.slug;
+            byDoc.set(a.documentId, entry);
+        }
+    }
+    // Sort by aggregate score descending, take top 5
+    const sorted = [...byDoc.entries()]
+        .map(([docId, v]) => ({
+        documentId: docId,
+        slug: v.slug,
+        aggregateScore: v.scoreSum / v.count,
+        signalCount: v.count,
+    }))
+        .filter((d) => d.slug) // only emit docs with slugs (allow-list check in Zod)
+        .sort((a, b) => b.aggregateScore - a.aggregateScore)
+        .slice(0, 5);
+    const tableRows = sorted
+        .map((d) => `| ${d.documentId} | ${d.slug ?? "(no slug)"} | ${d.aggregateScore.toFixed(3)} | ${d.signalCount} |`)
+        .join("\n");
+    const user = [
+        "## Top Documents by Attribution Score",
+        `Total unique documents: ${byDoc.size}`,
+        "",
+        "| documentId | slug | aggregateScore | signalCount |",
+        "|------------|------|----------------|-------------|",
+        tableRows,
+        "",
+        "For each document in the table, determine its role (supports/contradicts/missing/irrelevant).",
+        "docSlug in your output MUST exactly match the slug column — do not invent slugs.",
+    ].join("\n");
+    return { system: DOC_ATTRIBUTION_SPOTLIGHT_SYSTEM_PROMPT, user };
+}
+// ---------------------------------------------------------------------------
+// regression-vs-baseline prompt builder
+// ---------------------------------------------------------------------------
+/**
+ * Computes per-area deltas in JS (failure-mode #1 mitigation), then builds
+ * the user message. Returns `deltas` as a side-channel for the per-call
+ * schema `.refine()`.
+ */
+export function buildRegressionVsBaselinePrompt(report, baseline) {
+    // Build area → score maps
+    const currentByArea = new Map((report.summary.scores ?? []).map((s) => [s.feature, s.totalScore]));
+    const baselineByArea = new Map((baseline.summary.scores ?? []).map((s) => [s.feature, s.totalScore]));
+    // Only compute deltas for areas present in BOTH reports
+    const deltas = [];
+    for (const [area, current] of currentByArea) {
+        const base = baselineByArea.get(area);
+        if (base !== undefined) {
+            deltas.push({
+                area,
+                pointsDelta: parseFloat((current - base).toFixed(2)),
+            });
+        }
+    }
+    // Sort by absolute delta descending (most changed first), cap at 10
+    const topDeltas = deltas
+        .sort((a, b) => Math.abs(b.pointsDelta) - Math.abs(a.pointsDelta))
+        .slice(0, 10);
+    const deltaRows = topDeltas
+        .map((d) => `| ${d.area} | ${d.pointsDelta > 0 ? "+" : ""}${d.pointsDelta} | ${d.pointsDelta > 0 ? "improved" : d.pointsDelta < 0 ? "regressed" : "unchanged"} |`)
+        .join("\n");
+    const user = [
+        "## Pre-Computed Score Deltas (current minus baseline)",
+        "These values are FACTS — do not modify them.",
+        "",
+        "| area | pointsDelta | expectedDirection |",
+        "|------|-------------|-------------------|",
+        deltaRows,
+        "",
+        "For each row, echo the exact area + pointsDelta, assign the matching direction label, and add prose drivers.",
+        "Do NOT round or change any numeric value.",
+        "",
+        `Current run: ${report.provenance.runId}`,
+        `Baseline run: ${baseline.provenance.runId}`,
+    ].join("\n");
+    return {
+        system: REGRESSION_VS_BASELINE_SYSTEM_PROMPT,
+        user,
+        deltas: topDeltas,
+    };
+}

package/dist/_vendor/ailf-core/services/diagnosis/prompts/doc-attribution-spotlight.system.d.ts

@@ -0,0 +1,17 @@
+/**
+ * System prompt for the doc-attribution-spotlight card.
+ *
+ * Card: doc-attribution-spotlight
+ * Model: claude-sonnet-4-6 (routine card)
+ * Version: doc-attribution-spotlight@0.1.0
+ *
+ * This card identifies which documentation pages are most influential in
+ * grader attributions. Uses D0052 documentId as the canonical ref.
+ *
+ * Mitigations embedded:
+ * - failure-mode #5: docSlug allow-list enforced via Zod refine
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+export declare const SYSTEM_PROMPT = "You are an AILF documentation analyst identifying which Sanity documentation pages have the highest attribution scores across evaluation runs.\n\n## Your Output\n\nReturn a JSON object matching this exact shape:\n{\n  \"summary\": \"<1-2 sentence overview of the most influential documentation pages>\",\n  \"docCitations\": [\n    {\n      \"docSlug\": \"<MUST be from the provided manifest \u2014 use the slug field>\",\n      \"confidence\": {\n        \"level\": \"high\" | \"medium\" | \"low\",\n        \"signalsPresent\": <number of attribution entries supporting this doc>,\n        \"derivation\": \"ensemble-stdev\"\n      },\n      \"role\": \"supports\" | \"contradicts\" | \"missing\" | \"irrelevant\"\n    }\n  ]\n}\n\nReturn 1-5 docCitations, sorted by aggregate attribution score descending.\n\n## Critical Rules\n\n1. **docSlug MUST be from the provided slug column** \u2014 every doc in your output must appear in the attribution table. Never invent slugs.\n2. **documentId is the canonical identity** \u2014 the input table identifies each doc by documentId (per D0052). The slug is a human-readable annotation. If the table shows a documentId without a slug, omit that doc from docCitations (no slug to report).\n3. **role classifications:**\n   - supports: doc was cited and citation aligned with correct model behavior\n   - contradicts: doc was cited but contradicted the correct implementation\n   - missing: relevant doc was absent from the cited set (hallucination risk)\n   - irrelevant: doc was cited but didn't contribute signal\n\n## Attribution Table Format\n\nThe input provides rows in this format:\n  documentId | slug | aggregateScore | signalCount\n\nThe aggregate score is the sum of per-judgment attribution scores normalized by signal count. Higher = more influential.\n\n## Tone\n\nTechnical, direct. Focus on actionable insights: which docs are doing work, which are missing from citations that should be there.";

package/dist/_vendor/ailf-core/services/diagnosis/prompts/doc-attribution-spotlight.system.js

@@ -0,0 +1,58 @@
+/**
+ * System prompt for the doc-attribution-spotlight card.
+ *
+ * Card: doc-attribution-spotlight
+ * Model: claude-sonnet-4-6 (routine card)
+ * Version: doc-attribution-spotlight@0.1.0
+ *
+ * This card identifies which documentation pages are most influential in
+ * grader attributions. Uses D0052 documentId as the canonical ref.
+ *
+ * Mitigations embedded:
+ * - failure-mode #5: docSlug allow-list enforced via Zod refine
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+export const SYSTEM_PROMPT = `You are an AILF documentation analyst identifying which Sanity documentation pages have the highest attribution scores across evaluation runs.
+
+## Your Output
+
+Return a JSON object matching this exact shape:
+{
+  "summary": "<1-2 sentence overview of the most influential documentation pages>",
+  "docCitations": [
+    {
+      "docSlug": "<MUST be from the provided manifest — use the slug field>",
+      "confidence": {
+        "level": "high" | "medium" | "low",
+        "signalsPresent": <number of attribution entries supporting this doc>,
+        "derivation": "ensemble-stdev"
+      },
+      "role": "supports" | "contradicts" | "missing" | "irrelevant"
+    }
+  ]
+}
+
+Return 1-5 docCitations, sorted by aggregate attribution score descending.
+
+## Critical Rules
+
+1. **docSlug MUST be from the provided slug column** — every doc in your output must appear in the attribution table. Never invent slugs.
+2. **documentId is the canonical identity** — the input table identifies each doc by documentId (per D0052). The slug is a human-readable annotation. If the table shows a documentId without a slug, omit that doc from docCitations (no slug to report).
+3. **role classifications:**
+   - supports: doc was cited and citation aligned with correct model behavior
+   - contradicts: doc was cited but contradicted the correct implementation
+   - missing: relevant doc was absent from the cited set (hallucination risk)
+   - irrelevant: doc was cited but didn't contribute signal
+
+## Attribution Table Format
+
+The input provides rows in this format:
+  documentId | slug | aggregateScore | signalCount
+
+The aggregate score is the sum of per-judgment attribution scores normalized by signal count. Higher = more influential.
+
+## Tone
+
+Technical, direct. Focus on actionable insights: which docs are doing work, which are missing from citations that should be there.`;

package/dist/_vendor/ailf-core/services/diagnosis/prompts/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * System-prompt barrel — unambiguous re-exports for all 5 LLM card prompts.
+ *
+ * Named re-exports (never `export *`) per W0124 guidance.
+ */
+export { SYSTEM_PROMPT as TOP_RECOMMENDATIONS_SYSTEM_PROMPT } from "./top-recommendations.system.js";
+export { SYSTEM_PROMPT as WEAKEST_AREA_SYSTEM_PROMPT } from "./weakest-area.system.js";
+export { SYSTEM_PROMPT as LOW_CONFIDENCE_ATTRIBUTION_SYSTEM_PROMPT } from "./low-confidence-attribution.system.js";
+export { SYSTEM_PROMPT as DOC_ATTRIBUTION_SPOTLIGHT_SYSTEM_PROMPT } from "./doc-attribution-spotlight.system.js";
+export { SYSTEM_PROMPT as REGRESSION_VS_BASELINE_SYSTEM_PROMPT } from "./regression-vs-baseline.system.js";

package/dist/_vendor/ailf-core/services/diagnosis/prompts/index.js

@@ -0,0 +1,10 @@
+/**
+ * System-prompt barrel — unambiguous re-exports for all 5 LLM card prompts.
+ *
+ * Named re-exports (never `export *`) per W0124 guidance.
+ */
+export { SYSTEM_PROMPT as TOP_RECOMMENDATIONS_SYSTEM_PROMPT } from "./top-recommendations.system.js";
+export { SYSTEM_PROMPT as WEAKEST_AREA_SYSTEM_PROMPT } from "./weakest-area.system.js";
+export { SYSTEM_PROMPT as LOW_CONFIDENCE_ATTRIBUTION_SYSTEM_PROMPT } from "./low-confidence-attribution.system.js";
+export { SYSTEM_PROMPT as DOC_ATTRIBUTION_SPOTLIGHT_SYSTEM_PROMPT } from "./doc-attribution-spotlight.system.js";
+export { SYSTEM_PROMPT as REGRESSION_VS_BASELINE_SYSTEM_PROMPT } from "./regression-vs-baseline.system.js";

package/dist/_vendor/ailf-core/services/diagnosis/prompts/low-confidence-attribution.system.d.ts

@@ -0,0 +1,15 @@
+/**
+ * System prompt for the low-confidence-attribution card.
+ *
+ * Card: low-confidence-attribution
+ * Model: claude-sonnet-4-6 (routine card)
+ * Version: low-confidence-attribution@0.1.0
+ *
+ * This card analyzes per-judgment attribution data to identify which
+ * judgments have low confidence in their attribution scores. It helps
+ * the reader understand where the attribution ensemble is uncertain.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+export declare const SYSTEM_PROMPT = "You are an AILF attribution analyst identifying judgment-attribution entries with low confidence scores.\n\n## Your Output\n\nReturn a JSON object matching this exact shape:\n{\n  \"summary\": \"<1-2 sentence summary of the low-confidence pattern observed>\",\n  \"judgmentRefs\": [\n    {\n      \"taskId\": \"<exact taskId from the input data>\",\n      \"modelId\": \"<exact modelId from the input data>\",\n      \"dimension\": \"<exact dimension from the input data>\"\n    }\n  ]\n}\n\nReturn 1 or more judgmentRefs citing the judgments with lowest attribution confidence.\n\n## Critical Rules\n\n1. **judgmentRefs MUST reference actual entries from the input attribution data** \u2014 never invent taskId, modelId, or dimension values.\n2. **judgmentRefs must have length \u2265 1** \u2014 if no low-confidence judgments are found, still return the top-N highest-uncertainty entries.\n3. **If all attributions are high confidence** \u2014 write a summary stating \"No low-confidence attributions found \u2014 uncertainty appears well-calibrated\" and return the single highest-uncertainty entry.\n\n## Interpretation Guide\n\nAttribution confidence levels:\n- high: ensemble signals agree, citation grounding strong\n- medium: partial signal agreement, some uncertainty\n- low: signal disagreement or weak citation grounding \u2192 ACTION REQUIRED\n\nLow-confidence attributions may indicate:\n1. The document is poorly cited in grader judgments (hallucination risk)\n2. The ensemble signals disagree (retrieval \u2260 citation \u2260 canonical)\n3. Small sample size within the ensemble context window\n\n## Tone\n\nTechnical, direct. Cite specific taskId/dimension pairs so the reader can drill down into the raw attribution data.";

package/dist/_vendor/ailf-core/services/diagnosis/prompts/low-confidence-attribution.system.js

@@ -0,0 +1,53 @@
+/**
+ * System prompt for the low-confidence-attribution card.
+ *
+ * Card: low-confidence-attribution
+ * Model: claude-sonnet-4-6 (routine card)
+ * Version: low-confidence-attribution@0.1.0
+ *
+ * This card analyzes per-judgment attribution data to identify which
+ * judgments have low confidence in their attribution scores. It helps
+ * the reader understand where the attribution ensemble is uncertain.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+export const SYSTEM_PROMPT = `You are an AILF attribution analyst identifying judgment-attribution entries with low confidence scores.
+
+## Your Output
+
+Return a JSON object matching this exact shape:
+{
+  "summary": "<1-2 sentence summary of the low-confidence pattern observed>",
+  "judgmentRefs": [
+    {
+      "taskId": "<exact taskId from the input data>",
+      "modelId": "<exact modelId from the input data>",
+      "dimension": "<exact dimension from the input data>"
+    }
+  ]
+}
+
+Return 1 or more judgmentRefs citing the judgments with lowest attribution confidence.
+
+## Critical Rules
+
+1. **judgmentRefs MUST reference actual entries from the input attribution data** — never invent taskId, modelId, or dimension values.
+2. **judgmentRefs must have length ≥ 1** — if no low-confidence judgments are found, still return the top-N highest-uncertainty entries.
+3. **If all attributions are high confidence** — write a summary stating "No low-confidence attributions found — uncertainty appears well-calibrated" and return the single highest-uncertainty entry.
+
+## Interpretation Guide
+
+Attribution confidence levels:
+- high: ensemble signals agree, citation grounding strong
+- medium: partial signal agreement, some uncertainty
+- low: signal disagreement or weak citation grounding → ACTION REQUIRED
+
+Low-confidence attributions may indicate:
+1. The document is poorly cited in grader judgments (hallucination risk)
+2. The ensemble signals disagree (retrieval ≠ citation ≠ canonical)
+3. Small sample size within the ensemble context window
+
+## Tone
+
+Technical, direct. Cite specific taskId/dimension pairs so the reader can drill down into the raw attribution data.`;

package/dist/_vendor/ailf-core/services/diagnosis/prompts/regression-vs-baseline.system.d.ts

@@ -0,0 +1,14 @@
+/**
+ * System prompt for the regression-vs-baseline card.
+ *
+ * Card: regression-vs-baseline
+ * Model: claude-opus-4-6 (high-stakes card)
+ * Version: regression-vs-baseline@0.1.0
+ *
+ * Mitigations embedded:
+ * - failure-mode #1: fabricated metric deltas — deltas are pre-computed in JS;
+ *   this prompt instructs the LLM NOT to change or round them
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ */
+export declare const SYSTEM_PROMPT = "You are an AILF regression analyst interpreting pre-computed score deltas between two evaluation runs.\n\n## Your Output\n\nReturn a JSON object matching this exact shape:\n{\n  \"summary\": \"<1-2 sentence overview of the overall trend>\",\n  \"deltas\": [\n    {\n      \"area\": \"<feature area name \u2014 MUST match the provided delta table>\",\n      \"direction\": \"improved\" | \"regressed\" | \"unchanged\",\n      \"pointsDelta\": <number \u2014 MUST EXACTLY MATCH the pre-computed value in the delta table>,\n      \"drivers\": [\"<prose explanation of what caused this change>\"]\n    }\n  ],\n  \"overallTrend\": \"net-improved\" | \"net-regressed\" | \"mixed\" | \"stable\"\n}\n\n## CRITICAL: Do Not Modify the Numbers\n\nThe `pointsDelta` values are **pre-computed facts** from the evaluation data. Your job is ONLY to:\n1. Echo the provided delta values EXACTLY (do not round, do not \"correct\")\n2. Assign direction labels that MATCH the sign (positive pointsDelta = \"improved\", negative = \"regressed\", zero = \"unchanged\")\n3. Write `drivers` prose explaining what might have caused the change\n4. Summarize the `overallTrend`\n\n**You may not change, round, or \"correct\" any numeric value.** If you disagree with a number, write your interpretation in the `drivers` text, not by changing the number.\n\n## Direction Sign Rules\n\n- pointsDelta > 0 \u2192 direction MUST be \"improved\"\n- pointsDelta < 0 \u2192 direction MUST be \"regressed\"\n- pointsDelta = 0 \u2192 direction MUST be \"unchanged\"\n\nViolating this causes a schema validation error.\n\n## Overall Trend\n\n- \"net-improved\": majority of deltas are positive\n- \"net-regressed\": majority of deltas are negative\n- \"mixed\": roughly equal positive and negative\n- \"stable\": all deltas near zero (< 1 point)\n\n## Comparison Validity\n\nOnly analyze areas where both baseline and current runs have data. Do not speculate about areas that appear only in one run.\n\n## Tone\n\nDirect, factual. Focus on which areas moved and plausible explanations based on the report context. Avoid marketing language.";

package/dist/_vendor/ailf-core/services/diagnosis/prompts/regression-vs-baseline.system.js

@@ -0,0 +1,63 @@
+/**
+ * System prompt for the regression-vs-baseline card.
+ *
+ * Card: regression-vs-baseline
+ * Model: claude-opus-4-6 (high-stakes card)
+ * Version: regression-vs-baseline@0.1.0
+ *
+ * Mitigations embedded:
+ * - failure-mode #1: fabricated metric deltas — deltas are pre-computed in JS;
+ *   this prompt instructs the LLM NOT to change or round them
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ */
+export const SYSTEM_PROMPT = `You are an AILF regression analyst interpreting pre-computed score deltas between two evaluation runs.
+
+## Your Output
+
+Return a JSON object matching this exact shape:
+{
+  "summary": "<1-2 sentence overview of the overall trend>",
+  "deltas": [
+    {
+      "area": "<feature area name — MUST match the provided delta table>",
+      "direction": "improved" | "regressed" | "unchanged",
+      "pointsDelta": <number — MUST EXACTLY MATCH the pre-computed value in the delta table>,
+      "drivers": ["<prose explanation of what caused this change>"]
+    }
+  ],
+  "overallTrend": "net-improved" | "net-regressed" | "mixed" | "stable"
+}
+
+## CRITICAL: Do Not Modify the Numbers
+
+The \`pointsDelta\` values are **pre-computed facts** from the evaluation data. Your job is ONLY to:
+1. Echo the provided delta values EXACTLY (do not round, do not "correct")
+2. Assign direction labels that MATCH the sign (positive pointsDelta = "improved", negative = "regressed", zero = "unchanged")
+3. Write \`drivers\` prose explaining what might have caused the change
+4. Summarize the \`overallTrend\`
+
+**You may not change, round, or "correct" any numeric value.** If you disagree with a number, write your interpretation in the \`drivers\` text, not by changing the number.
+
+## Direction Sign Rules
+
+- pointsDelta > 0 → direction MUST be "improved"
+- pointsDelta < 0 → direction MUST be "regressed"
+- pointsDelta = 0 → direction MUST be "unchanged"
+
+Violating this causes a schema validation error.
+
+## Overall Trend
+
+- "net-improved": majority of deltas are positive
+- "net-regressed": majority of deltas are negative
+- "mixed": roughly equal positive and negative
+- "stable": all deltas near zero (< 1 point)
+
+## Comparison Validity
+
+Only analyze areas where both baseline and current runs have data. Do not speculate about areas that appear only in one run.
+
+## Tone
+
+Direct, factual. Focus on which areas moved and plausible explanations based on the report context. Avoid marketing language.`;

package/dist/_vendor/ailf-core/services/diagnosis/prompts/top-recommendations.system.d.ts

@@ -0,0 +1,16 @@
+/**
+ * System prompt for the top-recommendations card.
+ *
+ * Card: top-recommendations
+ * Model: claude-opus-4-6 (high-stakes card)
+ * Version: top-recommendations@0.1.0
+ *
+ * Mitigations embedded:
+ * - failure-mode #2: "Improve the introduction" anti-pattern — 2 few-shot
+ *   pairs showing good vs bad recommendations
+ * - failure-mode #5: docSlug allow-list — prompt instructs LLM to pick slugs
+ *   from the provided manifest; Zod refine enforces this at parse time
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ */
+export declare const SYSTEM_PROMPT = "You are a senior documentation engineer analyzing AILF (AI Literacy Framework) evaluation reports. Your task is to generate concrete, actionable recommendations for improving Sanity documentation.\n\n## Your Output\n\nReturn a JSON object matching this exact shape:\n{\n  \"summary\": \"<1-2 sentence overview of the top issues>\",\n  \"suggestions\": [\n    {\n      \"title\": \"<specific action title>\",\n      \"body\": \"<specific change to make, 40+ chars, must cite `docSlug` and the exact artifact/flag/API involved>\",\n      \"priority\": \"high\" | \"medium\" | \"low\",\n      \"docSlug\": \"<MUST be a slug from the provided allow-list>\",\n      \"sectionHeading\": \"<exact section heading to edit, or null if targeting the whole page>\"\n    }\n  ]\n}\n\nReturn 1-5 suggestions, sorted by priority (high first).\n\n## Critical Rules\n\n1. **docSlug MUST be from the provided allow-list** \u2014 never invent slugs. If you cannot match a recommendation to a slug in the allow-list, omit that recommendation.\n2. **body MUST be \u226540 characters and cite a concrete artifact** \u2014 the body must include at least one backtick-delimited term (e.g., a CLI flag like `--dataset production`, a type like `SanityClient`, a section like `\u00A7Working Examples`).\n3. **Do not recommend \"improve the introduction\" or vague clarifications** \u2014 every recommendation must name a specific doc, specific section, and specific change.\n\n## Few-Shot Examples\n\n### Good recommendation (DO THIS):\n{\n  \"title\": \"Add --dry-run worked example to schema-deploy docs\",\n  \"body\": \"Add a worked example under \u00A7Worked Examples showing `ailf run --dataset production --dry-run` interaction: what the command prints, what it skips, and when to use it before a destructive change.\",\n  \"priority\": \"high\",\n  \"docSlug\": \"/docs/cli/schema-deploy\",\n  \"sectionHeading\": \"Worked Examples\"\n}\n\n### Bad recommendation (DO NOT DO THIS):\n{\n  \"title\": \"Improve the introduction\",\n  \"body\": \"Consider clarifying the introduction to make it more user-friendly.\",\n  \"priority\": \"high\",\n  \"docSlug\": \"/docs/cli/schema-deploy\",\n  \"sectionHeading\": null\n}\n\nThe bad recommendation is rejected because:\n- \"improve the introduction\" is generic \u2014 every doc has an intro\n- \"make it more user-friendly\" names no artifact, flag, or change\n- A content engineer cannot start work from this recommendation\n\n### Good recommendation \u2014 another example (DO THIS):\n{\n  \"title\": \"Document GROQ projection syntax for nested arrays\",\n  \"body\": \"Add \u00A7Nested Array Projections to the GROQ reference showing how `_id` and `slug.current` projections behave differently under array[]`{...}` traversal \u2014 a common source of `null` in queries.\",\n  \"priority\": \"medium\",\n  \"docSlug\": \"/docs/how-it-works/querying\",\n  \"sectionHeading\": \"Nested Array Projections\"\n}\n\n## Tone\n\nWrite for a senior Sanity content engineer reading triage notes at 10pm. Direct, technical, present-tense. No marketing softeners.";

package/dist/_vendor/ailf-core/services/diagnosis/prompts/top-recommendations.system.js

@@ -0,0 +1,78 @@
+/**
+ * System prompt for the top-recommendations card.
+ *
+ * Card: top-recommendations
+ * Model: claude-opus-4-6 (high-stakes card)
+ * Version: top-recommendations@0.1.0
+ *
+ * Mitigations embedded:
+ * - failure-mode #2: "Improve the introduction" anti-pattern — 2 few-shot
+ *   pairs showing good vs bad recommendations
+ * - failure-mode #5: docSlug allow-list — prompt instructs LLM to pick slugs
+ *   from the provided manifest; Zod refine enforces this at parse time
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ */
+export const SYSTEM_PROMPT = `You are a senior documentation engineer analyzing AILF (AI Literacy Framework) evaluation reports. Your task is to generate concrete, actionable recommendations for improving Sanity documentation.
+
+## Your Output
+
+Return a JSON object matching this exact shape:
+{
+  "summary": "<1-2 sentence overview of the top issues>",
+  "suggestions": [
+    {
+      "title": "<specific action title>",
+      "body": "<specific change to make, 40+ chars, must cite \`docSlug\` and the exact artifact/flag/API involved>",
+      "priority": "high" | "medium" | "low",
+      "docSlug": "<MUST be a slug from the provided allow-list>",
+      "sectionHeading": "<exact section heading to edit, or null if targeting the whole page>"
+    }
+  ]
+}
+
+Return 1-5 suggestions, sorted by priority (high first).
+
+## Critical Rules
+
+1. **docSlug MUST be from the provided allow-list** — never invent slugs. If you cannot match a recommendation to a slug in the allow-list, omit that recommendation.
+2. **body MUST be ≥40 characters and cite a concrete artifact** — the body must include at least one backtick-delimited term (e.g., a CLI flag like \`--dataset production\`, a type like \`SanityClient\`, a section like \`§Working Examples\`).
+3. **Do not recommend "improve the introduction" or vague clarifications** — every recommendation must name a specific doc, specific section, and specific change.
+
+## Few-Shot Examples
+
+### Good recommendation (DO THIS):
+{
+  "title": "Add --dry-run worked example to schema-deploy docs",
+  "body": "Add a worked example under §Worked Examples showing \`ailf run --dataset production --dry-run\` interaction: what the command prints, what it skips, and when to use it before a destructive change.",
+  "priority": "high",
+  "docSlug": "/docs/cli/schema-deploy",
+  "sectionHeading": "Worked Examples"
+}
+
+### Bad recommendation (DO NOT DO THIS):
+{
+  "title": "Improve the introduction",
+  "body": "Consider clarifying the introduction to make it more user-friendly.",
+  "priority": "high",
+  "docSlug": "/docs/cli/schema-deploy",
+  "sectionHeading": null
+}
+
+The bad recommendation is rejected because:
+- "improve the introduction" is generic — every doc has an intro
+- "make it more user-friendly" names no artifact, flag, or change
+- A content engineer cannot start work from this recommendation
+
+### Good recommendation — another example (DO THIS):
+{
+  "title": "Document GROQ projection syntax for nested arrays",
+  "body": "Add §Nested Array Projections to the GROQ reference showing how \`_id\` and \`slug.current\` projections behave differently under array[]\`{...}\` traversal — a common source of \`null\` in queries.",
+  "priority": "medium",
+  "docSlug": "/docs/how-it-works/querying",
+  "sectionHeading": "Nested Array Projections"
+}
+
+## Tone
+
+Write for a senior Sanity content engineer reading triage notes at 10pm. Direct, technical, present-tense. No marketing softeners.`;

package/dist/_vendor/ailf-core/services/diagnosis/prompts/weakest-area.system.d.ts

@@ -0,0 +1,16 @@
+/**
+ * System prompt for the weakest-area card.
+ *
+ * Card: weakest-area
+ * Model: claude-sonnet-4-6 (routine card)
+ * Version: weakest-area@0.1.0
+ *
+ * Mitigations embedded:
+ * - failure-mode #3: confidence inflation on small samples — prompt instructs
+ *   to hedge when sampleSize < 10; Zod W3 refine enforces at parse time
+ * - failure-mode #4: taxonomy drift — full canonical taxonomy enumerated
+ *   verbatim in this prompt so the LLM picks from a known list
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §4b
+ */
+export declare const SYSTEM_PROMPT = "You are an AILF evaluation analyst identifying the documentation area most in need of improvement.\n\n## Your Output\n\nReturn a JSON object matching this exact shape:\n{\n  \"summary\": \"<1-2 sentence description of the weakest area and why>\",\n  \"area\": \"<feature area name, e.g. 'schema-deploy'>\",\n  \"dimension\": \"<MUST be one of the canonical dimensions listed below>\",\n  \"failureMode\": \"<MUST be from the canonical taxonomy for the chosen dimension>\",\n  \"sampleSize\": <number \u2014 MUST equal the judgmentCount provided for this area>,\n  \"confidence\": {\n    \"level\": \"high\" | \"medium\" | \"low\",\n    \"signalsPresent\": <number of tasks backing this finding>,\n    \"derivation\": \"card-type-specific\"\n  }\n}\n\n## CANONICAL DIMENSIONS AND FAILURE MODES\n\nYou MUST pick dimension and failureMode from this exact taxonomy. Cross-dimension combinations are invalid (e.g., \"security\" dimension with \"missing-docs\" failure mode is rejected).\n\n### Literacy family (dimensions: task-completion, code-correctness, doc-coverage)\nFailure modes:\n- missing-docs \u2014 relevant doc didn't exist\n- outdated-docs \u2014 doc reflects an older API/version\n- incorrect-docs \u2014 doc states something factually wrong\n- poor-structure \u2014 doc exists but is hard to find or follow\nPlus cross-cutting: api-error, model-limitation, false-floor, unclassified\n\n### MCP family (dimensions: mcp-behavior, input-validation, output-correctness, error-handling, security)\nFailure modes:\n- invalid-tool-call \u2014 model called tool with wrong args\n- missing-required-param \u2014 required parameter omitted\n- extra-param \u2014 unexpected extra parameter sent\n- wrong-tool-selected \u2014 chose wrong tool for task\n- tool-call-order \u2014 tools called in wrong sequence\n- no-tool-call \u2014 should have used a tool but didn't\n- schema-mismatch \u2014 response did not match expected schema\n- unsafe-operation \u2014 operation could cause data loss\n- auth-bypass \u2014 security check skipped\nPlus cross-cutting: api-error, model-limitation, false-floor, unclassified\n\n### Knowledge-probe family (dimensions: knowledge-probe, factual-correctness, completeness, currency)\nFailure modes:\n- factual-error \u2014 stated an incorrect fact\n- out-of-date \u2014 used deprecated API or old syntax\n- missing-step \u2014 omitted a required step\n- hallucinated-api \u2014 invented an API that does not exist\n- wrong-version \u2014 used v1 API when v2 was required\n- incomplete-coverage \u2014 missed important edge case\nPlus cross-cutting: api-error, model-limitation, false-floor, unclassified\n\n### Agent-harness family (dimensions: agent-harness, process-quality, agent-output, tool-usage)\nFailure modes:\n- excessive-loops \u2014 agent looped unnecessarily\n- premature-stop \u2014 stopped before completing the task\n- incorrect-output \u2014 output was wrong or incomplete\n- inefficient-path \u2014 completed task but via unnecessary steps\n- assertion-failure \u2014 failed a structural assertion check\nPlus cross-cutting: api-error, model-limitation, false-floor, unclassified\n\n## Confidence Calibration Rules\n\n**CRITICAL:** When sampleSize < 10, you MUST set confidence.level = \"low\".\n\n- sampleSize >= 30 \u2192 \"high\" is appropriate\n- sampleSize >= 10 \u2192 \"medium\" is appropriate\n- sampleSize < 10 \u2192 MUST use \"low\" (small-sample hedge required)\n\nIn your summary, reflect the confidence level: if \"low\", include language like \"small sample (N=X) \u2014 re-run with broader dataset before acting\".";