@sanity/ailf 5.0.0 → 6.0.0

package/config/diagnosis-cards.ts

@@ -0,0 +1,318 @@
+/**
+ * diagnosis-cards.ts — Diagnosis eval matrix config.
+ *
+ * TS-first config (per .claude/rules/config.md) defining the 5 LLM card types
+ * × 3 first-class models eval matrix. Consumed by
+ * `scripts/generate-diagnosis-config.ts` to emit
+ * `promptfooconfig-diagnosis.yaml`. Never hand-edit the YAML — run
+ * `pnpm generate-configs` instead.
+ *
+ * Per AI-SPEC §5 and CONTEXT D-04 (path b: standalone generator entry point
+ * for the diagnosis config, additive — does not modify the existing literacy
+ * generate-configs pipeline).
+ *
+ * @see packages/eval/scripts/generate-diagnosis-config.ts — generator
+ * @see packages/eval/promptfooconfig-diagnosis.yaml — generated output
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/**
+ * A first-class model entry in the diagnosis eval matrix.
+ * Mirrors the shape of model entries in `config/models.ts`.
+ */
+export interface DiagnosisModelEntry {
+  /** Promptfoo provider string (e.g. "anthropic:messages:claude-opus-4-6") */
+  id: string
+  /** Human-readable label for reports */
+  label: string
+  /** Per-model config overrides (temperature, max_tokens, etc.) */
+  config?: Record<string, unknown>
+}
+
+/**
+ * The 5 LLM-driven card types under evaluation.
+ * Deterministic cards (area-summary, failure-mode-summary, no-issues) are
+ * tested via `fixture-matrix.test.ts` (vitest), not via the promptfoo matrix.
+ */
+export type LLMCardType =
+  | "top-recommendations"
+  | "weakest-area"
+  | "low-confidence-attribution"
+  | "doc-attribution-spotlight"
+  | "regression-vs-baseline"
+
+/**
+ * A single evaluation scenario: one fixture path × one expected outcome.
+ *
+ * The `fixturePath` is relative to `packages/eval/` so the promptfoo config
+ * can resolve it from any working directory. `expectedStatus` drives the
+ * pass/fail assertion in the generated YAML.
+ */
+export interface DiagnosisScenario {
+  /** Short slug used in promptfoo `description` fields */
+  name: string
+  /** Path to the Report JSON fixture, relative to `packages/eval/` */
+  fixturePath: string
+  /**
+   * Card type this scenario exercises. The eval matrix runs all LLM cards
+   * per scenario; this field annotates which card type is the primary focus
+   * for the rubric.
+   */
+  primaryCard: LLMCardType
+  /** Expected card status when all LLM calls succeed */
+  expectedStatus: "ready" | "degraded" | "missing"
+  /** Optional: path to canned LLM response for adversarial scenarios */
+  cannedResponsePath?: string
+  /**
+   * Optional: cardId to key the canned response against (for FakeLLMClient
+   * keyedResponses in vitest; mirrored in the promptfoo scenario description
+   * for documentation).
+   */
+  cannedCardId?: LLMCardType
+  /** Free-text note about what this scenario tests */
+  note?: string
+}
+
+/**
+ * Top-level diagnosis eval matrix config.
+ * Exported as the default export of this file (mirrors models.ts convention).
+ */
+export interface DiagnosisCardsConfig {
+  /** All LLM card evaluation scenarios */
+  scenarios: DiagnosisScenario[]
+  /** Models to run each scenario against */
+  models: DiagnosisModelEntry[]
+  /** Grader model for LLM-judge assertions */
+  grader: DiagnosisModelEntry
+  /** Eval budget in milliseconds (kill switch) */
+  evalBudgetMs: number
+  /** Max parallel API calls */
+  maxConcurrency: number
+  /** Default per-model config */
+  defaults: {
+    temperature: number
+    max_tokens: number
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helper
+// ---------------------------------------------------------------------------
+
+export function defineDiagnosisCards(
+  config: DiagnosisCardsConfig
+): DiagnosisCardsConfig {
+  return config
+}
+
+// ---------------------------------------------------------------------------
+// Config definition
+// ---------------------------------------------------------------------------
+
+const diagnosisCardsConfig: DiagnosisCardsConfig = defineDiagnosisCards({
+  // ── Models under evaluation ────────────────────────────────────────────────
+  models: [
+    {
+      id: "anthropic:messages:claude-opus-4-6",
+      label: "Claude Opus 4.6",
+      config: { temperature: 0.2, max_tokens: 4096 },
+    },
+    {
+      id: "anthropic:messages:claude-sonnet-4-6",
+      label: "Claude Sonnet 4.6",
+      config: { temperature: 0.2, max_tokens: 4096 },
+    },
+    {
+      id: "openai:chat:gpt-5.2",
+      label: "GPT 5.2",
+      config: { max_completion_tokens: 4096 },
+    },
+  ],
+
+  // ── Grader model ────────────────────────────────────────────────────────────
+  grader: {
+    id: "anthropic:messages:claude-opus-4-5-20251101",
+    label: "Claude Opus 4.5 (grader)",
+  },
+
+  // ── Eval budget ─────────────────────────────────────────────────────────────
+  evalBudgetMs: 3_600_000, // 60 min — full matrix across 3 models × 17 scenarios
+  maxConcurrency: 8, // conservative for diagnosis (longer prompts than literacy)
+
+  // ── Default config ──────────────────────────────────────────────────────────
+  defaults: {
+    temperature: 0.2,
+    max_tokens: 4096,
+  },
+
+  // ── Scenarios (17 logical fixtures) ─────────────────────────────────────────
+  scenarios: [
+    // ── Critical-path: top-recommendations ──────────────────────────────────
+    {
+      name: "healthy-top-recommendations",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/healthy-top-recommendations.json",
+      primaryCard: "top-recommendations",
+      expectedStatus: "ready",
+      note: "Healthy report (mean 91) — top-recommendations card should produce 2+ actionable suggestions with docSlug references from the manifest.",
+    },
+    {
+      name: "low-top-recommendations",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/low-top-recommendations.json",
+      primaryCard: "top-recommendations",
+      expectedStatus: "ready",
+      note: "Low-scoring report (mean 42) — top-recommendations card should produce high-priority suggestions addressing the dominant failure modes (outdated-docs, missing-docs).",
+    },
+
+    // ── Critical-path: weakest-area ──────────────────────────────────────────
+    {
+      name: "healthy-weakest-area",
+      fixturePath: "test-fixtures/diagnosis/reports/healthy-weakest-area.json",
+      primaryCard: "weakest-area",
+      expectedStatus: "ready",
+      note: "Healthy report with clear weakest area (content-modeling at 82) — weakest-area card should identify the area and provide high-confidence analysis.",
+    },
+    {
+      name: "low-weakest-area",
+      fixturePath: "test-fixtures/diagnosis/reports/low-weakest-area.json",
+      primaryCard: "weakest-area",
+      expectedStatus: "ready",
+      note: "Low-scoring report with clear weakest area (content-modeling at 28) — weakest-area card should identify the most critical area with multiple failure modes.",
+    },
+
+    // ── Critical-path: low-confidence-attribution ────────────────────────────
+    {
+      name: "healthy-low-confidence-attribution",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/healthy-low-confidence-attribution.json",
+      primaryCard: "low-confidence-attribution",
+      expectedStatus: "ready",
+      note: "Healthy report with small sample sizes (2-3 judgments per area) — low-confidence-attribution card should identify attribution uncertainty despite positive scores.",
+    },
+    {
+      name: "low-low-confidence-attribution",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/low-low-confidence-attribution.json",
+      primaryCard: "low-confidence-attribution",
+      expectedStatus: "ready",
+      note: "Low-scoring report with small sample sizes (2 judgments per area) — low-confidence-attribution card should flag both score quality and attribution uncertainty.",
+    },
+
+    // ── Critical-path: doc-attribution-spotlight ─────────────────────────────
+    {
+      name: "healthy-doc-attribution-spotlight",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/healthy-doc-attribution-spotlight.json",
+      primaryCard: "doc-attribution-spotlight",
+      expectedStatus: "ready",
+      note: "Healthy 5-area report — doc-attribution-spotlight card should identify the highest-impact document in the manifest.",
+    },
+    {
+      name: "low-doc-attribution-spotlight",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/low-doc-attribution-spotlight.json",
+      primaryCard: "doc-attribution-spotlight",
+      expectedStatus: "ready",
+      note: "Low-scoring 5-area report with multiple failure modes — doc-attribution-spotlight card should identify the most critical document.",
+    },
+
+    // ── Edge cases ───────────────────────────────────────────────────────────
+    {
+      name: "empty-report",
+      fixturePath: "test-fixtures/diagnosis/reports/empty.json",
+      primaryCard: "top-recommendations",
+      expectedStatus: "missing",
+      note: "Edge case (a): zero-area report — all LLM cards should emit status: missing (no data to reason about).",
+    },
+    {
+      name: "single-judgment-per-area",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/single-judgment-per-area.json",
+      primaryCard: "weakest-area",
+      expectedStatus: "ready",
+      note: "Edge case (b): single-judgment sample size — weakest-area card should reflect low-confidence calibration (sampleSize: 1).",
+    },
+    {
+      name: "all-areas-tied",
+      fixturePath: "test-fixtures/diagnosis/reports/all-areas-tied.json",
+      primaryCard: "weakest-area",
+      expectedStatus: "missing",
+      note: "Edge case (c): all areas scored identically (70) — weakest-area card should emit status: missing with reason: no-clear-weakest.",
+    },
+    {
+      name: "grader-major-mismatch-baseline",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/grader-major-mismatch-baseline.json",
+      primaryCard: "regression-vs-baseline",
+      expectedStatus: "missing",
+      note: "Edge case (d): grader-major-version mismatch — regression-vs-baseline should emit missing with reason: grader-major-version-mismatch. Run as pair with grader-major-mismatch-current.",
+    },
+    {
+      name: "grader-major-mismatch-current",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/grader-major-mismatch-current.json",
+      primaryCard: "regression-vs-baseline",
+      expectedStatus: "missing",
+      note: "Edge case (d) pair: current report with different graderModel — regression-vs-baseline mismatch guard triggers when paired with grader-major-mismatch-baseline.",
+    },
+    {
+      name: "near-deprecated-taxonomy",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/near-deprecated-taxonomy.json",
+      primaryCard: "weakest-area",
+      expectedStatus: "ready",
+      note: "Edge case (e): report using unclassified failure mode (currently canonical but watch for taxonomy retirement). Zod refine() must accept canonical modes.",
+    },
+
+    // ── Adversarial canned responses ─────────────────────────────────────────
+    {
+      name: "adversarial-fabricated-delta",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/grader-major-mismatch-current.json",
+      primaryCard: "regression-vs-baseline",
+      expectedStatus: "degraded",
+      cannedResponsePath:
+        "test-fixtures/diagnosis/canned-responses/fabricated-delta-regression.json",
+      cannedCardId: "regression-vs-baseline",
+      note: "Adversarial: fabricated delta (AI-SPEC §1b failure-mode #1). LLM claims -7.3 delta; direction-sign refine triggers degraded card.",
+    },
+    {
+      name: "adversarial-improve-introduction",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/low-top-recommendations.json",
+      primaryCard: "top-recommendations",
+      expectedStatus: "degraded",
+      cannedResponsePath:
+        "test-fixtures/diagnosis/canned-responses/improve-introduction.json",
+      cannedCardId: "top-recommendations",
+      note: "Adversarial: generic anti-pattern recommendation (AI-SPEC §1b failure-mode #2). Actionability refine triggers degraded card.",
+    },
+    {
+      name: "adversarial-hallucinated-docslug",
+      fixturePath:
+        "test-fixtures/diagnosis/reports/low-top-recommendations.json",
+      primaryCard: "top-recommendations",
+      expectedStatus: "degraded",
+      cannedResponsePath:
+        "test-fixtures/diagnosis/canned-responses/hallucinated-docslug.json",
+      cannedCardId: "top-recommendations",
+      note: "Adversarial: hallucinated docSlug (AI-SPEC §1b failure-mode #3). Allow-list refine triggers degraded card.",
+    },
+    {
+      name: "adversarial-taxonomy-drift",
+      fixturePath: "test-fixtures/diagnosis/reports/low-weakest-area.json",
+      primaryCard: "weakest-area",
+      expectedStatus: "degraded",
+      cannedResponsePath:
+        "test-fixtures/diagnosis/canned-responses/taxonomy-drift.json",
+      cannedCardId: "weakest-area",
+      note: "Adversarial: taxonomy drift (AI-SPEC §1b failure-mode #4). Per-dimension failureMode refine triggers degraded card.",
+    },
+  ],
+})
+
+export default diagnosisCardsConfig

package/config/models.ts

@@ -24,6 +24,18 @@ export default defineModels({
       // All literacy variants included by default (baseline, observed,
       // agentic-naive, agentic-optimized)
     },
+    {
+      // Phase 5 LLM card routing (D-07). AI-SPEC §4 routes 3 routine cards
+      // (top-recommendations, weakest-area, regression-vs-baseline) here.
+      // Pricing already in AnthropicLLMClient; baseline literacy variant only.
+      id: "anthropic:messages:claude-sonnet-4-6",
+      label: "Claude Sonnet 4.6",
+      config: { temperature: 0.2, max_tokens: 4096 },
+      modes: ["literacy"],
+      variants: {
+        literacy: ["baseline"],
+      },
+    },
 
     // ── Google ─────────────────────────────────────────────────
     // {

package/dist/_vendor/ailf-core/grader/failure-modes/agent-harness.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Agent-harness failure modes — valid for the `agent-harness` dimension
+ * family (process-quality, agent-output, tool-usage).
+ *
+ * Relocated from packages/eval/src/grader/agent-harness.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const AGENT_FAILURE_MODES: readonly ["tool-misuse", "chaotic-process", "missing-recovery"];
+export type AgentFailureMode = (typeof AGENT_FAILURE_MODES)[number];

package/dist/_vendor/ailf-core/grader/failure-modes/agent-harness.js

@@ -0,0 +1,16 @@
+/**
+ * Agent-harness failure modes — valid for the `agent-harness` dimension
+ * family (process-quality, agent-output, tool-usage).
+ *
+ * Relocated from packages/eval/src/grader/agent-harness.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const AGENT_FAILURE_MODES = [
+    "tool-misuse", // assistant calls tools incorrectly or with wrong args
+    "chaotic-process", // assistant flails — undirected exploration, no plan
+    "missing-recovery", // assistant doesn't recover from a tool error
+];

package/dist/_vendor/ailf-core/grader/failure-modes/common.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Cross-cutting failure modes — valid for any dimension family.
+ *
+ * Relocated from packages/eval/src/grader/common.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283 — the v0 lists)
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export declare const COMMON_FAILURE_MODES: readonly ["api-error", "model-limitation", "false-floor", "unclassified"];
+export type CommonFailureMode = (typeof COMMON_FAILURE_MODES)[number];

package/dist/_vendor/ailf-core/grader/failure-modes/common.js

@@ -0,0 +1,18 @@
+/**
+ * Cross-cutting failure modes — valid for any dimension family.
+ *
+ * Relocated from packages/eval/src/grader/common.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283 — the v0 lists)
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export const COMMON_FAILURE_MODES = [
+    "api-error", // infrastructure failure, not a docs problem
+    "model-limitation", // high ceiling, model can't reach it
+    "false-floor", // model already knew the answer; docs added no value
+    "unclassified", // grader could not pick a mode (low-confidence fallback)
+];

package/dist/_vendor/ailf-core/grader/failure-modes/index.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Per-dimension failure-mode taxonomy barrel — relocated to @sanity/ailf-core.
+ *
+ * Previously lived in packages/eval/src/grader/index.ts. Moved here (D-05)
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * `failureModesForDimension` without violating the core→eval import rule.
+ *
+ * Named re-exports only (W0124 — never `export *`).
+ *
+ * The eval-side packages/eval/src/grader/index.ts now re-exports from here,
+ * keeping all existing eval-side callers working without source changes.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export { COMMON_FAILURE_MODES, type CommonFailureMode } from "./common.js";
+export { LITERACY_FAILURE_MODES, type LiteracyFailureMode } from "./literacy.js";
+export { MCP_FAILURE_MODES, type MCPFailureMode } from "./mcp.js";
+export { KP_FAILURE_MODES, type KPFailureMode } from "./knowledge-probe.js";
+export { AGENT_FAILURE_MODES, type AgentFailureMode } from "./agent-harness.js";
+/**
+ * Return the legal failure-mode list for a given rubric dimension.
+ *
+ * Accepts both family-level keys (`mcp-behavior`, `knowledge-probe`,
+ * `agent-harness`) and the per-template `dimension` strings used in
+ * `config/rubrics.ts` (`task-completion`, `input-validation`,
+ * `factual-correctness`, `process-quality`, …). The cross-cutting
+ * `COMMON_FAILURE_MODES` is always included.
+ *
+ * Unknown dimensions fall through to `COMMON_FAILURE_MODES` only — safe
+ * default, the grader can still pick `unclassified`.
+ */
+export declare function failureModesForDimension(dimension: string): readonly string[];
+/**
+ * Flat list of all dimension names recognized by `failureModesForDimension`.
+ * Used by card-validators and test calibration.
+ */
+export declare const CANONICAL_DIMENSIONS: readonly string[];
+/**
+ * Returns `true` when `mode` appears in the failure-mode list of any
+ * canonical dimension family.
+ */
+export declare function isCanonicalFailureMode(mode: string): boolean;

package/dist/_vendor/ailf-core/grader/failure-modes/index.js

@@ -0,0 +1,109 @@
+/**
+ * Per-dimension failure-mode taxonomy barrel — relocated to @sanity/ailf-core.
+ *
+ * Previously lived in packages/eval/src/grader/index.ts. Moved here (D-05)
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * `failureModesForDimension` without violating the core→eval import rule.
+ *
+ * Named re-exports only (W0124 — never `export *`).
+ *
+ * The eval-side packages/eval/src/grader/index.ts now re-exports from here,
+ * keeping all existing eval-side callers working without source changes.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export { COMMON_FAILURE_MODES } from "./common.js";
+export { LITERACY_FAILURE_MODES } from "./literacy.js";
+export { MCP_FAILURE_MODES } from "./mcp.js";
+export { KP_FAILURE_MODES } from "./knowledge-probe.js";
+export { AGENT_FAILURE_MODES } from "./agent-harness.js";
+import { COMMON_FAILURE_MODES } from "./common.js";
+import { LITERACY_FAILURE_MODES } from "./literacy.js";
+import { MCP_FAILURE_MODES } from "./mcp.js";
+import { KP_FAILURE_MODES } from "./knowledge-probe.js";
+import { AGENT_FAILURE_MODES } from "./agent-harness.js";
+/**
+ * Return the legal failure-mode list for a given rubric dimension.
+ *
+ * Accepts both family-level keys (`mcp-behavior`, `knowledge-probe`,
+ * `agent-harness`) and the per-template `dimension` strings used in
+ * `config/rubrics.ts` (`task-completion`, `input-validation`,
+ * `factual-correctness`, `process-quality`, …). The cross-cutting
+ * `COMMON_FAILURE_MODES` is always included.
+ *
+ * Unknown dimensions fall through to `COMMON_FAILURE_MODES` only — safe
+ * default, the grader can still pick `unclassified`.
+ */
+export function failureModesForDimension(dimension) {
+    switch (dimension) {
+        // ── Literacy family ──────────────────────────────────────
+        case "task-completion":
+        case "code-correctness":
+        case "doc-coverage":
+            return [...COMMON_FAILURE_MODES, ...LITERACY_FAILURE_MODES];
+        // ── MCP family ───────────────────────────────────────────
+        case "mcp-behavior":
+        case "input-validation":
+        case "output-correctness":
+        case "error-handling":
+        case "security":
+            return [...COMMON_FAILURE_MODES, ...MCP_FAILURE_MODES];
+        // ── Knowledge-probe family ───────────────────────────────
+        case "knowledge-probe":
+        case "factual-correctness":
+        case "completeness":
+        case "currency":
+            return [...COMMON_FAILURE_MODES, ...KP_FAILURE_MODES];
+        // ── Agent-harness family ─────────────────────────────────
+        case "agent-harness":
+        case "process-quality":
+        case "agent-output":
+        case "tool-usage":
+            return [...COMMON_FAILURE_MODES, ...AGENT_FAILURE_MODES];
+        default:
+            return COMMON_FAILURE_MODES;
+    }
+}
+/**
+ * Flat list of all dimension names recognized by `failureModesForDimension`.
+ * Used by card-validators and test calibration.
+ */
+export const CANONICAL_DIMENSIONS = [
+    // Literacy family
+    "task-completion",
+    "code-correctness",
+    "doc-coverage",
+    // MCP family
+    "mcp-behavior",
+    "input-validation",
+    "output-correctness",
+    "error-handling",
+    "security",
+    // Knowledge-probe family
+    "knowledge-probe",
+    "factual-correctness",
+    "completeness",
+    "currency",
+    // Agent-harness family
+    "agent-harness",
+    "process-quality",
+    "agent-output",
+    "tool-usage",
+];
+/**
+ * Returns `true` when `mode` appears in the failure-mode list of any
+ * canonical dimension family.
+ */
+export function isCanonicalFailureMode(mode) {
+    const allModes = new Set([
+        ...COMMON_FAILURE_MODES,
+        ...LITERACY_FAILURE_MODES,
+        ...MCP_FAILURE_MODES,
+        ...KP_FAILURE_MODES,
+        ...AGENT_FAILURE_MODES,
+    ]);
+    return allModes.has(mode);
+}

package/dist/_vendor/ailf-core/grader/failure-modes/knowledge-probe.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Knowledge-probe failure modes — valid for the `knowledge-probe` dimension
+ * family (factual-correctness, completeness, currency).
+ *
+ * Relocated from packages/eval/src/grader/knowledge-probe.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const KP_FAILURE_MODES: readonly ["factual-error", "incompleteness", "currency-violation", "hallucination"];
+export type KPFailureMode = (typeof KP_FAILURE_MODES)[number];

package/dist/_vendor/ailf-core/grader/failure-modes/knowledge-probe.js

@@ -0,0 +1,17 @@
+/**
+ * Knowledge-probe failure modes — valid for the `knowledge-probe` dimension
+ * family (factual-correctness, completeness, currency).
+ *
+ * Relocated from packages/eval/src/grader/knowledge-probe.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const KP_FAILURE_MODES = [
+    "factual-error", // assistant asserts something demonstrably false
+    "incompleteness", // assistant covers part of the answer; misses key piece
+    "currency-violation", // assistant cites stale facts beyond doc currency horizon
+    "hallucination", // assistant invents details not present in any doc
+];

package/dist/_vendor/ailf-core/grader/failure-modes/literacy.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Literacy failure modes — valid for `task-completion`, `code-correctness`,
+ * `doc-coverage` (the literacy dimension family).
+ *
+ * Relocated from packages/eval/src/grader/literacy.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const LITERACY_FAILURE_MODES: readonly ["missing-docs", "outdated-docs", "incorrect-docs", "poor-structure"];
+export type LiteracyFailureMode = (typeof LITERACY_FAILURE_MODES)[number];

package/dist/_vendor/ailf-core/grader/failure-modes/literacy.js

@@ -0,0 +1,17 @@
+/**
+ * Literacy failure modes — valid for `task-completion`, `code-correctness`,
+ * `doc-coverage` (the literacy dimension family).
+ *
+ * Relocated from packages/eval/src/grader/literacy.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const LITERACY_FAILURE_MODES = [
+    "missing-docs", // relevant doc didn't exist
+    "outdated-docs", // doc reflects an older API/version
+    "incorrect-docs", // doc states something factually wrong
+    "poor-structure", // doc exists but is hard to find or follow
+];

package/dist/_vendor/ailf-core/grader/failure-modes/mcp.d.ts

@@ -0,0 +1,13 @@
+/**
+ * MCP failure modes — valid for the `mcp-behavior` dimension family
+ * (input-validation, output-correctness, error-handling, security).
+ *
+ * Relocated from packages/eval/src/grader/mcp.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const MCP_FAILURE_MODES: readonly ["spec-mismatch", "missing-error-handling", "over-privileged", "missing-docs"];
+export type MCPFailureMode = (typeof MCP_FAILURE_MODES)[number];

package/dist/_vendor/ailf-core/grader/failure-modes/mcp.js

@@ -0,0 +1,17 @@
+/**
+ * MCP failure modes — valid for the `mcp-behavior` dimension family
+ * (input-validation, output-correctness, error-handling, security).
+ *
+ * Relocated from packages/eval/src/grader/mcp.ts to @sanity/ailf-core
+ * so card files in packages/core/src/services/diagnosis/cards/ can import
+ * them without violating the core→eval import direction rule (D-05).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const MCP_FAILURE_MODES = [
+    "spec-mismatch", // tool/server output doesn't match published MCP spec
+    "missing-error-handling", // tool failure path under-documented or absent
+    "over-privileged", // tool exposes operations the doc didn't sanction
+    "missing-docs", // re-export from literacy (cross-cutting)
+];