@sanity/ailf 4.5.0 → 5.0.0

package/dist/composition-root.js

@@ -26,6 +26,8 @@ import { LocalFilesystemArtifactWriter } from "./artifact-capture/local-fs-artif
 import { resolveUploadConcurrency, setDefaultUploadConcurrency, } from "./artifact-capture/parallel-emit.js";
 import { UploadMetrics } from "./artifact-capture/upload-metrics.js";
 import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
+import { AnthropicLLMClient, OpenAILLMClient } from "./adapters/llm/index.js";
+import { runBorderlineConsensus, } from "./pipeline/borderline-consensus-runner.js";
 import { loadExternalPresets } from "./pipeline/compiler/preset-loader.js";
 import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
 import { PromptfooEvalAdapter } from "./adapters/eval-runners/promptfoo-eval-adapter.js";
@@ -91,12 +93,20 @@ export function createAppContext(config) {
     // from the context (D0032).
     const runId = generateRunId();
     logger.debug(`Pipeline runId: ${runId}`);
+    // LLM client (D0051) — wired when an API key is present. The grader path
+    // does NOT consume this; D0051 defers grader migration as a follow-up.
+    // Env mapping happens here so `createLLMClient` stays pure and testable.
+    const llmClient = createLLMClient(config, {
+        anthropicApiKey: process.env.ANTHROPIC_API_KEY,
+        openaiApiKey: process.env.OPENAI_API_KEY,
+    }, logger);
     return {
         artifactWriter,
         cache,
         config,
         docFetcher,
         evalRunner,
+        ...(llmClient ? { llmClient } : {}),
         logger,
         packageSurfaceResolver,
         progress,
@@ -107,6 +117,44 @@ export function createAppContext(config) {
         taskSource,
     };
 }
+/**
+ * Select the LLMClient adapter based on `config.llmProvider` and the
+ * supplied API keys. Returns `undefined` when no usable credential is
+ * present — `AppContext.llmClient` stays unset and consumers handle that
+ * explicitly.
+ *
+ * Adapters never read `process.env` themselves (per
+ * `.claude/rules/typescript.md`); env mapping happens at the call site
+ * (typically `createAppContext`).
+ *
+ * Exported for unit-test access; not part of the public package API.
+ */
+export function createLLMClient(config, keys, logger) {
+    const explicit = config.llmProvider;
+    const anthropicKey = keys.anthropicApiKey;
+    const openaiKey = keys.openaiApiKey;
+    // Auto-select: prefer Anthropic when both are present (matches the
+    // current grader's default model in `config/models.ts`).
+    const provider = explicit ?? (anthropicKey ? "anthropic" : openaiKey ? "openai" : undefined);
+    if (!provider) {
+        logger.debug("LLM client: not wired — no Anthropic or OpenAI API key supplied");
+        return undefined;
+    }
+    if (provider === "anthropic") {
+        if (!anthropicKey) {
+            logger.warn('llmProvider="anthropic" but no Anthropic API key supplied — LLMClient not wired');
+            return undefined;
+        }
+        logger.debug("LLM client: AnthropicLLMClient");
+        return new AnthropicLLMClient({ apiKey: anthropicKey, logger });
+    }
+    if (!openaiKey) {
+        logger.warn('llmProvider="openai" but no OpenAI API key supplied — LLMClient not wired');
+        return undefined;
+    }
+    logger.debug("LLM client: OpenAILLMClient");
+    return new OpenAILLMClient({ apiKey: openaiKey, logger });
+}
 // ---------------------------------------------------------------------------
 // Sub-factories (extracted to keep createAppContext readable)
 // ---------------------------------------------------------------------------
@@ -446,3 +494,50 @@ function createReportStore(config) {
             undefined,
     });
 }
+// ---------------------------------------------------------------------------
+// Borderline-consensus wiring (Plan 03-04 / GRAD-04)
+// ---------------------------------------------------------------------------
+/**
+ * Severity boundaries from `packages/eval/config/thresholds.ts`
+ * (severity.critical/warning/info `composite-below` at L50/54/58 — 30, 50,
+ * 60). The borderline detector flags a judgment when its score is within
+ * ±5 of any of these. Composition-root reads them ONCE and threads the
+ * typed `readonly number[]` into `runBorderlineConsensus` rather than
+ * re-deriving them at each call site (Pitfall 5 — single source of truth
+ * for the scale).
+ */
+export const BORDERLINE_SEVERITY_THRESHOLDS = [
+    30, 50, 60,
+];
+/**
+ * Default replications per borderline judgment when the caller's
+ * `RepoConfig.execution.borderlineReplications` is unset (locked answer
+ * #4 in plan 03-04). Three replications + the original score = four
+ * scores per consistency record, which is the minimum that produces a
+ * non-degenerate stdDev / median split.
+ */
+export const DEFAULT_BORDERLINE_REPLICATIONS = 3;
+/**
+ * Factory for the borderline-consensus runner. Returns a function that
+ * applies the severity-threshold and replication defaults from
+ * composition-root, leaving the live grader entry point (the `regrade`
+ * callback) and the candidate `judgments` array as runtime inputs.
+ *
+ * The pipeline-side caller (currently `pipeline/calculate-scores.ts`'s
+ * post-extraction junction) supplies the `regrade` callback that maps a
+ * `GraderJudgment` to a fresh score via the response/rubric text from
+ * the original Promptfoo result. See the runner's header for the
+ * rationale on injecting the regrader rather than calling `gradeOnce`
+ * inline (Pitfall 6 — preserve the runner's purity wrt the existing
+ * grader-comparison split).
+ */
+export function createBorderlineConsensusRunner(opts) {
+    const replications = opts.borderlineReplications ?? DEFAULT_BORDERLINE_REPLICATIONS;
+    return (args) => runBorderlineConsensus({
+        judgments: args.judgments,
+        ...(args.logger ? { logger: args.logger } : {}),
+        regrade: args.regrade,
+        replications,
+        thresholds: BORDERLINE_SEVERITY_THRESHOLDS,
+    });
+}

package/dist/config/rubrics.ts

@@ -11,6 +11,15 @@
 
 import { defineRubrics } from "../_vendor/ailf-core/index.js"
 
+// Plan 03-02 — per-dimension failure-mode taxonomies stamped onto each
+// template entry below. Source of truth lives in packages/eval/src/grader/;
+// the helper picks the right list by dimension family.
+import { failureModesForDimension } from "../grader/index.js"
+// Single source of truth for the wire-format version stamped into the
+// grader-prompt footer (VER-01 D-02). Interpolated below so the
+// announced version cannot drift from the schema's expected value.
+import { graderJudgmentsVersion } from "../adapters/grader-outputs/index.js"
+
 export default defineRubrics({
   templates: {
     // ── Core literacy dimensions ────────────────────────────
@@ -25,6 +34,7 @@ export default defineRubrics({
         "100: Fully functional code — works as expected",
       ],
       criteria_label: "Must demonstrate:",
+      failureModes: failureModesForDimension("task-completion"),
     },
     "code-correctness": {
       dimension: "code-correctness",
@@ -37,6 +47,7 @@ export default defineRubrics({
         "100: Follows all best practices, idiomatic implementation",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("code-correctness"),
     },
     "doc-coverage": {
       dimension: "doc-coverage",
@@ -48,6 +59,7 @@ export default defineRubrics({
         "80: Minor gaps — almost everything was documented",
         "100: Complete coverage — all necessary info was in docs",
       ],
+      failureModes: failureModesForDimension("doc-coverage"),
     },
 
     // ── MCP server dimensions ───────────────────────────────
@@ -62,6 +74,7 @@ export default defineRubrics({
         "100: Perfect tool inputs — all parameters correct and well-formed",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("input-validation"),
     },
     "mcp-output-correctness": {
       dimension: "output-correctness",
@@ -74,6 +87,7 @@ export default defineRubrics({
         "100: Perfect output handling — correctly interpreted all tool responses",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("output-correctness"),
     },
     "mcp-error-handling": {
       dimension: "error-handling",
@@ -86,6 +100,7 @@ export default defineRubrics({
         "100: Excellent — handled all errors appropriately with clear messaging",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("error-handling"),
     },
     "mcp-security": {
       dimension: "security",
@@ -98,6 +113,7 @@ export default defineRubrics({
         "100: Perfect security — only used authorized tools with safe inputs",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("security"),
     },
 
     // ── Knowledge probe dimensions ──────────────────────────
@@ -112,6 +128,7 @@ export default defineRubrics({
         "100: Fully correct — all statements are accurate and verifiable",
       ],
       criteria_label: "Verify:",
+      failureModes: failureModesForDimension("factual-correctness"),
     },
     completeness: {
       dimension: "completeness",
@@ -124,6 +141,7 @@ export default defineRubrics({
         "100: Comprehensive — thorough coverage of all important aspects",
       ],
       criteria_label: "Check coverage of:",
+      failureModes: failureModesForDimension("completeness"),
     },
     currency: {
       dimension: "currency",
@@ -136,6 +154,7 @@ export default defineRubrics({
         "100: Fully current — references latest APIs, patterns, and best practices",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("currency"),
     },
 
     // ── Agent harness dimensions ────────────────────────────
@@ -151,6 +170,7 @@ export default defineRubrics({
         "100: Excellent process — optimal tool usage, clear planning, graceful recovery",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("process-quality"),
     },
     "agent-output": {
       dimension: "agent-output",
@@ -163,6 +183,7 @@ export default defineRubrics({
         "100: Excellent output — fully correct, clean, and complete",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("agent-output"),
     },
     "agent-tool-usage": {
       dimension: "tool-usage",
@@ -175,6 +196,7 @@ export default defineRubrics({
         "100: Excellent — optimal tool selection, correct inputs, minimal redundancy",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("tool-usage"),
     },
   },
 
@@ -220,6 +242,20 @@ export default defineRubrics({
     "agent-harness": { gold: "agent-harness" },
   },
 
-  footer:
-    'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+  // Phase 3 GRAD-05 (Plan 03-01) — structured GraderJudgment JSON sketch.
+  // Documents the target wire format the grader emits. The strict schema's
+  // GRAD-02 additive fields stay optional in this plan; Plan 03-04 flips
+  // them to required and bumps graderJudgmentsVersion to 1.0.0.
+  footer: `Return ONLY a JSON object with this exact shape:
+{
+  "judgmentId": "<string>",
+  "score": <number 0-100>,
+  "reason": "<explanation, ≤500 chars>",
+  "subJudgments": [{ "criterionId": "<id>", "met": <bool>, "evidence": "<≤280 chars>", "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" } }],
+  "docCitations": [{ "documentId": "<id>", "slug": "<optional slug>", "role": "supports|contradicts|missing|irrelevant", "hallucinated": <bool> }],
+  "failureMode": "<one of the declared modes for this dimension or 'unclassified'>",
+  "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" },
+  "hallucinationCheckedAgainst": ["<doc id>"],
+  "metadata": { "graderModel": "<string>", "graderJudgmentsVersion": "${graderJudgmentsVersion}" }
+}`,
 })

package/dist/grader/agent-harness.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Agent-harness failure modes — valid for the `agent-harness` dimension
+ * family (process-quality, agent-output, tool-usage).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). Agent-harness failures track how an agent
+ * uses tools and handles multi-step processes; the v0 modes are tool-misuse,
+ * chaotic-process (no plan), and missing-recovery (doesn't recover from
+ * tool errors).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const AGENT_FAILURE_MODES: readonly ["tool-misuse", "chaotic-process", "missing-recovery"];
+export type AgentFailureMode = (typeof AGENT_FAILURE_MODES)[number];

package/dist/grader/agent-harness.js

@@ -0,0 +1,17 @@
+/**
+ * Agent-harness failure modes — valid for the `agent-harness` dimension
+ * family (process-quality, agent-output, tool-usage).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). Agent-harness failures track how an agent
+ * uses tools and handles multi-step processes; the v0 modes are tool-misuse,
+ * chaotic-process (no plan), and missing-recovery (doesn't recover from
+ * tool errors).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const AGENT_FAILURE_MODES = [
+    "tool-misuse", // assistant calls tools incorrectly or with wrong args
+    "chaotic-process", // assistant flails — undirected exploration, no plan
+    "missing-recovery", // assistant doesn't recover from a tool error
+];

package/dist/grader/common.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Cross-cutting failure modes — valid for any dimension family.
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). The four cross-cutting modes capture failures
+ * that aren't tied to a specific dimension family: infrastructure failures,
+ * model ceiling effects, false-floor (model already knew the answer; docs
+ * added no value), and the low-confidence fallback. The per-dimension
+ * taxonomies (literacy, MCP, knowledge-probe, agent-harness) extend this
+ * cross-cutting list.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283 — the v0 lists)
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export declare const COMMON_FAILURE_MODES: readonly ["api-error", "model-limitation", "false-floor", "unclassified"];
+export type CommonFailureMode = (typeof COMMON_FAILURE_MODES)[number];

package/dist/grader/common.js

@@ -0,0 +1,21 @@
+/**
+ * Cross-cutting failure modes — valid for any dimension family.
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). The four cross-cutting modes capture failures
+ * that aren't tied to a specific dimension family: infrastructure failures,
+ * model ceiling effects, false-floor (model already knew the answer; docs
+ * added no value), and the low-confidence fallback. The per-dimension
+ * taxonomies (literacy, MCP, knowledge-probe, agent-harness) extend this
+ * cross-cutting list.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283 — the v0 lists)
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export const COMMON_FAILURE_MODES = [
+    "api-error", // infrastructure failure, not a docs problem
+    "model-limitation", // high ceiling, model can't reach it
+    "false-floor", // model already knew the answer; docs added no value
+    "unclassified", // grader could not pick a mode (low-confidence fallback)
+];

package/dist/grader/index.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Per-dimension failure-mode taxonomy barrel.
+ *
+ * Named re-exports only (W0124 — never `export *`).
+ *
+ * Consumers:
+ * - `packages/eval/config/rubrics.ts` — calls `failureModesForDimension()` to
+ *   stamp a per-template legal-mode list onto every rubric template entry.
+ * - `packages/eval/src/pipeline/compiler/rubric-resolution.ts` — reads
+ *   `template.failureModes` at prompt-assembly time and announces the legal
+ *   modes to the grader before the structured-shape footer (Plan 03-01).
+ * - `packages/eval/src/grader/__tests__/calibration.test.ts` — fixture-driven
+ *   ≥90% non-`unclassified` static calibration check (ROADMAP success
+ *   criterion 1).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export { COMMON_FAILURE_MODES, type CommonFailureMode } from "./common.js";
+export { LITERACY_FAILURE_MODES, type LiteracyFailureMode } from "./literacy.js";
+export { MCP_FAILURE_MODES, type MCPFailureMode } from "./mcp.js";
+export { KP_FAILURE_MODES, type KPFailureMode } from "./knowledge-probe.js";
+export { AGENT_FAILURE_MODES, type AgentFailureMode } from "./agent-harness.js";
+/**
+ * Return the legal failure-mode list for a given rubric dimension.
+ *
+ * Accepts both family-level keys (`mcp-behavior`, `knowledge-probe`,
+ * `agent-harness`) and the per-template `dimension` strings used in
+ * `config/rubrics.ts` (`task-completion`, `input-validation`,
+ * `factual-correctness`, `process-quality`, …). The cross-cutting
+ * `COMMON_FAILURE_MODES` is always included.
+ *
+ * Unknown dimensions fall through to `COMMON_FAILURE_MODES` only — safe
+ * default, the grader can still pick `unclassified`.
+ */
+export declare function failureModesForDimension(dimension: string): readonly string[];

package/dist/grader/index.js

@@ -0,0 +1,75 @@
+/**
+ * Per-dimension failure-mode taxonomy barrel.
+ *
+ * Named re-exports only (W0124 — never `export *`).
+ *
+ * Consumers:
+ * - `packages/eval/config/rubrics.ts` — calls `failureModesForDimension()` to
+ *   stamp a per-template legal-mode list onto every rubric template entry.
+ * - `packages/eval/src/pipeline/compiler/rubric-resolution.ts` — reads
+ *   `template.failureModes` at prompt-assembly time and announces the legal
+ *   modes to the grader before the structured-shape footer (Plan 03-01).
+ * - `packages/eval/src/grader/__tests__/calibration.test.ts` — fixture-driven
+ *   ≥90% non-`unclassified` static calibration check (ROADMAP success
+ *   criterion 1).
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader model;
+ *      taxonomies travel with the rubric prompt for reproducibility.
+ */
+export { COMMON_FAILURE_MODES } from "./common.js";
+export { LITERACY_FAILURE_MODES } from "./literacy.js";
+export { MCP_FAILURE_MODES } from "./mcp.js";
+export { KP_FAILURE_MODES } from "./knowledge-probe.js";
+export { AGENT_FAILURE_MODES } from "./agent-harness.js";
+import { COMMON_FAILURE_MODES } from "./common.js";
+import { LITERACY_FAILURE_MODES } from "./literacy.js";
+import { MCP_FAILURE_MODES } from "./mcp.js";
+import { KP_FAILURE_MODES } from "./knowledge-probe.js";
+import { AGENT_FAILURE_MODES } from "./agent-harness.js";
+/**
+ * Return the legal failure-mode list for a given rubric dimension.
+ *
+ * Accepts both family-level keys (`mcp-behavior`, `knowledge-probe`,
+ * `agent-harness`) and the per-template `dimension` strings used in
+ * `config/rubrics.ts` (`task-completion`, `input-validation`,
+ * `factual-correctness`, `process-quality`, …). The cross-cutting
+ * `COMMON_FAILURE_MODES` is always included.
+ *
+ * Unknown dimensions fall through to `COMMON_FAILURE_MODES` only — safe
+ * default, the grader can still pick `unclassified`.
+ */
+export function failureModesForDimension(dimension) {
+    switch (dimension) {
+        // ── Literacy family ──────────────────────────────────────
+        case "task-completion":
+        case "code-correctness":
+        case "doc-coverage":
+            return [...COMMON_FAILURE_MODES, ...LITERACY_FAILURE_MODES];
+        // ── MCP family ───────────────────────────────────────────
+        // `mcp-behavior` is the family-level key (profile / depends-on
+        // shorthand). The per-template `dimension` strings are the four
+        // entries from config/rubrics.ts mcp-* templates.
+        case "mcp-behavior":
+        case "input-validation":
+        case "output-correctness":
+        case "error-handling":
+        case "security":
+            return [...COMMON_FAILURE_MODES, ...MCP_FAILURE_MODES];
+        // ── Knowledge-probe family ───────────────────────────────
+        case "knowledge-probe":
+        case "factual-correctness":
+        case "completeness":
+        case "currency":
+            return [...COMMON_FAILURE_MODES, ...KP_FAILURE_MODES];
+        // ── Agent-harness family ─────────────────────────────────
+        case "agent-harness":
+        case "process-quality":
+        case "agent-output":
+        case "tool-usage":
+            return [...COMMON_FAILURE_MODES, ...AGENT_FAILURE_MODES];
+        default:
+            return COMMON_FAILURE_MODES;
+    }
+}

package/dist/grader/knowledge-probe.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Knowledge-probe failure modes — valid for the `knowledge-probe` dimension
+ * family (factual-correctness, completeness, currency).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). Knowledge-probe failures track the model's
+ * ability to recall facts about Sanity's surface area; the v0 modes
+ * differentiate factual errors from omissions, currency drift, and
+ * hallucination.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const KP_FAILURE_MODES: readonly ["factual-error", "incompleteness", "currency-violation", "hallucination"];
+export type KPFailureMode = (typeof KP_FAILURE_MODES)[number];

package/dist/grader/knowledge-probe.js

@@ -0,0 +1,18 @@
+/**
+ * Knowledge-probe failure modes — valid for the `knowledge-probe` dimension
+ * family (factual-correctness, completeness, currency).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). Knowledge-probe failures track the model's
+ * ability to recall facts about Sanity's surface area; the v0 modes
+ * differentiate factual errors from omissions, currency drift, and
+ * hallucination.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const KP_FAILURE_MODES = [
+    "factual-error", // assistant asserts something demonstrably false
+    "incompleteness", // assistant covers part of the answer; misses key piece
+    "currency-violation", // assistant cites stale facts beyond doc currency horizon
+    "hallucination", // assistant invents details not present in any doc
+];

package/dist/grader/literacy.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Literacy failure modes — valid for `task-completion`, `code-correctness`,
+ * `doc-coverage` (the literacy dimension family).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). The v0 list is the four canonical
+ * documentation-quality failure modes. Conservative on purpose; calibration
+ * (Plan 03-02 Task 3) reveals whether expansion is needed in a follow-on.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const LITERACY_FAILURE_MODES: readonly ["missing-docs", "outdated-docs", "incorrect-docs", "poor-structure"];
+export type LiteracyFailureMode = (typeof LITERACY_FAILURE_MODES)[number];

package/dist/grader/literacy.js

@@ -0,0 +1,17 @@
+/**
+ * Literacy failure modes — valid for `task-completion`, `code-correctness`,
+ * `doc-coverage` (the literacy dimension family).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). The v0 list is the four canonical
+ * documentation-quality failure modes. Conservative on purpose; calibration
+ * (Plan 03-02 Task 3) reveals whether expansion is needed in a follow-on.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const LITERACY_FAILURE_MODES = [
+    "missing-docs", // relevant doc didn't exist
+    "outdated-docs", // doc reflects an older API/version
+    "incorrect-docs", // doc states something factually wrong
+    "poor-structure", // doc exists but is hard to find or follow
+];

package/dist/grader/mcp.d.ts

@@ -0,0 +1,14 @@
+/**
+ * MCP failure modes — valid for the `mcp-behavior` dimension family
+ * (input-validation, output-correctness, error-handling, security).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). `missing-docs` is intentionally re-exported
+ * from the literacy family — MCP server tasks frequently fail because the
+ * MCP spec itself is under-documented; that's a literacy failure even when
+ * surfaced through MCP grading.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export declare const MCP_FAILURE_MODES: readonly ["spec-mismatch", "missing-error-handling", "over-privileged", "missing-docs"];
+export type MCPFailureMode = (typeof MCP_FAILURE_MODES)[number];

package/dist/grader/mcp.js

@@ -0,0 +1,18 @@
+/**
+ * MCP failure modes — valid for the `mcp-behavior` dimension family
+ * (input-validation, output-correctness, error-handling, security).
+ *
+ * Phase 3 GRAD-03 (Plan 03-02). `missing-docs` is intentionally re-exported
+ * from the literacy family — MCP server tasks frequently fail because the
+ * MCP spec itself is under-documented; that's a literacy failure even when
+ * surfaced through MCP grading.
+ *
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ *      §"Per-dimension failure-mode taxonomies" (lines 239-283).
+ */
+export const MCP_FAILURE_MODES = [
+    "spec-mismatch", // tool/server output doesn't match published MCP spec
+    "missing-error-handling", // tool failure path under-documented or absent
+    "over-privileged", // tool exposes operations the doc didn't sanction
+    "missing-docs", // re-export from literacy (cross-cutting)
+];

package/dist/orchestration/build-app-context.js

@@ -49,6 +49,7 @@ export function mapToResolvedConfig(opts, rootDir) {
         noCache: opts.noCache,
         noRemoteCache: opts.noRemoteCache,
         graderReplications: opts.graderReplications,
+        borderlineReplications: opts.borderlineReplications,
         graderContext: opts.graderContext,
         outputDir: opts.outputDir,
         outputPath: opts.outputPath,

package/dist/orchestration/build-step-sequence.js

@@ -8,6 +8,7 @@
 import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 import { CallbackStep } from "./steps/callback-step.js";
 import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
+import { ComputeAttributionStep } from "./steps/compute-attribution-step.js";
 import { CompareStep } from "./steps/compare-step.js";
 import { FetchDocsStep } from "./steps/fetch-docs-step.js";
 import { FinalizeRunStep } from "./steps/finalize-run-step.js";
@@ -75,6 +76,10 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
     if (config.gapAnalysisEnabled) {
         steps.push(new GapAnalysisStep());
     }
+    // Step 4b2: Per-judgment attribution ensemble (default-on).
+    // Depends on documentManifest being enriched onto score-summary.json
+    // by gap-analysis. Skipped silently when upstream files are missing.
+    steps.push(new ComputeAttributionStep());
     // Step 4c: Finalize the run — write `runs/{runId}/manifest.json` with the
     // catalog of artifacts produced so far. Skipped silently when no
     // artifactWriter is wired (D0032).

package/dist/orchestration/steps/calculate-scores-step.js

@@ -14,6 +14,8 @@ import { buildCacheContext } from "../cache-context.js";
 import { calculateAndWriteScores } from "../../pipeline/calculate-scores.js";
 import { checkResultsExist, checkScoreSummaryValid, } from "../../pipeline/checks.js";
 import { resultsFileForMode } from "../../pipeline/eval-constants.js";
+import { gradeOnce, loadGraderModel } from "../../pipeline/grader-api.js";
+import { createBorderlineConsensusRunner } from "../../composition-root.js";
 import { loadPreflightScoring } from "../../pipeline/preflight/load-preflight-scoring.js";
 import { loadSource } from "../../sources.js";
 import { uploadTestOutputs } from "../../pipeline/upload-test-outputs.js";
@@ -85,10 +87,30 @@ export class CalculateScoresStep {
             ctx.logger.warn(`[warn] W0198 preflight: failed to load preflight-scoring config — ${err instanceof Error ? err.message : String(err)}`);
             return undefined;
         });
+        // CR-01 — wire the borderline-consensus runner end-to-end. The
+        // composition root owns the threshold + replication defaults; the
+        // orchestration step supplies the regrade entry point (gradeOnce
+        // against the configured grader model). Built lazily — when no
+        // judgments are extracted (or none land in the ±5 borderline band),
+        // the runner short-circuits without paying the grader-model load.
+        let borderlineRegradeOnce;
+        try {
+            const grader = loadGraderModel(ctx.config.rootDir);
+            borderlineRegradeOnce = (responseText, rubricText) => gradeOnce(grader.id, responseText, rubricText, ctx.logger);
+        }
+        catch (err) {
+            ctx.logger.warn(`[warn] borderline consensus skipped — grader model not loadable: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        const borderlineConsensusRunner = createBorderlineConsensusRunner(ctx.config.borderlineReplications !== undefined
+            ? { borderlineReplications: ctx.config.borderlineReplications }
+            : {});
         let belowCritical = [];
         try {
-            const result = calculateAndWriteScores({
+            const result = await calculateAndWriteScores({
                 allowedOrigins: ctx.config.allowedOrigins,
+                ...(borderlineRegradeOnce
+                    ? { borderlineConsensusRunner, borderlineRegradeOnce }
+                    : {}),
                 logger: ctx.logger,
                 // Pass the variant for literacy (scoring uses it to decide
                 // whether to read agentic results), or mode for other modes