@sanity/ailf 7.0.0 → 7.1.0

package/dist/pipeline/calculate-scores.js

@@ -32,7 +32,7 @@ import { join } from "path";
 import { classifyRubric, detectFeatureArea, extractUrlMetadata, generateJudgmentId, mergeScores, parseRubricScore, resolveVariantMode, } from "../_vendor/ailf-core/index.js";
 import { calculateCost } from "../agent-observer/pricing.js";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
-import { GraderJudgmentSchema, graderJudgmentsVersion, } from "../adapters/grader-outputs/promptfoo-grader-output.js";
+import { GraderEmittedJudgmentSchema, graderJudgmentsVersion, } from "../adapters/grader-outputs/promptfoo-grader-output.js";
 import { validateFailureMode } from "./failure-modes.js";
 import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
 import { checkResultsExist } from "./checks.js";
@@ -184,34 +184,70 @@ export function extractGraderJudgments(resultsPath, telemetry) {
                 continue;
             }
             const score = parseRubricScore(comp);
-            // Extract the reason text — the grader's reasoning. Plan 03-01
-            // (D0045 trust boundary): the inline `JSON.parse + as`-cast at
-            // this site is replaced with `GraderJudgmentSchema.safeParse`
-            // so that grader output flows through a validated schema before
-            // it enters the scoring pipeline. On parse failure we fall to
-            // an `unclassified`-shape Phase 1 judgment built from the raw
-            // reason string — NEVER fall back to the legacy parser (Pitfall
-            // 4: strict and legacy schemas are deliberate siblings, not a
-            // fall-through chain).
+            // Extract the reason text — the grader's reasoning. W0273 splits
+            // the parse boundary into a wire shape (`GraderEmittedJudgmentSchema`
+            // — only fields the LLM controls) and a storage shape
+            // (`GraderJudgmentSchema` — full strict surface). The pipeline
+            // parses against the wire shape, then synthesizes the pipeline-owned
+            // fields (judgmentId, metadata.{graderModel,graderJudgmentsVersion},
+            // hallucinationCheckedAgainst) plus the result-context fields
+            // (taskId, modelId, dimension) to build the full storage shape.
+            //
+            // On parse failure we fall to an `unclassified`-shape Phase 1
+            // judgment built from the raw reason string — NEVER fall back to
+            // the legacy parser (Pitfall 4: strict and legacy schemas are
+            // deliberate siblings, not a fall-through chain).
             const reasonRaw = comp.reason ?? "";
             let parsedJudgment = null;
             let reason = reasonRaw;
             if (reasonRaw) {
                 try {
                     const candidate = JSON.parse(reasonRaw);
-                    // The strict schema asserts the full GraderJudgment surface.
-                    // safeParse handles non-object inputs (number, array, etc.)
-                    // by failing — we don't pre-narrow here.
+                    // The wire schema asserts only the LLM-emit subset. safeParse
+                    // handles non-object inputs (number, array, etc.) by failing —
+                    // we don't pre-narrow here.
                     const candidateObj = candidate && typeof candidate === "object" ? candidate : {};
-                    const result = GraderJudgmentSchema.safeParse({
-                        ...candidateObj,
-                        taskId,
-                        modelId,
-                        dimension: kind,
-                    });
+                    const result = GraderEmittedJudgmentSchema.safeParse(candidateObj);
                     if (result.success) {
-                        parsedJudgment = result.data;
-                        reason = result.data.reason;
+                        const emitted = result.data;
+                        parsedJudgment = {
+                            // Result-context fields — pipeline-supplied:
+                            taskId,
+                            modelId,
+                            dimension: kind,
+                            // Wire-emitted fields — LLM-controlled:
+                            score: emitted.score,
+                            reason: emitted.reason,
+                            failureMode: emitted.failureMode,
+                            subJudgments: emitted.subJudgments,
+                            docCitations: emitted.docCitations,
+                            confidence: emitted.confidence,
+                            ...(emitted.outputFailure && {
+                                outputFailure: emitted.outputFailure,
+                            }),
+                            // Pipeline-owned fields — synthesized:
+                            judgmentId: generateJudgmentId({
+                                taskId,
+                                modelId,
+                                dimension: kind,
+                                ...(telemetry?.runId ? { runId: telemetry.runId } : {}),
+                            }),
+                            // hallucinationCheckedAgainst is filled in later by
+                            // populateHallucinationFields (gap-analysis-step.ts) — it
+                            // needs the run.documentManifest union that isn't visible
+                            // here. Empty array is the documented pre-fill placeholder.
+                            hallucinationCheckedAgainst: [],
+                            metadata: {
+                                // graderModel is threaded via the existing
+                                // telemetry.reliability channel. When upstream wires the
+                                // real grader-provider alias into reliability.graderModel,
+                                // it propagates here automatically; today it's "unknown"
+                                // (matching the pre-W0273 synthesized-fallback default).
+                                graderModel: telemetry?.reliability.graderModel ?? "unknown",
+                                graderJudgmentsVersion,
+                            },
+                        };
+                        reason = emitted.reason;
                     }
                     else {
                         // Parse failure — drop to failureMode='unclassified' below.

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.d.ts

@@ -6,7 +6,7 @@
  */
 import type { LiteracyTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
 import type { PromptfooAssertion } from "../../assertion-mapper.js";
-import type { LiteracyCompileOptions } from "./types.js";
+import type { LiteracyCompileOptions, RubricResolutionInput } from "./types.js";
 export declare function resolveAssertions(task: LiteracyTaskDefinition, options: LiteracyCompileOptions | undefined, warnings: string[], canonicalReference?: string): PromptfooAssertion[];
 /**
  * Build baseline assertions matching the legacy expand-tasks behavior.
@@ -14,5 +14,10 @@ export declare function resolveAssertions(task: LiteracyTaskDefinition, options:
  * - "full": all assertions carried over
  * - "abbreviated": only first llm-rubric with shortened prompt
  * - "none": no assertions
+ *
+ * `rubricConfig` supplies the W0273 wire-shape footer for the abbreviated
+ * mode's synthetic rubric. Without it the abbreviated emission would fail
+ * `GraderEmittedJudgmentSchema.safeParse` (missing failureMode,
+ * subJudgments, docCitations, confidence).
  */
-export declare function buildBaselineAssertions(goldAssertions: PromptfooAssertion[], rubricMode?: "abbreviated" | "full" | "none"): PromptfooAssertion[];
+export declare function buildBaselineAssertions(goldAssertions: PromptfooAssertion[], rubricMode?: "abbreviated" | "full" | "none", rubricConfig?: RubricResolutionInput): PromptfooAssertion[];

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js

@@ -45,8 +45,10 @@ function buildDocCoverageAssertion(rubricConfig, graderProvider, canonicalRefere
         return null;
     const template = rubricConfig.templates["doc-coverage"];
     const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
-        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    // W0273 — use the centralized wire-shape footer so the grader emission
+    // parses against GraderEmittedJudgmentSchema. The pre-W0273 short
+    // {score, reason} footer caused 100% parse failures starting 2026-05-11.
+    const rubricValue = `${template.header}\n${scaleText}\n\n` + `${rubricConfig.footer}`;
     // doc-coverage benefits from the same authoritative reference — the grader
     // needs the doc content to judge whether the candidate actually used what
     // was documented.
@@ -92,8 +94,13 @@ function buildDocCoverageRubricPrompt(rubric, reference) {
  * - "full": all assertions carried over
  * - "abbreviated": only first llm-rubric with shortened prompt
  * - "none": no assertions
+ *
+ * `rubricConfig` supplies the W0273 wire-shape footer for the abbreviated
+ * mode's synthetic rubric. Without it the abbreviated emission would fail
+ * `GraderEmittedJudgmentSchema.safeParse` (missing failureMode,
+ * subJudgments, docCitations, confidence).
  */
-export function buildBaselineAssertions(goldAssertions, rubricMode) {
+export function buildBaselineAssertions(goldAssertions, rubricMode, rubricConfig) {
     const mode = rubricMode ?? "full";
     if (mode === "none")
         return [];
@@ -106,10 +113,12 @@ export function buildBaselineAssertions(goldAssertions, rubricMode) {
         if (a.type === "llm-rubric") {
             if (!foundFirst) {
                 foundFirst = true;
+                const footer = rubricConfig?.footer ??
+                    'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}';
                 abbreviated.push({
                     type: "llm-rubric",
                     value: "Score task completion from 0 to 100 (same criteria as above).\n" +
-                        'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+                        footer,
                     ...(a.provider ? { provider: a.provider } : {}),
                 });
             }

package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js

@@ -134,7 +134,7 @@ function buildTestCases(task, evalMode, options, warnings) {
     if (evalMode !== "agentic") {
         const baselineEnabled = task.baseline?.enabled !== false;
         if (baselineEnabled) {
-            const baselineAssertions = buildBaselineAssertions(assertions, task.baseline?.rubric);
+            const baselineAssertions = buildBaselineAssertions(assertions, task.baseline?.rubric, options?.rubricConfig);
             tests.push({
                 description: `${taskTitle} (baseline)`,
                 vars: {

package/dist/pipeline/compiler/provider-assembler.d.ts

@@ -50,8 +50,15 @@ export interface ModelsAndProviders {
  * Returns provider arrays keyed by literacy variant name (baseline,
  * agentic, observed). These are consumed by the YAML writer to produce
  * the per-variant promptfoo config files.
+ *
+ * `loaded` (optional) lets callers pre-load and pre-filter the
+ * `ModelsConfig` so a caller-side filter (e.g. W0281's
+ * `filterModelsByRequest`) actually takes effect on the assembled
+ * providers — building providers from the unfiltered set would silently
+ * defeat the filter, since promptfoo decides which LLMs to call from the
+ * providers array, not the returned `models` field.
  */
-export declare function loadModelsAndProviders(rootDir: string, source?: ResolvedSourceConfig, searchMode?: string, allowedOrigins?: string[]): ModelsAndProviders;
+export declare function loadModelsAndProviders(rootDir: string, source?: ResolvedSourceConfig, searchMode?: string, allowedOrigins?: string[], loaded?: ModelsConfig): ModelsAndProviders;
 /**
  * Resolve `maxToolRounds` for an agentic variant (W0134).
  *
@@ -60,3 +67,10 @@ export declare function loadModelsAndProviders(rootDir: string, source?: Resolve
  * > hard fallback (5).
  */
 export declare function resolveMaxToolRounds(models: ModelsConfig, model: ModelsConfig["models"][number], variant: "agentic-naive" | "agentic-optimized"): number;
+/**
+ * Load the `ModelsConfig` for `rootDir` from disk. Exported so callers
+ * that need to pre-filter the model set before provider assembly (e.g.
+ * `PipelineRequest.models`) can hand the filtered config back to
+ * `loadModelsAndProviders` via its optional `loaded` parameter.
+ */
+export declare function loadModelsYaml(rootDir: string): ModelsConfig;

package/dist/pipeline/compiler/provider-assembler.js

@@ -64,9 +64,16 @@ function applyReplaySwap(providers) {
  * Returns provider arrays keyed by literacy variant name (baseline,
  * agentic, observed). These are consumed by the YAML writer to produce
  * the per-variant promptfoo config files.
+ *
+ * `loaded` (optional) lets callers pre-load and pre-filter the
+ * `ModelsConfig` so a caller-side filter (e.g. W0281's
+ * `filterModelsByRequest`) actually takes effect on the assembled
+ * providers — building providers from the unfiltered set would silently
+ * defeat the filter, since promptfoo decides which LLMs to call from the
+ * providers array, not the returned `models` field.
  */
-export function loadModelsAndProviders(rootDir, source, searchMode, allowedOrigins) {
-    const models = loadModelsYaml(rootDir);
+export function loadModelsAndProviders(rootDir, source, searchMode, allowedOrigins, loaded) {
+    const models = loaded ?? loadModelsYaml(rootDir);
     return {
         models,
         providers: {
@@ -203,6 +210,12 @@ export function resolveMaxToolRounds(models, model, variant) {
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
-function loadModelsYaml(rootDir) {
+/**
+ * Load the `ModelsConfig` for `rootDir` from disk. Exported so callers
+ * that need to pre-filter the model set before provider assembly (e.g.
+ * `PipelineRequest.models`) can hand the filtered config back to
+ * `loadModelsAndProviders` via its optional `loaded` parameter.
+ */
+export function loadModelsYaml(rootDir) {
     return loadConfigFile("models", rootDir).data;
 }

package/dist/pipeline/failure-modes.d.ts

@@ -1,24 +1,34 @@
 /**
  * pipeline/failure-modes.ts
  *
- * Ceiling-cross-check failure-mode validator + report assembly.
+ * Ceiling-cross-check failure-mode validator + report assembly + keyword
+ * fallback classifier.
  *
  * The grader emits `failureMode` directly under the per-dimension taxonomy
- * (Plan 03-02 — `packages/eval/src/grader/`). This module consumes the
- * grader's emission as the source of truth and uses the surviving ceiling
- * decomposition (`classifyByCeiling`) only as a CONFIDENCE VALIDATOR — it
- * cross-checks the emitted mode against structural score signals and emits
- * a D0049 `Confidence` triple stamped with `derivation: "ceiling-cross-check"`.
+ * (Plan 03-02 — `packages/eval/src/grader/`) when its structured response
+ * is available to the pipeline. In practice (W0273 discovery), Promptfoo's
+ * `llm-rubric` post-processor extracts `score` + `reason` from the grader's
+ * JSON envelope and discards the rest of the structured surface — including
+ * `failureMode`. The wire-shape footer instructs the LLM correctly but the
+ * structured fields never reach `extractGraderJudgments`, so every emission
+ * arrives as the synthesized `failureMode: "unclassified"` placeholder.
  *
- * The legacy keyword-pattern classifier (and its five regex pattern
- * constants) was deleted in Plan 03-03 — its production coverage was ~1%
- * (Doc 03 §"Where classification lives"), and resurrecting it as a fallback
- * is explicitly out of scope.
+ * To restore the pre-2026-05-11 classification rate (15-23% → 0% → 15-23%),
+ * a keyword-pattern classifier is run as a FALLBACK when the grader's
+ * emitted mode is `"unclassified"` and the score is below the classification
+ * threshold. Plan 03-03 deleted this classifier in favor of grader-emission
+ * source-of-truth; W0273 reinstates it because the grader-emission path is
+ * blocked by Promptfoo's lossy `llm-rubric` parsing. The long-term fix
+ * (capturing the grader's full structured response) is tracked separately.
+ *
+ * `classifyByCeiling` continues to serve as the confidence cross-check.
  *
  * @see docs/decisions/D0005-grader-model-separation.md — single grader emits
  *      failureMode under the per-dimension taxonomy
  * @see docs/decisions/D0049-shared-confidence-contract.md — Confidence triple
  *      shape and `ceiling-cross-check` derivation tag
+ * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md — W0273 root
+ *      cause (Promptfoo strips structured fields)
  */
 import type { Confidence } from "../_vendor/ailf-core/index.d.ts";
 import type { FailureModeReport, FeatureScore, GraderJudgment } from "./types.js";

package/dist/pipeline/failure-modes.js

@@ -1,24 +1,34 @@
 /**
  * pipeline/failure-modes.ts
  *
- * Ceiling-cross-check failure-mode validator + report assembly.
+ * Ceiling-cross-check failure-mode validator + report assembly + keyword
+ * fallback classifier.
  *
  * The grader emits `failureMode` directly under the per-dimension taxonomy
- * (Plan 03-02 — `packages/eval/src/grader/`). This module consumes the
- * grader's emission as the source of truth and uses the surviving ceiling
- * decomposition (`classifyByCeiling`) only as a CONFIDENCE VALIDATOR — it
- * cross-checks the emitted mode against structural score signals and emits
- * a D0049 `Confidence` triple stamped with `derivation: "ceiling-cross-check"`.
+ * (Plan 03-02 — `packages/eval/src/grader/`) when its structured response
+ * is available to the pipeline. In practice (W0273 discovery), Promptfoo's
+ * `llm-rubric` post-processor extracts `score` + `reason` from the grader's
+ * JSON envelope and discards the rest of the structured surface — including
+ * `failureMode`. The wire-shape footer instructs the LLM correctly but the
+ * structured fields never reach `extractGraderJudgments`, so every emission
+ * arrives as the synthesized `failureMode: "unclassified"` placeholder.
  *
- * The legacy keyword-pattern classifier (and its five regex pattern
- * constants) was deleted in Plan 03-03 — its production coverage was ~1%
- * (Doc 03 §"Where classification lives"), and resurrecting it as a fallback
- * is explicitly out of scope.
+ * To restore the pre-2026-05-11 classification rate (15-23% → 0% → 15-23%),
+ * a keyword-pattern classifier is run as a FALLBACK when the grader's
+ * emitted mode is `"unclassified"` and the score is below the classification
+ * threshold. Plan 03-03 deleted this classifier in favor of grader-emission
+ * source-of-truth; W0273 reinstates it because the grader-emission path is
+ * blocked by Promptfoo's lossy `llm-rubric` parsing. The long-term fix
+ * (capturing the grader's full structured response) is tracked separately.
+ *
+ * `classifyByCeiling` continues to serve as the confidence cross-check.
  *
  * @see docs/decisions/D0005-grader-model-separation.md — single grader emits
  *      failureMode under the per-dimension taxonomy
  * @see docs/decisions/D0049-shared-confidence-contract.md — Confidence triple
  *      shape and `ceiling-cross-check` derivation tag
+ * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md — W0273 root
+ *      cause (Promptfoo strips structured fields)
  */
 import { LEGACY_FAILURE_MODES, detectFeatureArea } from "../_vendor/ailf-core/index.js";
 // ---------------------------------------------------------------------------
@@ -27,6 +37,20 @@ import { LEGACY_FAILURE_MODES, detectFeatureArea } from "../_vendor/ailf-core/in
 /** Only classify judgments with scores below this threshold */
 const CLASSIFICATION_THRESHOLD = 60;
 // ---------------------------------------------------------------------------
+// Keyword patterns (W0273 fallback)
+//
+// Verbatim from the pre-Plan-03-03 implementation. Used only when the
+// grader's emitted `failureMode` is `"unclassified"` — the grader's
+// emission still wins whenever it actually reaches the pipeline.
+// ---------------------------------------------------------------------------
+/** API error pattern — checked FIRST to prevent timeout errors containing
+ *  "deprecated" from being misclassified as outdated-docs. */
+const API_ERROR_PATTERN = /\[api-error\]|timeout|timed out|rate limit|429|503|ECONNRESET|ETIMEDOUT|socket hang up|fetch failed/i;
+const OUTDATED_PATTERN = /deprecated|old api|v[0-9]+ syntax|no longer supported|legacy|previous version|outdated|superseded|replaced by/i;
+const MISSING_PATTERN = /no documentation|not covered|had to guess|not found|missing.*doc|no.*information|undocumented|couldn't find|without.*documentation/i;
+const INCORRECT_PATTERN = /contradicts|incorrect.*doc|doc.*incorrect|wrong.*doc|doc.*wrong|documentation says.*but|factual error|inaccurate|misleading.*doc/i;
+const POOR_STRUCTURE_PATTERN = /unclear|ambiguous|couldn't determine|conflicting|confusing|hard to follow|poorly organized|scattered|fragmented/i;
+// ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
 /**
@@ -69,13 +93,25 @@ export function buildFailureModeReport(judgments, scores) {
         // grader's actual taxonomy choice rather than a collapsed
         // `"unclassified"` bucket.
         const emittedMode = readEmittedMode(judgment);
+        // W0273 fallback — when the grader's emitted mode is "unclassified"
+        // (the synthesized-unparsed-judgment placeholder; in practice this
+        // is every judgment today because Promptfoo's llm-rubric strips the
+        // grader's structured response), try keyword classification against
+        // the reason prose. Gated on score < CLASSIFICATION_THRESHOLD so
+        // passing judgments don't get spurious classifications.
+        const keywordFallback = emittedMode === "unclassified" &&
+            judgment.score < CLASSIFICATION_THRESHOLD
+            ? classifyByKeyword(judgment.reason)
+            : null;
         // Cross-check the grader's emission against ceiling decomposition.
         const stamp = validateFailureMode(judgment, ceilingScore, floorScore);
-        const classification = {
-            confidence: stamp.level,
-            mode: emittedMode,
-            source: "ceiling",
-        };
+        const classification = keywordFallback
+            ? keywordFallback
+            : {
+                confidence: stamp.level,
+                mode: emittedMode,
+                source: "ceiling",
+            };
         classifiedJudgments.push({ classification, judgment });
         summary[classification.mode] = (summary[classification.mode] ?? 0) + 1;
         // Per-area tracking
@@ -282,6 +318,39 @@ function readEmittedMode(judgment) {
     }
     return emitted;
 }
+/**
+ * Classify the failure mode of a low-scoring grader judgment by matching
+ * keyword patterns against the reason prose. Returns `null` when no
+ * pattern matches. Patterns checked in priority order (API errors first
+ * so timeout messages containing "deprecated" don't get misclassified
+ * as outdated-docs).
+ *
+ * W0273 — reinstated as a fallback when the grader's emitted failureMode
+ * is "unclassified". Plan 03-03 deleted this code in favor of grader-
+ * emission source-of-truth; the deletion is reversed here because
+ * Promptfoo's llm-rubric post-processor strips the grader's structured
+ * response (only score + reason survive into `comp.*`), so the
+ * grader-emission path produces 0% classification on every run.
+ */
+function classifyByKeyword(reason) {
+    const lower = reason.toLowerCase();
+    if (API_ERROR_PATTERN.test(lower)) {
+        return { confidence: "high", mode: "api-error", source: "keyword" };
+    }
+    if (OUTDATED_PATTERN.test(lower)) {
+        return { confidence: "high", mode: "outdated-docs", source: "keyword" };
+    }
+    if (MISSING_PATTERN.test(lower)) {
+        return { confidence: "high", mode: "missing-docs", source: "keyword" };
+    }
+    if (INCORRECT_PATTERN.test(lower)) {
+        return { confidence: "medium", mode: "incorrect-docs", source: "keyword" };
+    }
+    if (POOR_STRUCTURE_PATTERN.test(lower)) {
+        return { confidence: "medium", mode: "poor-structure", source: "keyword" };
+    }
+    return null;
+}
 /**
  * Classify by ceiling-decomposition structural signals — preserved
  * verbatim from the pre-Plan-03-03 implementation. The function itself

package/dist/pipeline/map-request-to-config.js

@@ -37,6 +37,7 @@ export function mapRequestToConfig(request, rootDir) {
         mode,
         variant,
         debug: mapDebug(request.debug),
+        models: request.models,
         areas: request.areas,
         tasks: request.tasks,
         changedDocs: request.changedDocs,
@@ -46,6 +47,7 @@ export function mapRequestToConfig(request, rootDir) {
         compareEnabled: request.compare ?? false,
         compareThreshold: request.compareThreshold,
         compareBaseline: request.compareBaseline,
+        compareBaselineReportId: request.compareBaselineReportId,
         gapAnalysisEnabled: request.gapAnalysis ?? true,
         publishEnabled: request.publish ?? publishDefault,
         publishTag: request.publishTag,

package/dist/pipeline/normalize-mode.d.ts

@@ -35,7 +35,7 @@ export type LiteracyVariantName = (typeof LiteracyVariant)[keyof typeof Literacy
 export type LiteracyEvalSubMode = typeof LiteracyVariant.STANDARD | typeof LiteracyVariant.AGENTIC;
 export interface NormalizedMode {
     mode: EvalMode;
-    variant?: string;
+    variant?: LiteracyVariantName;
 }
 /**
  * Normalize a raw CLI mode string to a canonical mode + optional variant.

package/dist/pipeline/normalize-mode.js

@@ -55,6 +55,8 @@ const ALL_ACCEPTED = [
 export function normalizeMode(input) {
     if (LEGACY_LITERACY_VARIANTS.has(input)) {
         console.warn(`⚠ Deprecated: --mode ${input} is a legacy alias. Use --mode literacy --variant ${input} instead.`);
+        // The membership check above narrows `input` to LITERACY_VARIANTS — the
+        // cast is to the closed type, not a widening.
         return { mode: "literacy", variant: input };
     }
     if (CANONICAL_MODES.has(input)) {

package/dist/pipeline/run-context.d.ts

@@ -13,7 +13,7 @@
  * @see docs/decisions/D0032-run-anchored-artifact-store.md (§ Move 5 — Drift Prevention)
  */
 import { type Logger, type RunContext } from "../_vendor/ailf-core/index.d.ts";
-import { type RunClassification, type RunExecutor, type RunExecutorSurface, type RunHost, type RunLineage, type RunOwner, type RunTool } from "../_vendor/ailf-shared/index.d.ts";
+import { type LiteracyVariant, type RunClassification, type RunExecutor, type RunExecutorSurface, type RunHost, type RunLineage, type RunOwner, type RunTool } from "../_vendor/ailf-shared/index.d.ts";
 import type { ResolvedSourceConfig } from "../sources.js";
 import type { EvalMode } from "./types.js";
 /**
@@ -74,6 +74,21 @@ export interface RunContextInput {
     source: ResolvedSourceConfig;
     /** Specific task IDs evaluated (if scoped) */
     taskIds?: string[];
+    /**
+     * Literacy mode variant (`baseline | agentic | observed | full`). Only
+     * meaningful when `mode === "literacy"`; ignored for other modes. Lands
+     * on `RunContext.variant` and `ReportProvenance.variant` so consumers
+     * can disambiguate which literacy variant the run executed.
+     */
+    variant?: LiteracyVariant;
+    /**
+     * Model IDs the caller requested via `PipelineRequest.models`. When
+     * present, `RunContext.models` is filtered to this subset so the report's
+     * `provenance.models` reflects what was actually evaluated. Unknown IDs
+     * are silently filtered out — the upstream rejection path (W0281
+     * `filterModelsByRequest`) has already failed the run or warned.
+     */
+    requestedModelIds?: string[];
 }
 /**
  * Derive `RunContext` from pipeline inputs. The only construction path.

package/dist/pipeline/run-context.js

@@ -68,8 +68,18 @@ export function buildRunContext(input) {
     // config/models.ts model matrix — listing those models would be
     // misleading. Only include them for literacy mode where they're the
     // actual eval targets.
+    //
+    // When `PipelineRequest.models` pinned a subset, filter here too so
+    // `provenance.models` matches what actually ran (W0281). Without this
+    // the report would advertise the full cohort even though only the
+    // requested subset reached the LLMs.
+    const requestedSet = input.requestedModelIds?.length
+        ? new Set(input.requestedModelIds)
+        : undefined;
     const evaluatedModels = input.mode === "literacy"
-        ? models.models.map((m) => ({ id: m.id, label: m.label }))
+        ? models.models
+            .filter((m) => !requestedSet || requestedSet.has(m.id))
+            .map((m) => ({ id: m.id, label: m.label }))
         : [];
     return {
         areas: input.areas,
@@ -95,6 +105,7 @@ export function buildRunContext(input) {
         taskIds: input.taskIds,
         tool,
         trigger,
+        variant: input.mode === "literacy" ? input.variant : undefined,
     };
 }
 // ---------------------------------------------------------------------------

package/dist/pipeline/validate.d.ts

@@ -14,11 +14,15 @@ import type { ValidationIssue, ValidationResult } from "./types.js";
  */
 export declare function validateConfiguration(rootDir: string): ValidationResult;
 /**
- * Check that canonical context files exist. These are the per-task
- * gold-retrieval contexts actually referenced by task definitions.
+ * Check that the canonical-contexts directory exists.
  *
- * Contexts are generated by fetch-docs and may not exist yet —
- * returns warnings, not errors.
+ * Contexts are populated by fetch-docs, which scopes to the tasks
+ * actually being evaluated (not every task in the registry). Warning
+ * on individual missing files here would fire for every task the user
+ * didn't select — pure noise that previously crowded out real errors
+ * in the GHA safety-net's tail-of-log capture (W0282). The per-task
+ * precondition is enforced by `run-eval-step.ts:checkCanonicalContextsExist`
+ * against the filtered task set, where missing files are real errors.
  */
 export declare function validateContexts(rootDir: string): ValidationIssue[];
 /**

package/dist/pipeline/validate.js

@@ -34,11 +34,15 @@ export function validateConfiguration(rootDir) {
     return { issues, valid };
 }
 /**
- * Check that canonical context files exist. These are the per-task
- * gold-retrieval contexts actually referenced by task definitions.
+ * Check that the canonical-contexts directory exists.
  *
- * Contexts are generated by fetch-docs and may not exist yet —
- * returns warnings, not errors.
+ * Contexts are populated by fetch-docs, which scopes to the tasks
+ * actually being evaluated (not every task in the registry). Warning
+ * on individual missing files here would fire for every task the user
+ * didn't select — pure noise that previously crowded out real errors
+ * in the GHA safety-net's tail-of-log capture (W0282). The per-task
+ * precondition is enforced by `run-eval-step.ts:checkCanonicalContextsExist`
+ * against the filtered task set, where missing files are real errors.
  */
 export function validateContexts(rootDir) {
     const source = "validateContexts";
@@ -46,20 +50,6 @@ export function validateContexts(rootDir) {
     const canonicalDir = path.join(rootDir, "contexts", "canonical");
     if (!fs.existsSync(canonicalDir)) {
         issues.push(warning(source, "contexts/canonical/ directory not found — run 'pnpm fetch-docs' to generate", canonicalDir));
-        return issues;
-    }
-    const mappings = resolveMappings(rootDir);
-    for (const [, areaConfig] of Object.entries(mappings.feature_areas)) {
-        if (!areaConfig?.tasks)
-            continue;
-        for (const task of areaConfig.tasks) {
-            if (!task.id)
-                continue;
-            const contextFile = path.join(canonicalDir, `${task.id}.md`);
-            if (!fs.existsSync(contextFile)) {
-                issues.push(warning(source, `Missing canonical context for task '${task.id}' — run 'pnpm fetch-docs' to generate`, contextFile));
-            }
-        }
     }
     return issues;
 }

package/dist/report-store.d.ts

@@ -15,7 +15,7 @@
  * @see docs/design-docs/report-store/domain-model.md
  */
 import type { SanityClient } from "@sanity/client";
-import type { ArtifactRef, ArtifactType, SynthesisCostTelemetry } from "./_vendor/ailf-core/index.d.ts";
+import type { ArtifactRef, ArtifactType, LoadBaselineResult, SynthesisCostTelemetry } from "./_vendor/ailf-core/index.d.ts";
 import type { ComparisonReport, ISOTimestamp, LineageQuery, Report, ReportId, ReportProvenance, ScoreSummary } from "./pipeline/types.js";
 /**
  * Result of an auto-comparison, bundling the ComparisonReport with the
@@ -113,6 +113,19 @@ export declare class ReportStore {
      *   W0191 runtime schema gate. Sanity API failures still return null.
      */
     read(id: ReportId): Promise<null | Report>;
+    /**
+     * Load a previously-published report's score summary as a baseline
+     * for comparison. Returns a discriminated result so the caller can
+     * distinguish a genuine 404 (skip compare with a clear reason) from
+     * a transport failure (fail the step — the user pinned a baseline
+     * and deserves to know it didn't actually compare).
+     *
+     * The report's `summary` field is a `ReportSummary` — a superset of
+     * `ComparableSummary` — so the projection below carries everything
+     * the `compare()` primitive needs (`overall`, `perModel`, `scores`)
+     * without re-hydrating the slim prose/array fields.
+     */
+    loadBaselineFromReport(reportId: string): Promise<LoadBaselineResult>;
     /**
      * Write a report to the Sanity Content Lake.
      *