@sanity/ailf 7.0.1 → 7.1.0

package/dist/pipeline/compiler/provider-assembler.js

@@ -64,9 +64,16 @@ function applyReplaySwap(providers) {
  * Returns provider arrays keyed by literacy variant name (baseline,
  * agentic, observed). These are consumed by the YAML writer to produce
  * the per-variant promptfoo config files.
+ *
+ * `loaded` (optional) lets callers pre-load and pre-filter the
+ * `ModelsConfig` so a caller-side filter (e.g. W0281's
+ * `filterModelsByRequest`) actually takes effect on the assembled
+ * providers — building providers from the unfiltered set would silently
+ * defeat the filter, since promptfoo decides which LLMs to call from the
+ * providers array, not the returned `models` field.
  */
-export function loadModelsAndProviders(rootDir, source, searchMode, allowedOrigins) {
-    const models = loadModelsYaml(rootDir);
+export function loadModelsAndProviders(rootDir, source, searchMode, allowedOrigins, loaded) {
+    const models = loaded ?? loadModelsYaml(rootDir);
     return {
         models,
         providers: {
@@ -203,6 +210,12 @@ export function resolveMaxToolRounds(models, model, variant) {
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
-function loadModelsYaml(rootDir) {
+/**
+ * Load the `ModelsConfig` for `rootDir` from disk. Exported so callers
+ * that need to pre-filter the model set before provider assembly (e.g.
+ * `PipelineRequest.models`) can hand the filtered config back to
+ * `loadModelsAndProviders` via its optional `loaded` parameter.
+ */
+export function loadModelsYaml(rootDir) {
     return loadConfigFile("models", rootDir).data;
 }

package/dist/pipeline/failure-modes.d.ts

@@ -1,24 +1,34 @@
 /**
  * pipeline/failure-modes.ts
  *
- * Ceiling-cross-check failure-mode validator + report assembly.
+ * Ceiling-cross-check failure-mode validator + report assembly + keyword
+ * fallback classifier.
  *
  * The grader emits `failureMode` directly under the per-dimension taxonomy
- * (Plan 03-02 — `packages/eval/src/grader/`). This module consumes the
- * grader's emission as the source of truth and uses the surviving ceiling
- * decomposition (`classifyByCeiling`) only as a CONFIDENCE VALIDATOR — it
- * cross-checks the emitted mode against structural score signals and emits
- * a D0049 `Confidence` triple stamped with `derivation: "ceiling-cross-check"`.
+ * (Plan 03-02 — `packages/eval/src/grader/`) when its structured response
+ * is available to the pipeline. In practice (W0273 discovery), Promptfoo's
+ * `llm-rubric` post-processor extracts `score` + `reason` from the grader's
+ * JSON envelope and discards the rest of the structured surface — including
+ * `failureMode`. The wire-shape footer instructs the LLM correctly but the
+ * structured fields never reach `extractGraderJudgments`, so every emission
+ * arrives as the synthesized `failureMode: "unclassified"` placeholder.
  *
- * The legacy keyword-pattern classifier (and its five regex pattern
- * constants) was deleted in Plan 03-03 — its production coverage was ~1%
- * (Doc 03 §"Where classification lives"), and resurrecting it as a fallback
- * is explicitly out of scope.
+ * To restore the pre-2026-05-11 classification rate (15-23% → 0% → 15-23%),
+ * a keyword-pattern classifier is run as a FALLBACK when the grader's
+ * emitted mode is `"unclassified"` and the score is below the classification
+ * threshold. Plan 03-03 deleted this classifier in favor of grader-emission
+ * source-of-truth; W0273 reinstates it because the grader-emission path is
+ * blocked by Promptfoo's lossy `llm-rubric` parsing. The long-term fix
+ * (capturing the grader's full structured response) is tracked separately.
+ *
+ * `classifyByCeiling` continues to serve as the confidence cross-check.
  *
  * @see docs/decisions/D0005-grader-model-separation.md — single grader emits
  *      failureMode under the per-dimension taxonomy
  * @see docs/decisions/D0049-shared-confidence-contract.md — Confidence triple
  *      shape and `ceiling-cross-check` derivation tag
+ * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md — W0273 root
+ *      cause (Promptfoo strips structured fields)
  */
 import type { Confidence } from "../_vendor/ailf-core/index.d.ts";
 import type { FailureModeReport, FeatureScore, GraderJudgment } from "./types.js";

package/dist/pipeline/failure-modes.js

@@ -1,24 +1,34 @@
 /**
  * pipeline/failure-modes.ts
  *
- * Ceiling-cross-check failure-mode validator + report assembly.
+ * Ceiling-cross-check failure-mode validator + report assembly + keyword
+ * fallback classifier.
  *
  * The grader emits `failureMode` directly under the per-dimension taxonomy
- * (Plan 03-02 — `packages/eval/src/grader/`). This module consumes the
- * grader's emission as the source of truth and uses the surviving ceiling
- * decomposition (`classifyByCeiling`) only as a CONFIDENCE VALIDATOR — it
- * cross-checks the emitted mode against structural score signals and emits
- * a D0049 `Confidence` triple stamped with `derivation: "ceiling-cross-check"`.
+ * (Plan 03-02 — `packages/eval/src/grader/`) when its structured response
+ * is available to the pipeline. In practice (W0273 discovery), Promptfoo's
+ * `llm-rubric` post-processor extracts `score` + `reason` from the grader's
+ * JSON envelope and discards the rest of the structured surface — including
+ * `failureMode`. The wire-shape footer instructs the LLM correctly but the
+ * structured fields never reach `extractGraderJudgments`, so every emission
+ * arrives as the synthesized `failureMode: "unclassified"` placeholder.
  *
- * The legacy keyword-pattern classifier (and its five regex pattern
- * constants) was deleted in Plan 03-03 — its production coverage was ~1%
- * (Doc 03 §"Where classification lives"), and resurrecting it as a fallback
- * is explicitly out of scope.
+ * To restore the pre-2026-05-11 classification rate (15-23% → 0% → 15-23%),
+ * a keyword-pattern classifier is run as a FALLBACK when the grader's
+ * emitted mode is `"unclassified"` and the score is below the classification
+ * threshold. Plan 03-03 deleted this classifier in favor of grader-emission
+ * source-of-truth; W0273 reinstates it because the grader-emission path is
+ * blocked by Promptfoo's lossy `llm-rubric` parsing. The long-term fix
+ * (capturing the grader's full structured response) is tracked separately.
+ *
+ * `classifyByCeiling` continues to serve as the confidence cross-check.
  *
  * @see docs/decisions/D0005-grader-model-separation.md — single grader emits
  *      failureMode under the per-dimension taxonomy
  * @see docs/decisions/D0049-shared-confidence-contract.md — Confidence triple
  *      shape and `ceiling-cross-check` derivation tag
+ * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md — W0273 root
+ *      cause (Promptfoo strips structured fields)
  */
 import { LEGACY_FAILURE_MODES, detectFeatureArea } from "../_vendor/ailf-core/index.js";
 // ---------------------------------------------------------------------------
@@ -27,6 +37,20 @@ import { LEGACY_FAILURE_MODES, detectFeatureArea } from "../_vendor/ailf-core/in
 /** Only classify judgments with scores below this threshold */
 const CLASSIFICATION_THRESHOLD = 60;
 // ---------------------------------------------------------------------------
+// Keyword patterns (W0273 fallback)
+//
+// Verbatim from the pre-Plan-03-03 implementation. Used only when the
+// grader's emitted `failureMode` is `"unclassified"` — the grader's
+// emission still wins whenever it actually reaches the pipeline.
+// ---------------------------------------------------------------------------
+/** API error pattern — checked FIRST to prevent timeout errors containing
+ *  "deprecated" from being misclassified as outdated-docs. */
+const API_ERROR_PATTERN = /\[api-error\]|timeout|timed out|rate limit|429|503|ECONNRESET|ETIMEDOUT|socket hang up|fetch failed/i;
+const OUTDATED_PATTERN = /deprecated|old api|v[0-9]+ syntax|no longer supported|legacy|previous version|outdated|superseded|replaced by/i;
+const MISSING_PATTERN = /no documentation|not covered|had to guess|not found|missing.*doc|no.*information|undocumented|couldn't find|without.*documentation/i;
+const INCORRECT_PATTERN = /contradicts|incorrect.*doc|doc.*incorrect|wrong.*doc|doc.*wrong|documentation says.*but|factual error|inaccurate|misleading.*doc/i;
+const POOR_STRUCTURE_PATTERN = /unclear|ambiguous|couldn't determine|conflicting|confusing|hard to follow|poorly organized|scattered|fragmented/i;
+// ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
 /**
@@ -69,13 +93,25 @@ export function buildFailureModeReport(judgments, scores) {
         // grader's actual taxonomy choice rather than a collapsed
         // `"unclassified"` bucket.
         const emittedMode = readEmittedMode(judgment);
+        // W0273 fallback — when the grader's emitted mode is "unclassified"
+        // (the synthesized-unparsed-judgment placeholder; in practice this
+        // is every judgment today because Promptfoo's llm-rubric strips the
+        // grader's structured response), try keyword classification against
+        // the reason prose. Gated on score < CLASSIFICATION_THRESHOLD so
+        // passing judgments don't get spurious classifications.
+        const keywordFallback = emittedMode === "unclassified" &&
+            judgment.score < CLASSIFICATION_THRESHOLD
+            ? classifyByKeyword(judgment.reason)
+            : null;
         // Cross-check the grader's emission against ceiling decomposition.
         const stamp = validateFailureMode(judgment, ceilingScore, floorScore);
-        const classification = {
-            confidence: stamp.level,
-            mode: emittedMode,
-            source: "ceiling",
-        };
+        const classification = keywordFallback
+            ? keywordFallback
+            : {
+                confidence: stamp.level,
+                mode: emittedMode,
+                source: "ceiling",
+            };
         classifiedJudgments.push({ classification, judgment });
         summary[classification.mode] = (summary[classification.mode] ?? 0) + 1;
         // Per-area tracking
@@ -282,6 +318,39 @@ function readEmittedMode(judgment) {
     }
     return emitted;
 }
+/**
+ * Classify the failure mode of a low-scoring grader judgment by matching
+ * keyword patterns against the reason prose. Returns `null` when no
+ * pattern matches. Patterns checked in priority order (API errors first
+ * so timeout messages containing "deprecated" don't get misclassified
+ * as outdated-docs).
+ *
+ * W0273 — reinstated as a fallback when the grader's emitted failureMode
+ * is "unclassified". Plan 03-03 deleted this code in favor of grader-
+ * emission source-of-truth; the deletion is reversed here because
+ * Promptfoo's llm-rubric post-processor strips the grader's structured
+ * response (only score + reason survive into `comp.*`), so the
+ * grader-emission path produces 0% classification on every run.
+ */
+function classifyByKeyword(reason) {
+    const lower = reason.toLowerCase();
+    if (API_ERROR_PATTERN.test(lower)) {
+        return { confidence: "high", mode: "api-error", source: "keyword" };
+    }
+    if (OUTDATED_PATTERN.test(lower)) {
+        return { confidence: "high", mode: "outdated-docs", source: "keyword" };
+    }
+    if (MISSING_PATTERN.test(lower)) {
+        return { confidence: "high", mode: "missing-docs", source: "keyword" };
+    }
+    if (INCORRECT_PATTERN.test(lower)) {
+        return { confidence: "medium", mode: "incorrect-docs", source: "keyword" };
+    }
+    if (POOR_STRUCTURE_PATTERN.test(lower)) {
+        return { confidence: "medium", mode: "poor-structure", source: "keyword" };
+    }
+    return null;
+}
 /**
  * Classify by ceiling-decomposition structural signals — preserved
  * verbatim from the pre-Plan-03-03 implementation. The function itself

package/dist/pipeline/map-request-to-config.js

@@ -37,6 +37,7 @@ export function mapRequestToConfig(request, rootDir) {
         mode,
         variant,
         debug: mapDebug(request.debug),
+        models: request.models,
         areas: request.areas,
         tasks: request.tasks,
         changedDocs: request.changedDocs,
@@ -46,6 +47,7 @@ export function mapRequestToConfig(request, rootDir) {
         compareEnabled: request.compare ?? false,
         compareThreshold: request.compareThreshold,
         compareBaseline: request.compareBaseline,
+        compareBaselineReportId: request.compareBaselineReportId,
         gapAnalysisEnabled: request.gapAnalysis ?? true,
         publishEnabled: request.publish ?? publishDefault,
         publishTag: request.publishTag,

package/dist/pipeline/normalize-mode.d.ts

@@ -35,7 +35,7 @@ export type LiteracyVariantName = (typeof LiteracyVariant)[keyof typeof Literacy
 export type LiteracyEvalSubMode = typeof LiteracyVariant.STANDARD | typeof LiteracyVariant.AGENTIC;
 export interface NormalizedMode {
     mode: EvalMode;
-    variant?: string;
+    variant?: LiteracyVariantName;
 }
 /**
  * Normalize a raw CLI mode string to a canonical mode + optional variant.

package/dist/pipeline/normalize-mode.js

@@ -55,6 +55,8 @@ const ALL_ACCEPTED = [
 export function normalizeMode(input) {
     if (LEGACY_LITERACY_VARIANTS.has(input)) {
         console.warn(`⚠ Deprecated: --mode ${input} is a legacy alias. Use --mode literacy --variant ${input} instead.`);
+        // The membership check above narrows `input` to LITERACY_VARIANTS — the
+        // cast is to the closed type, not a widening.
         return { mode: "literacy", variant: input };
     }
     if (CANONICAL_MODES.has(input)) {

package/dist/pipeline/run-context.d.ts

@@ -13,7 +13,7 @@
  * @see docs/decisions/D0032-run-anchored-artifact-store.md (§ Move 5 — Drift Prevention)
  */
 import { type Logger, type RunContext } from "../_vendor/ailf-core/index.d.ts";
-import { type RunClassification, type RunExecutor, type RunExecutorSurface, type RunHost, type RunLineage, type RunOwner, type RunTool } from "../_vendor/ailf-shared/index.d.ts";
+import { type LiteracyVariant, type RunClassification, type RunExecutor, type RunExecutorSurface, type RunHost, type RunLineage, type RunOwner, type RunTool } from "../_vendor/ailf-shared/index.d.ts";
 import type { ResolvedSourceConfig } from "../sources.js";
 import type { EvalMode } from "./types.js";
 /**
@@ -74,6 +74,21 @@ export interface RunContextInput {
     source: ResolvedSourceConfig;
     /** Specific task IDs evaluated (if scoped) */
     taskIds?: string[];
+    /**
+     * Literacy mode variant (`baseline | agentic | observed | full`). Only
+     * meaningful when `mode === "literacy"`; ignored for other modes. Lands
+     * on `RunContext.variant` and `ReportProvenance.variant` so consumers
+     * can disambiguate which literacy variant the run executed.
+     */
+    variant?: LiteracyVariant;
+    /**
+     * Model IDs the caller requested via `PipelineRequest.models`. When
+     * present, `RunContext.models` is filtered to this subset so the report's
+     * `provenance.models` reflects what was actually evaluated. Unknown IDs
+     * are silently filtered out — the upstream rejection path (W0281
+     * `filterModelsByRequest`) has already failed the run or warned.
+     */
+    requestedModelIds?: string[];
 }
 /**
  * Derive `RunContext` from pipeline inputs. The only construction path.

package/dist/pipeline/run-context.js

@@ -68,8 +68,18 @@ export function buildRunContext(input) {
     // config/models.ts model matrix — listing those models would be
     // misleading. Only include them for literacy mode where they're the
     // actual eval targets.
+    //
+    // When `PipelineRequest.models` pinned a subset, filter here too so
+    // `provenance.models` matches what actually ran (W0281). Without this
+    // the report would advertise the full cohort even though only the
+    // requested subset reached the LLMs.
+    const requestedSet = input.requestedModelIds?.length
+        ? new Set(input.requestedModelIds)
+        : undefined;
     const evaluatedModels = input.mode === "literacy"
-        ? models.models.map((m) => ({ id: m.id, label: m.label }))
+        ? models.models
+            .filter((m) => !requestedSet || requestedSet.has(m.id))
+            .map((m) => ({ id: m.id, label: m.label }))
         : [];
     return {
         areas: input.areas,
@@ -95,6 +105,7 @@ export function buildRunContext(input) {
         taskIds: input.taskIds,
         tool,
         trigger,
+        variant: input.mode === "literacy" ? input.variant : undefined,
     };
 }
 // ---------------------------------------------------------------------------

package/dist/pipeline/validate.d.ts

@@ -14,11 +14,15 @@ import type { ValidationIssue, ValidationResult } from "./types.js";
  */
 export declare function validateConfiguration(rootDir: string): ValidationResult;
 /**
- * Check that canonical context files exist. These are the per-task
- * gold-retrieval contexts actually referenced by task definitions.
+ * Check that the canonical-contexts directory exists.
  *
- * Contexts are generated by fetch-docs and may not exist yet —
- * returns warnings, not errors.
+ * Contexts are populated by fetch-docs, which scopes to the tasks
+ * actually being evaluated (not every task in the registry). Warning
+ * on individual missing files here would fire for every task the user
+ * didn't select — pure noise that previously crowded out real errors
+ * in the GHA safety-net's tail-of-log capture (W0282). The per-task
+ * precondition is enforced by `run-eval-step.ts:checkCanonicalContextsExist`
+ * against the filtered task set, where missing files are real errors.
  */
 export declare function validateContexts(rootDir: string): ValidationIssue[];
 /**

package/dist/pipeline/validate.js

@@ -34,11 +34,15 @@ export function validateConfiguration(rootDir) {
     return { issues, valid };
 }
 /**
- * Check that canonical context files exist. These are the per-task
- * gold-retrieval contexts actually referenced by task definitions.
+ * Check that the canonical-contexts directory exists.
  *
- * Contexts are generated by fetch-docs and may not exist yet —
- * returns warnings, not errors.
+ * Contexts are populated by fetch-docs, which scopes to the tasks
+ * actually being evaluated (not every task in the registry). Warning
+ * on individual missing files here would fire for every task the user
+ * didn't select — pure noise that previously crowded out real errors
+ * in the GHA safety-net's tail-of-log capture (W0282). The per-task
+ * precondition is enforced by `run-eval-step.ts:checkCanonicalContextsExist`
+ * against the filtered task set, where missing files are real errors.
  */
 export function validateContexts(rootDir) {
     const source = "validateContexts";
@@ -46,20 +50,6 @@ export function validateContexts(rootDir) {
     const canonicalDir = path.join(rootDir, "contexts", "canonical");
     if (!fs.existsSync(canonicalDir)) {
         issues.push(warning(source, "contexts/canonical/ directory not found — run 'pnpm fetch-docs' to generate", canonicalDir));
-        return issues;
-    }
-    const mappings = resolveMappings(rootDir);
-    for (const [, areaConfig] of Object.entries(mappings.feature_areas)) {
-        if (!areaConfig?.tasks)
-            continue;
-        for (const task of areaConfig.tasks) {
-            if (!task.id)
-                continue;
-            const contextFile = path.join(canonicalDir, `${task.id}.md`);
-            if (!fs.existsSync(contextFile)) {
-                issues.push(warning(source, `Missing canonical context for task '${task.id}' — run 'pnpm fetch-docs' to generate`, contextFile));
-            }
-        }
     }
     return issues;
 }

package/dist/report-store.d.ts

@@ -15,7 +15,7 @@
  * @see docs/design-docs/report-store/domain-model.md
  */
 import type { SanityClient } from "@sanity/client";
-import type { ArtifactRef, ArtifactType, SynthesisCostTelemetry } from "./_vendor/ailf-core/index.d.ts";
+import type { ArtifactRef, ArtifactType, LoadBaselineResult, SynthesisCostTelemetry } from "./_vendor/ailf-core/index.d.ts";
 import type { ComparisonReport, ISOTimestamp, LineageQuery, Report, ReportId, ReportProvenance, ScoreSummary } from "./pipeline/types.js";
 /**
  * Result of an auto-comparison, bundling the ComparisonReport with the
@@ -113,6 +113,19 @@ export declare class ReportStore {
      *   W0191 runtime schema gate. Sanity API failures still return null.
      */
     read(id: ReportId): Promise<null | Report>;
+    /**
+     * Load a previously-published report's score summary as a baseline
+     * for comparison. Returns a discriminated result so the caller can
+     * distinguish a genuine 404 (skip compare with a clear reason) from
+     * a transport failure (fail the step — the user pinned a baseline
+     * and deserves to know it didn't actually compare).
+     *
+     * The report's `summary` field is a `ReportSummary` — a superset of
+     * `ComparableSummary` — so the projection below carries everything
+     * the `compare()` primitive needs (`overall`, `perModel`, `scores`)
+     * without re-hydrating the slim prose/array fields.
+     */
+    loadBaselineFromReport(reportId: string): Promise<LoadBaselineResult>;
     /**
      * Write a report to the Sanity Content Lake.
      *

package/dist/report-store.js

@@ -270,6 +270,38 @@ export class ReportStore {
             return null;
         }
     }
+    /**
+     * Load a previously-published report's score summary as a baseline
+     * for comparison. Returns a discriminated result so the caller can
+     * distinguish a genuine 404 (skip compare with a clear reason) from
+     * a transport failure (fail the step — the user pinned a baseline
+     * and deserves to know it didn't actually compare).
+     *
+     * The report's `summary` field is a `ReportSummary` — a superset of
+     * `ComparableSummary` — so the projection below carries everything
+     * the `compare()` primitive needs (`overall`, `perModel`, `scores`)
+     * without re-hydrating the slim prose/array fields.
+     */
+    async loadBaselineFromReport(reportId) {
+        try {
+            const doc = await this.client.fetch(`*[_type == $type && reportId == $id][0]{ summary }`, { id: reportId, type: REPORT_TYPE });
+            const summary = doc?.summary;
+            if (!summary)
+                return { kind: "not_found" };
+            return {
+                kind: "ok",
+                baseline: {
+                    overall: summary.overall,
+                    perModel: summary.perModel,
+                    scores: summary.scores,
+                },
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return { kind: "error", message };
+        }
+    }
     /**
      * Write a report to the Sanity Content Lake.
      *

package/dist/sanity/client.js

@@ -108,8 +108,8 @@ export function getSanityClient(overrides, source) {
  * fall back to `SANITY_DATASET` so existing CI workflows that pin a
  * test/staging dataset (e.g. Tier 2 with `SANITY_DATASET=ailf-test`)
  * continue to work without a new env var. The hard-coded fallback is
- * the editorial dataset name during the D0043 cutover window — the flip
- * to `ailf-prod-private` happens after the migration script runs.
+ * `AILF_DATASET_DEFAULT` (`ailf-prod-private`, D0043) — only reached for
+ * ad-hoc runs with no env at all.
  *
  * Token resolution prefers the AILF-scoped token, falling back to
  * the shared `SANITY_API_TOKEN`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "7.0.1",
+  "version": "7.1.0",
   "private": false,
   "publishConfig": {
     "access": "public"