@sanity/ailf 3.8.1 → 3.9.0

package/config/canary-tasks.ts

@@ -0,0 +1,64 @@
+/**
+ * canary-tasks.ts — The Tier 3 canary set.
+ *
+ * Five tasks the Tier 3 nightly workflow runs against live LLMs every day.
+ * Composition follows the design doc's "weighted toward modes/areas with
+ * the most production usage and the highest historical regression rates"
+ * recommendation: GROQ and Content Lake (foundational consumer surfaces),
+ * Portable Text (historically drift-prone), Studio schema authoring (the
+ * second-most-used surface after queries), and a knowledge-probe pairing
+ * for cross-mode coverage.
+ *
+ * Each entry's `rationale` is the canary's load-bearing field — without it,
+ * future maintainers can't reason about whether a regression is meaningful
+ * or whether the slot has lost value. Update the rationale when you swap a
+ * canary entry; never silently replace one.
+ *
+ * Validated against the live task inventory by `scripts/check-canary-tasks.ts`
+ * (`pnpm check`). Dangling task IDs fail the build.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see .github/workflows/tier-3-nightly.yml — consumer
+ */
+
+import { defineCanaryTasks } from "@sanity/ailf-core"
+
+export default defineCanaryTasks({
+  tasks: [
+    {
+      taskId: "groq-blog-queries",
+      mode: "literacy",
+      rationale:
+        "Canonical first-use path for Sanity's most-used API. GROQ is the largest doc surface and the highest-leverage canary slot — drift here means drift in the most-consumed documentation. Filtering and pagination together exercise the largest cross-section of GROQ syntax in a single task.",
+    },
+    {
+      taskId: "content-lake-mutations",
+      mode: "literacy",
+      rationale:
+        "Foundational client API. CRUD is structurally distinct from query reasoning, so this catches regressions in mutation/transaction documentation that GROQ canary slots cannot reach. Every Sanity consumer eventually writes to the Content Lake.",
+    },
+    {
+      taskId: "portable-text-rendering",
+      mode: "literacy",
+      rationale:
+        "Major doc surface flagged as historically drift-prone in the testing audit. React-rendering of Portable Text mixes documentation, type definitions, and worked examples — a regression on any axis surfaces here first.",
+    },
+    {
+      taskId: "studio-blog-schema",
+      mode: "literacy",
+      rationale:
+        "Schema authoring (`defineType` / `defineField`) is the second-most-used surface after queries. Tests structural Studio docs that change shape across versions; pairs naturally with the GROQ canary because consumers typically author schemas before querying them.",
+    },
+    {
+      taskId: "kp-groq-projections",
+      mode: "knowledge-probe",
+      rationale:
+        "Cross-mode coverage. Pairs with `groq-blog-queries` (literacy) so we catch GROQ drift in both implementation (write code) and recall (explain syntax) modes. Knowledge-probe is the only non-literacy mode in the canary today; expand once mcp-server tasks land in the repo.",
+    },
+    // mcp-server canary slot — add a third mode here when a committed
+    // mcp-server task lands under packages/eval/tasks/mcp-server/. Today
+    // there are no production mcp-server tasks (only fixtures); the trigger
+    // is upstream and adding a placeholder slot would dangle. Surfaced at
+    // Phase 5 close (2026-04-27) — see W0116 retrospective.
+  ],
+})

package/config/test-budgets.ts

@@ -0,0 +1,24 @@
+/**
+ * test-budgets.ts — Per-provider daily USD spend caps for Tier 3 CI runs.
+ *
+ * Each cap is the maximum cost a single Tier 3 nightly run may incur for
+ * that provider. The Tier 3 workflow (`.github/workflows/tier-3-nightly.yml`)
+ * fails loudly if any provider's actual spend exceeds its cap.
+ *
+ * The design doc names a $30–60/day envelope across all providers. Caps
+ * here divide that envelope per-provider; tighten as baseline canary spend
+ * becomes measurable.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see scripts/tier-3-budget-check.mjs — enforcement
+ */
+
+import { defineTestBudgets } from "@sanity/ailf-core"
+
+export default defineTestBudgets({
+  perProviderDaily: {
+    anthropic: 30,
+    openai: 30,
+  },
+  warnFraction: 0.8,
+})

package/dist/_vendor/ailf-core/config-helpers.d.ts

@@ -26,9 +26,11 @@
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md (canonical)
  */
 import type { EvalConfig } from "./schemas/eval-config.js";
+import type { CanaryTaskSetConfig } from "./schemas/canary-tasks.js";
 import type { FeatureRegistry, RubricConfig, ThresholdConfig } from "./schemas/pipeline.js";
 import type { SchedulesFile } from "./schemas/schedules.js";
 import type { SinksFile } from "./schemas/sinks.js";
+import type { TestBudgetConfig } from "./schemas/test-budgets.js";
 import type { ModelsConfig } from "./types/index.js";
 import type { GeneralizedTaskDefinition } from "./types/generalized-task.js";
 import type { ModeBase, PresetDefinition } from "./types/plugin-registry.js";
@@ -89,6 +91,23 @@ export declare function defineSinks(sinks: SinksFile): SinksFile;
  * Used in `config/schedules.ts` for typed schedule configuration.
  */
 export declare function defineSchedules(schedules: SchedulesFile): SchedulesFile;
+/**
+ * Define per-provider daily USD spend caps for Tier 3 (live-LLM) CI runs.
+ *
+ * Used in `config/test-budgets.ts` for typed budget configuration.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export declare function defineTestBudgets(budgets: TestBudgetConfig): TestBudgetConfig;
+/**
+ * Define the curated canary task set for the Tier 3 nightly workflow.
+ *
+ * Used in `config/canary-tasks.ts`. Validation against the live task
+ * inventory happens in `scripts/check-canary-tasks.ts`, not here.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export declare function defineCanaryTasks(canary: CanaryTaskSetConfig): CanaryTaskSetConfig;
 /**
  * Source configuration — typed inline until a dedicated schema exists.
  *

package/dist/_vendor/ailf-core/config-helpers.js

@@ -145,6 +145,33 @@ export function defineSinks(sinks) {
 export function defineSchedules(schedules) {
     return schedules;
 }
+// ---------------------------------------------------------------------------
+// Test-budget helpers
+// ---------------------------------------------------------------------------
+/**
+ * Define per-provider daily USD spend caps for Tier 3 (live-LLM) CI runs.
+ *
+ * Used in `config/test-budgets.ts` for typed budget configuration.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export function defineTestBudgets(budgets) {
+    return budgets;
+}
+// ---------------------------------------------------------------------------
+// Canary task-set helpers
+// ---------------------------------------------------------------------------
+/**
+ * Define the curated canary task set for the Tier 3 nightly workflow.
+ *
+ * Used in `config/canary-tasks.ts`. Validation against the live task
+ * inventory happens in `scripts/check-canary-tasks.ts`, not here.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export function defineCanaryTasks(canary) {
+    return canary;
+}
 /**
  * Define documentation source configurations.
  *

package/dist/_vendor/ailf-core/index.d.ts

@@ -17,7 +17,7 @@ export * from "./services/index.js";
 export * from "./examples/index.js";
 export * from "./artifact-registry.js";
 export * from "./batch-signing.js";
-export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
+export { defineCanaryTasks, defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineTestBudgets, defineThresholds, } from "./config-helpers.js";
 export type { PricingEntry, PromptEntry, SourceEntry, } from "./config-helpers.js";
 export { env } from "./env-helper.js";
 export { NoOpArtifactWriter, NotImplementedError, } from "./ports/artifact-writer.js";

package/dist/_vendor/ailf-core/index.js

@@ -20,7 +20,7 @@ export * from "./batch-signing.js";
 // ---------------------------------------------------------------------------
 // Architecture overhaul — Phase 0 helpers
 // ---------------------------------------------------------------------------
-export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
+export { defineCanaryTasks, defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineTestBudgets, defineThresholds, } from "./config-helpers.js";
 export { env } from "./env-helper.js";
 export { NoOpArtifactWriter, NotImplementedError, } from "./ports/artifact-writer.js";
 export { assoc, resolveVariantMode, splitTaskVariant, } from "./artifact-capture/association.js";

package/dist/_vendor/ailf-core/schemas/canary-tasks.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @sanity/ailf-core — Canary task-set schemas.
+ *
+ * The canary task set is the curated subset of evaluation tasks the Tier 3
+ * nightly workflow runs against live LLMs. Each entry pins a `taskId` and
+ * `mode` together with a one-paragraph rationale documenting why the task
+ * earned a slot — the rationale is the canary set's single most important
+ * field; without it, future maintainers can't reason about whether a
+ * regression is meaningful or whether the slot has lost value.
+ *
+ * Validation that canary IDs map to real tasks lives in
+ * `scripts/check-canary-tasks.ts` (run by `pnpm check`); it can't live in
+ * Zod because the inventory comes from the repo task glob, not the schema.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/canary-tasks.ts — authored config
+ */
+import { z } from "zod";
+export declare const CanaryTaskEntrySchema: z.ZodObject<{
+    taskId: z.ZodString;
+    mode: z.ZodEnum<{
+        custom: "custom";
+        agentic: "agentic";
+        literacy: "literacy";
+        "mcp-server": "mcp-server";
+        "agent-harness": "agent-harness";
+        "knowledge-probe": "knowledge-probe";
+        baseline: "baseline";
+        observed: "observed";
+        full: "full";
+    }>;
+    rationale: z.ZodString;
+}, z.core.$strip>;
+export type CanaryTaskEntry = z.infer<typeof CanaryTaskEntrySchema>;
+export declare const CanaryTaskSetConfigSchema: z.ZodObject<{
+    tasks: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        mode: z.ZodEnum<{
+            custom: "custom";
+            agentic: "agentic";
+            literacy: "literacy";
+            "mcp-server": "mcp-server";
+            "agent-harness": "agent-harness";
+            "knowledge-probe": "knowledge-probe";
+            baseline: "baseline";
+            observed: "observed";
+            full: "full";
+        }>;
+        rationale: z.ZodString;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type CanaryTaskSetConfig = z.infer<typeof CanaryTaskSetConfigSchema>;

package/dist/_vendor/ailf-core/schemas/canary-tasks.js

@@ -0,0 +1,46 @@
+/**
+ * @sanity/ailf-core — Canary task-set schemas.
+ *
+ * The canary task set is the curated subset of evaluation tasks the Tier 3
+ * nightly workflow runs against live LLMs. Each entry pins a `taskId` and
+ * `mode` together with a one-paragraph rationale documenting why the task
+ * earned a slot — the rationale is the canary set's single most important
+ * field; without it, future maintainers can't reason about whether a
+ * regression is meaningful or whether the slot has lost value.
+ *
+ * Validation that canary IDs map to real tasks lives in
+ * `scripts/check-canary-tasks.ts` (run by `pnpm check`); it can't live in
+ * Zod because the inventory comes from the repo task glob, not the schema.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/canary-tasks.ts — authored config
+ */
+import { z } from "zod";
+import { RAW_EVAL_MODES } from "../../ailf-shared/index.js";
+// ---------------------------------------------------------------------------
+// Canary entry
+// ---------------------------------------------------------------------------
+export const CanaryTaskEntrySchema = z.object({
+    taskId: z.string().min(1, "canary entries must have a non-empty taskId"),
+    mode: z.enum(RAW_EVAL_MODES),
+    rationale: z
+        .string()
+        .min(40, "canary rationale must be at least one informative sentence"),
+});
+// ---------------------------------------------------------------------------
+// Canary set config
+// ---------------------------------------------------------------------------
+export const CanaryTaskSetConfigSchema = z
+    .object({
+    tasks: z.array(CanaryTaskEntrySchema),
+})
+    .refine((config) => {
+    const seen = new Set();
+    for (const entry of config.tasks) {
+        const key = `${entry.mode}:${entry.taskId}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+    }
+    return true;
+}, { message: "duplicate (mode, taskId) entries in canary set" });

package/dist/_vendor/ailf-core/schemas/index.d.ts

@@ -9,8 +9,10 @@
  * (Phase 0d). Original files are now re-export barrels.
  */
 export * from "./callback-payload.js";
+export * from "./canary-tasks.js";
 export * from "./eval-config.js";
 export * from "./pipeline-request.js";
 export * from "./pipeline.js";
 export * from "./schedules.js";
 export * from "./sinks.js";
+export * from "./test-budgets.js";

package/dist/_vendor/ailf-core/schemas/index.js

@@ -9,8 +9,10 @@
  * (Phase 0d). Original files are now re-export barrels.
  */
 export * from "./callback-payload.js";
+export * from "./canary-tasks.js";
 export * from "./eval-config.js";
 export * from "./pipeline-request.js";
 export * from "./pipeline.js";
 export * from "./schedules.js";
 export * from "./sinks.js";
+export * from "./test-budgets.js";

package/dist/_vendor/ailf-core/schemas/test-budgets.d.ts

@@ -0,0 +1,19 @@
+/**
+ * @sanity/ailf-core — Test-budget schemas.
+ *
+ * Per-provider daily USD spend caps for Tier 3 (live-LLM) CI workflows.
+ * The cap is the maximum cost a single Tier 3 nightly run may incur for
+ * a given provider; the workflow fails loudly if any provider's actual
+ * spend exceeds its cap.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/test-budgets.ts — authored config
+ */
+import { z } from "zod";
+export declare const ProviderBudgetCapsSchema: z.ZodRecord<z.ZodString, z.ZodNumber>;
+export type ProviderBudgetCaps = z.infer<typeof ProviderBudgetCapsSchema>;
+export declare const TestBudgetConfigSchema: z.ZodObject<{
+    perProviderDaily: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    warnFraction: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+export type TestBudgetConfig = z.infer<typeof TestBudgetConfigSchema>;

package/dist/_vendor/ailf-core/schemas/test-budgets.js

@@ -0,0 +1,34 @@
+/**
+ * @sanity/ailf-core — Test-budget schemas.
+ *
+ * Per-provider daily USD spend caps for Tier 3 (live-LLM) CI workflows.
+ * The cap is the maximum cost a single Tier 3 nightly run may incur for
+ * a given provider; the workflow fails loudly if any provider's actual
+ * spend exceeds its cap.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/test-budgets.ts — authored config
+ */
+import { z } from "zod";
+// ---------------------------------------------------------------------------
+// Provider cap entry
+// ---------------------------------------------------------------------------
+const NonNegativeUsd = z
+    .number()
+    .nonnegative("budget caps must be non-negative USD amounts");
+export const ProviderBudgetCapsSchema = z
+    .record(z.string().min(1), NonNegativeUsd)
+    .refine((caps) => Object.keys(caps).length > 0, {
+    message: "perProviderDaily must declare at least one provider",
+});
+// ---------------------------------------------------------------------------
+// Test budget config
+// ---------------------------------------------------------------------------
+export const TestBudgetConfigSchema = z.object({
+    perProviderDaily: ProviderBudgetCapsSchema,
+    warnFraction: z
+        .number()
+        .gt(0)
+        .lte(1, "warnFraction must be in (0, 1]")
+        .default(0.8),
+});

package/dist/_vendor/ailf-shared/canary-drift.d.ts

@@ -0,0 +1,84 @@
+/**
+ * canary/drift.ts — Pure drift-statistic computation for the Tier 3
+ * framework-tests-framework loop.
+ *
+ * Consumes the projection shape returned by Studio's `latestReportsQuery`
+ * (we accept a slim subset so the function stays a pure-domain dependency
+ * with no Studio-package import). Computes per-area Δscore between the
+ * most-recent canary run and the trailing-N median, plus an overall
+ * Δscore for the run as a whole. Output classifies each delta as `ok`,
+ * `warn`, or `regression` against caller-provided thresholds.
+ *
+ * The function is total — it never throws. Edge cases (empty trailing
+ * window, missing scores) surface as `verdict: "no-baseline"` so the
+ * caller can decide whether to treat the missing baseline as a fail.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/studio/src/queries.ts — `latestReportsQuery`
+ */
+/** Slim projection of a canary run report — subset of `ReportListItem`. */
+export interface CanaryReportSlim {
+    reportId: string;
+    completedAt: string;
+    overall: number;
+    scores: {
+        feature: string;
+        totalScore: number;
+    }[];
+}
+/** Verdict for a single Δ computation. */
+export type DriftVerdict = "ok" | "warn" | "regression" | "no-baseline";
+/** Δ between the most-recent run and the trailing-N median. */
+export interface DriftEntry {
+    /** "overall" for the run-level avg, or the area slug for a per-area Δ. */
+    feature: string;
+    /** Score in the most-recent run. */
+    current: number;
+    /** Median of trailing-N runs (excluding the most-recent). Null when no baseline. */
+    trailingMedian: number | null;
+    /** current − trailingMedian. Null when no baseline. */
+    delta: number | null;
+    verdict: DriftVerdict;
+}
+/** Tunable thresholds — caller decides what counts as warn vs regression. */
+export interface DriftThresholds {
+    /**
+     * How many prior runs (excluding the most-recent) form the trailing
+     * baseline. Sensible defaults sit between 5 and 10 for a daily canary.
+     */
+    trailingN: number;
+    /** Drop ≥ this magnitude (and < failDelta) → `warn`. */
+    warnDelta: number;
+    /** Drop ≥ this magnitude → `regression`. */
+    failDelta: number;
+    /**
+     * Minimum trailing-window size required to compute a delta. When the
+     * window is smaller, the entry's verdict is `no-baseline`. Defaults to
+     * 1 — a single prior run is enough to detect *some* movement.
+     */
+    minBaselineRuns?: number;
+}
+/** Aggregate result of `computeCanaryDrift`. */
+export interface CanaryDriftReport {
+    /** ID + timestamp of the most-recent run. */
+    reportId: string;
+    completedAt: string;
+    /** Run-level Δscore (overall avg). */
+    overall: DriftEntry;
+    /** Per-area Δscores. One entry per area present in the most-recent run. */
+    byArea: DriftEntry[];
+    /** True when any verdict is `regression`. */
+    hasRegression: boolean;
+    /** True when any verdict is `warn` or `regression`. */
+    hasMovement: boolean;
+}
+/**
+ * Compute per-area + overall drift for a sequence of canary runs.
+ *
+ * `reports` must be ordered **newest-first** (matching `latestReportsQuery`'s
+ * `order(completedAt desc)`). The most-recent run is `reports[0]`; the
+ * trailing window is `reports.slice(1, 1 + trailingN)`.
+ *
+ * @throws never — all error states surface as `no-baseline` verdicts.
+ */
+export declare function computeCanaryDrift(reports: CanaryReportSlim[], thresholds: DriftThresholds): CanaryDriftReport | null;

package/dist/_vendor/ailf-shared/canary-drift.js

@@ -0,0 +1,86 @@
+/**
+ * canary/drift.ts — Pure drift-statistic computation for the Tier 3
+ * framework-tests-framework loop.
+ *
+ * Consumes the projection shape returned by Studio's `latestReportsQuery`
+ * (we accept a slim subset so the function stays a pure-domain dependency
+ * with no Studio-package import). Computes per-area Δscore between the
+ * most-recent canary run and the trailing-N median, plus an overall
+ * Δscore for the run as a whole. Output classifies each delta as `ok`,
+ * `warn`, or `regression` against caller-provided thresholds.
+ *
+ * The function is total — it never throws. Edge cases (empty trailing
+ * window, missing scores) surface as `verdict: "no-baseline"` so the
+ * caller can decide whether to treat the missing baseline as a fail.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/studio/src/queries.ts — `latestReportsQuery`
+ */
+/**
+ * Compute per-area + overall drift for a sequence of canary runs.
+ *
+ * `reports` must be ordered **newest-first** (matching `latestReportsQuery`'s
+ * `order(completedAt desc)`). The most-recent run is `reports[0]`; the
+ * trailing window is `reports.slice(1, 1 + trailingN)`.
+ *
+ * @throws never — all error states surface as `no-baseline` verdicts.
+ */
+export function computeCanaryDrift(reports, thresholds) {
+    if (reports.length === 0)
+        return null;
+    const minBaseline = thresholds.minBaselineRuns ?? 1;
+    const current = reports[0];
+    const trailing = reports.slice(1, 1 + thresholds.trailingN);
+    const overall = scoreDrift("overall", current.overall, trailing.map((r) => r.overall), thresholds, minBaseline);
+    const byArea = [];
+    for (const score of current.scores) {
+        const trailingArea = [];
+        for (const t of trailing) {
+            const match = t.scores.find((s) => s.feature === score.feature);
+            if (match)
+                trailingArea.push(match.totalScore);
+        }
+        byArea.push(scoreDrift(score.feature, score.totalScore, trailingArea, thresholds, minBaseline));
+    }
+    const hasRegression = overall.verdict === "regression" ||
+        byArea.some((e) => e.verdict === "regression");
+    const hasMovement = hasRegression ||
+        overall.verdict === "warn" ||
+        byArea.some((e) => e.verdict === "warn");
+    return {
+        reportId: current.reportId,
+        completedAt: current.completedAt,
+        overall,
+        byArea,
+        hasRegression,
+        hasMovement,
+    };
+}
+function scoreDrift(feature, current, trailing, thresholds, minBaseline) {
+    if (trailing.length < minBaseline) {
+        return {
+            feature,
+            current,
+            trailingMedian: null,
+            delta: null,
+            verdict: "no-baseline",
+        };
+    }
+    const trailingMedian = median(trailing);
+    const delta = current - trailingMedian;
+    const drop = -delta;
+    let verdict = "ok";
+    if (drop >= thresholds.failDelta)
+        verdict = "regression";
+    else if (drop >= thresholds.warnDelta)
+        verdict = "warn";
+    return { feature, current, trailingMedian, delta, verdict };
+}
+function median(values) {
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0) {
+        return (sorted[mid - 1] + sorted[mid]) / 2;
+    }
+    return sorted[mid];
+}

package/dist/_vendor/ailf-shared/index.d.ts

@@ -9,6 +9,7 @@
  * from @sanity/ailf-core, @sanity/ailf, or
  * @sanity/ailf-studio. It is the leaf of the dependency graph.
  */
+export * from "./canary-drift.js";
 export * from "./document-ref.js";
 export * from "./feature-flags.js";
 export * from "./score-grades.js";

package/dist/_vendor/ailf-shared/index.js

@@ -9,6 +9,7 @@
  * from @sanity/ailf-core, @sanity/ailf, or
  * @sanity/ailf-studio. It is the leaf of the dependency graph.
  */
+export * from "./canary-drift.js";
 export * from "./document-ref.js";
 export * from "./feature-flags.js";
 export * from "./score-grades.js";

package/dist/config/canary-tasks.ts

@@ -0,0 +1,64 @@
+/**
+ * canary-tasks.ts — The Tier 3 canary set.
+ *
+ * Five tasks the Tier 3 nightly workflow runs against live LLMs every day.
+ * Composition follows the design doc's "weighted toward modes/areas with
+ * the most production usage and the highest historical regression rates"
+ * recommendation: GROQ and Content Lake (foundational consumer surfaces),
+ * Portable Text (historically drift-prone), Studio schema authoring (the
+ * second-most-used surface after queries), and a knowledge-probe pairing
+ * for cross-mode coverage.
+ *
+ * Each entry's `rationale` is the canary's load-bearing field — without it,
+ * future maintainers can't reason about whether a regression is meaningful
+ * or whether the slot has lost value. Update the rationale when you swap a
+ * canary entry; never silently replace one.
+ *
+ * Validated against the live task inventory by `scripts/check-canary-tasks.ts`
+ * (`pnpm check`). Dangling task IDs fail the build.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see .github/workflows/tier-3-nightly.yml — consumer
+ */
+
+import { defineCanaryTasks } from "../_vendor/ailf-core/index.js"
+
+export default defineCanaryTasks({
+  tasks: [
+    {
+      taskId: "groq-blog-queries",
+      mode: "literacy",
+      rationale:
+        "Canonical first-use path for Sanity's most-used API. GROQ is the largest doc surface and the highest-leverage canary slot — drift here means drift in the most-consumed documentation. Filtering and pagination together exercise the largest cross-section of GROQ syntax in a single task.",
+    },
+    {
+      taskId: "content-lake-mutations",
+      mode: "literacy",
+      rationale:
+        "Foundational client API. CRUD is structurally distinct from query reasoning, so this catches regressions in mutation/transaction documentation that GROQ canary slots cannot reach. Every Sanity consumer eventually writes to the Content Lake.",
+    },
+    {
+      taskId: "portable-text-rendering",
+      mode: "literacy",
+      rationale:
+        "Major doc surface flagged as historically drift-prone in the testing audit. React-rendering of Portable Text mixes documentation, type definitions, and worked examples — a regression on any axis surfaces here first.",
+    },
+    {
+      taskId: "studio-blog-schema",
+      mode: "literacy",
+      rationale:
+        "Schema authoring (`defineType` / `defineField`) is the second-most-used surface after queries. Tests structural Studio docs that change shape across versions; pairs naturally with the GROQ canary because consumers typically author schemas before querying them.",
+    },
+    {
+      taskId: "kp-groq-projections",
+      mode: "knowledge-probe",
+      rationale:
+        "Cross-mode coverage. Pairs with `groq-blog-queries` (literacy) so we catch GROQ drift in both implementation (write code) and recall (explain syntax) modes. Knowledge-probe is the only non-literacy mode in the canary today; expand once mcp-server tasks land in the repo.",
+    },
+    // mcp-server canary slot — add a third mode here when a committed
+    // mcp-server task lands under packages/eval/tasks/mcp-server/. Today
+    // there are no production mcp-server tasks (only fixtures); the trigger
+    // is upstream and adding a placeholder slot would dangle. Surfaced at
+    // Phase 5 close (2026-04-27) — see W0116 retrospective.
+  ],
+})

package/dist/config/test-budgets.ts

@@ -0,0 +1,24 @@
+/**
+ * test-budgets.ts — Per-provider daily USD spend caps for Tier 3 CI runs.
+ *
+ * Each cap is the maximum cost a single Tier 3 nightly run may incur for
+ * that provider. The Tier 3 workflow (`.github/workflows/tier-3-nightly.yml`)
+ * fails loudly if any provider's actual spend exceeds its cap.
+ *
+ * The design doc names a $30–60/day envelope across all providers. Caps
+ * here divide that envelope per-provider; tighten as baseline canary spend
+ * becomes measurable.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see scripts/tier-3-budget-check.mjs — enforcement
+ */
+
+import { defineTestBudgets } from "../_vendor/ailf-core/index.js"
+
+export default defineTestBudgets({
+  perProviderDaily: {
+    anthropic: 30,
+    openai: 30,
+  },
+  warnFraction: 0.8,
+})

package/dist/pipeline/calculate-scores.d.ts

@@ -1,6 +1,6 @@
-import { type ActualScoreEntry, type ComponentResult, type Logger, type StoredTestResult, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
+import { type ActualScoreEntry, type ComponentResult, type Logger, type StoredTestResult, type TestResult, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
 import { type ResolvedSourceConfig } from "../sources.js";
-import type { GraderJudgment, PerModelEntry } from "./types.js";
+import type { FeatureScore, GraderJudgment, PerModelEntry } from "./types.js";
 export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, type ActualScoreEntry, type ComponentResult, type StoredTestResult, type TestResult, type UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
 export interface PromptfooResultsWrapper {
     results: RawTestResult[];
@@ -91,6 +91,21 @@ export declare function extractGraderJudgments(resultsPath: string): GraderJudgm
  * See D0029 and docs/design-docs/score-drill-down.md (Phase 1).
  */
 export declare function extractStoredTestResults(resultsPath: string): StoredTestResult[];
+/**
+ * Score knowledge-probe evaluation results.
+ *
+ * Knowledge-probe mode evaluates parametric recall: the model has no `docs`
+ * var and answers from training-data knowledge alone. The compiler explicitly
+ * deletes `vars.docs`, so every result lands in the without-docs bucket of
+ * the literacy scoring path — collapsing testCount and ceilingScore to zero.
+ *
+ * This branch mirrors the shape of `scoreAgentHarnessResults` but groups by
+ * feature area (KP results carry `__featureArea` from the compiler), and
+ * uses the `knowledge-probe` profile (factual-correctness / completeness /
+ * currency). Literacy-specific fields (ceilingScore, floorScore, docLift,
+ * docQualityGap) are zero — KP has no with-docs/without-docs decomposition.
+ */
+export declare function scoreKnowledgeProbeResults(results: TestResult[], profile: Record<string, number>): FeatureScore[];
 /**
  * Score agentic evaluation results. In agentic mode, all test entries are
  * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).

package/dist/pipeline/calculate-scores.js

@@ -719,6 +719,55 @@ function extractTaskId(description) {
     return description.trim() || "unknown";
 }
 // ---------------------------------------------------------------------------
+// Knowledge-probe scoring — closed-book recall with no docs context
+// ---------------------------------------------------------------------------
+/**
+ * Score knowledge-probe evaluation results.
+ *
+ * Knowledge-probe mode evaluates parametric recall: the model has no `docs`
+ * var and answers from training-data knowledge alone. The compiler explicitly
+ * deletes `vars.docs`, so every result lands in the without-docs bucket of
+ * the literacy scoring path — collapsing testCount and ceilingScore to zero.
+ *
+ * This branch mirrors the shape of `scoreAgentHarnessResults` but groups by
+ * feature area (KP results carry `__featureArea` from the compiler), and
+ * uses the `knowledge-probe` profile (factual-correctness / completeness /
+ * currency). Literacy-specific fields (ceilingScore, floorScore, docLift,
+ * docQualityGap) are zero — KP has no with-docs/without-docs decomposition.
+ */
+export function scoreKnowledgeProbeResults(results, profile) {
+    const byFeature = {};
+    for (const result of results) {
+        const feature = result.vars.__featureArea || detectFeatureArea(result.description);
+        if (!byFeature[feature]) {
+            byFeature[feature] = [];
+        }
+        byFeature[feature].push(result);
+    }
+    const scores = [];
+    for (const [feature, featureResults] of Object.entries(byFeature)) {
+        const scored = scoreTestGroup(featureResults, profile, feature);
+        scores.push({
+            assertionPassRate: scored.dimensions.assertionPassRate,
+            ceilingScore: 0,
+            codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+            dimensions: scored.dimensions,
+            docCoverage: scored.dimensions.docCoverage ?? 0,
+            docLift: 0,
+            docQualityGap: 0,
+            feature,
+            floorScore: 0,
+            groupType: "feature",
+            negativeDocLift: false,
+            taskCompletion: scored.dimensions.taskCompletion ?? 0,
+            testCount: featureResults.length,
+            totalCost: scored.totalCost,
+            totalScore: scored.composite,
+        });
+    }
+    return scores.sort((a, b) => a.feature.localeCompare(b.feature));
+}
+// ---------------------------------------------------------------------------
 // Agentic scoring — all results are "actual" (agent retrieves docs via tools)
 // ---------------------------------------------------------------------------
 /**
@@ -893,6 +942,56 @@ export function calculateAndWriteScores(options) {
         const testSummary = computeTestSummary(baselineResultsPath);
         return { belowCritical: summary.belowCritical, testSummary };
     }
+    // ── Knowledge-probe scoring path ────────────────────────────
+    // Knowledge-probe mode evaluates parametric recall (no docs context).
+    // The KP compiler deletes `vars.docs`, so the literacy path would bucket
+    // every result into `withoutDocs` and collapse testCount + dimensions
+    // to zero. This branch groups by feature area only and uses the
+    // `knowledge-probe` profile (factual-correctness / completeness /
+    // currency). See docs/design-docs/mode-agnostic-scoring.md.
+    if (mode === "knowledge-probe") {
+        const probeProfile = resolveProfile("knowledge-probe", "gold", rubricConfig);
+        log.debug("Knowledge-probe scoring profile", probeProfile);
+        const results = readAndNormalizeResults(baselineResultsPath);
+        const scores = scoreKnowledgeProbeResults(results, probeProfile);
+        log.debug("Knowledge-probe scores calculated", {
+            featureCount: scores.length,
+            features: scores.map((s) => ({
+                feature: s.feature,
+                totalScore: s.totalScore,
+                testCount: s.testCount,
+                dimensions: s.dimensions,
+            })),
+        });
+        const urlRefs = aggregateUrlReferences(baselineResultsPath);
+        const sourceVerification = buildSourceVerification(ROOT, source, {
+            allowedOrigins: options.allowedOrigins,
+            mode,
+            searchMode: options.searchMode,
+        });
+        const graderCost = extractGraderCost(baselineResultsPath);
+        const summary = printReport(scores, urlRefs, source, null, // no agent behavior — KP is closed-book
+        graderCost, null, // no per-model breakdown for now
+        null, // no source isolation — KP doesn't fetch sources
+        sourceVerification, "knowledge-probe", log);
+        // Persist
+        const outDir = join(ROOT, "results", "latest");
+        mkdirSync(outDir, { recursive: true });
+        writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
+        log.info("Score summary written to results/latest/score-summary.json");
+        const judgments = extractGraderJudgments(baselineResultsPath);
+        if (judgments.length > 0) {
+            writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
+            log.info(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
+        }
+        const testResults = extractStoredTestResults(baselineResultsPath);
+        if (testResults.length > 0) {
+            writeFileSync(join(outDir, "test-results.json"), JSON.stringify(testResults, null, 2));
+            log.info(`Test results written to results/latest/test-results.json (${testResults.length} results)`);
+        }
+        const testSummary = computeTestSummary(baselineResultsPath);
+        return { belowCritical: summary.belowCritical, testSummary };
+    }
     // ── Literacy scoring path ───────────────────────────────────
     // Gold (with-docs) entries use the "default" profile (3 dimensions).
     // Baseline (without-docs) entries use "output-only" (2 dimensions,

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.d.ts

@@ -9,6 +9,11 @@ import type { KnowledgeProbeCompileOptions } from "./types.js";
  * Tool-use assertions are rejected (knowledge probes don't use tools).
  * LLM-graded assertions receive the configured grader provider.
  * All other assertions are passed through.
+ *
+ * Templated `llm-rubric` assertions (those with `template` + `criteria`) go
+ * through the shared rubric resolver so the compiled assertion carries
+ * `metadata.dimension` — without this, the scoring engine can't classify
+ * KP grader output and dimension scores collapse to zero (W0128 / DOC-2077).
  */
 export declare function mapKnowledgeProbeAssertion(assertion: {
     type: string;

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.js

@@ -1,12 +1,18 @@
 /**
  * Assertion mapping for knowledge probe evaluations.
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 /**
  * Map a raw knowledge probe assertion to a Promptfoo assertion.
  *
  * Tool-use assertions are rejected (knowledge probes don't use tools).
  * LLM-graded assertions receive the configured grader provider.
  * All other assertions are passed through.
+ *
+ * Templated `llm-rubric` assertions (those with `template` + `criteria`) go
+ * through the shared rubric resolver so the compiled assertion carries
+ * `metadata.dimension` — without this, the scoring engine can't classify
+ * KP grader output and dimension scores collapse to zero (W0128 / DOC-2077).
  */
 export function mapKnowledgeProbeAssertion(assertion, options, warnings) {
     switch (assertion.type) {
@@ -27,9 +33,26 @@ export function mapKnowledgeProbeAssertion(assertion, options, warnings) {
                     ? { weight: assertion.weight }
                     : {}),
             };
-        // LLM-graded assertions — add grader provider
-        case "g-eval":
         case "llm-rubric":
+            // Templated form (template + criteria) → resolve to full rubric text
+            // with dimension metadata attached.
+            if ("template" in assertion && "criteria" in assertion) {
+                return resolveTemplatedAssertion(assertion, options?.rubricConfig, options?.graderProvider, warnings);
+            }
+            // Inline value form — pass through with grader provider, no metadata.
+            // Back-compat for tasks not yet migrated to the templated form.
+            return {
+                type: "llm-rubric",
+                ...("value" in assertion ? { value: assertion.value } : {}),
+                ...(typeof assertion.weight === "number"
+                    ? { weight: assertion.weight }
+                    : {}),
+                ...(options?.graderProvider
+                    ? { provider: options.graderProvider }
+                    : {}),
+            };
+        // Other LLM-graded assertions — add grader provider
+        case "g-eval":
         case "model-graded-closedqa":
         case "model-graded-factuality":
             return {

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js

@@ -37,7 +37,11 @@ export const handler = {
         if (!("mode" in task) || task.mode !== "knowledge-probe") {
             throw new Error(`Knowledge probe handler received task with mode "${task.mode ?? "undefined"}" — expected "knowledge-probe"`);
         }
-        const result = compileKnowledgeProbeTask(task, { graderProvider: ctx.graderProvider, models: ctx.models });
+        const result = compileKnowledgeProbeTask(task, {
+            graderProvider: ctx.graderProvider,
+            models: ctx.models,
+            rubricConfig: ctx.rubricConfig,
+        });
         return {
             providers: result.providers,
             tests: result.tests,

package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts

@@ -2,6 +2,7 @@
  * Public types for the knowledge-probe mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 /** Options for compiling a knowledge probe task */
 export interface KnowledgeProbeCompileOptions {
     /** Grader provider for LLM-graded assertions */
@@ -12,6 +13,9 @@ export interface KnowledgeProbeCompileOptions {
         label: string;
         config?: Record<string, unknown>;
     }[];
+    /** Rubric config (templates, weights, profiles) — needed to resolve
+     * templated `llm-rubric` assertions to dimension metadata. */
+    rubricConfig?: RubricConfig;
 }
 /** Result of compiling a single knowledge probe task */
 export interface KnowledgeProbeCompileResult {

package/dist/pipeline/compiler/promptfoo-compiler.js

@@ -11,10 +11,20 @@
  *
  * @see docs/archive/exec-plans/architecture-overhaul/phase-2-config-compiler.md
  */
+import { dirname, resolve as resolvePath } from "node:path";
+import { fileURLToPath } from "node:url";
 import { mapAssertions } from "./assertion-mapper.js";
 import { resolveTaskFixtures } from "./fixture-resolver.js";
 import { LiteracyVariant } from "../normalize-mode.js";
 import { resolveVariables } from "./variable-resolver.js";
+/**
+ * Absolute filesystem path to the AILF mock Promptfoo provider. Resolved
+ * once at module load relative to this file. Promptfoo's `file://` provider
+ * loader requires an absolute path. See buildProviders for the env-var
+ * gate that swaps real providers for this mock.
+ */
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const MOCK_PROVIDER_ABSPATH = resolvePath(__dirname, "..", "..", "promptfoo-providers", "mock-provider.cjs");
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -143,6 +153,19 @@ function buildProviders(models, mode) {
             },
         });
     }
+    // Replay swap — when AILF_REPLAY_LLMS=1 is set, rewrite every provider's
+    // `id` to the file-based AILF mock provider so the Promptfoo subprocess
+    // never makes a live LLM call. We preserve `label` and stash the
+    // original `id` in `config.originalId` so the mock provider can surface
+    // model identity in its output and reports remain interpretable.
+    // See W0110 (M5.1) and docs/design-docs/testing-strategy.md.
+    if (process.env.AILF_REPLAY_LLMS === "1") {
+        return providers.map((p) => ({
+            id: `file://${MOCK_PROVIDER_ABSPATH}`,
+            label: p.label,
+            config: { ...p.config, originalId: p.id },
+        }));
+    }
     return providers;
 }
 /**

package/dist/tasks/knowledge-probe/groq-projections.task.ts

@@ -41,22 +41,40 @@ export default defineTask({
   assertions: [
     { type: "contains", value: "->" },
     { type: "contains", value: "select(" },
+    // Templated rubrics so the compiled assertions carry `metadata.dimension`
+    // and the scoring engine can populate per-dimension scores from the KP
+    // profile (factual-correctness 0.45 / completeness 0.35 / currency 0.20).
     {
       type: "llm-rubric",
-      value:
-        "The response should demonstrate accurate knowledge of GROQ " +
-        "projection syntax with working code examples. Check that the " +
-        "dereference operator, spread syntax, and select() are correctly " +
-        "explained with valid GROQ code.",
-      weight: 0.6,
+      template: "factual-correctness",
+      criteria: [
+        "The dereference operator `->` is correctly explained for following references",
+        "The spread operator `...` is shown in a valid projection example",
+        "`select()` is used with valid syntax for conditional projections",
+        'Computed field names (e.g., `"label": title`) are demonstrated correctly',
+        "Code examples use valid GROQ — no fabricated operators or deprecated syntax",
+      ],
     },
     {
       type: "llm-rubric",
-      value:
-        "Evaluate whether the response reflects current GROQ syntax " +
-        "(post-2023). Check for deprecated patterns or outdated " +
-        "recommendations.",
-      weight: 0.4,
+      template: "completeness",
+      criteria: [
+        "Basic object projection with `{}` is covered",
+        "Nested projections and the spread operator are both addressed",
+        "Computed/aliased field names are demonstrated",
+        "The dereference operator `->` is included with a worked example",
+        "Both inclusive (`[0..5]`) and exclusive (`[0...5]`) array slicing are explained",
+        "Conditional projections via `select()` are covered",
+      ],
+    },
+    {
+      type: "llm-rubric",
+      template: "currency",
+      criteria: [
+        "Examples reflect current GROQ syntax (post-2023) — no deprecated patterns",
+        "Recommendations don't reference removed or legacy query forms",
+        "Modern projection idioms are used (e.g., spread + override)",
+      ],
     },
   ],
 })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "3.8.1",
+  "version": "3.9.0",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -77,6 +77,7 @@
     "test": "tsx --test src/__tests__/*.test.ts src/adapters/**/__tests__/*.adapter.test.ts",
     "test:e2e": "AILF_E2E=1 tsx --test src/__tests__/e2e/*.e2e.test.ts",
     "test:e2e:adapters": "AILF_E2E=1 tsx --test src/adapters/**/__tests__/*.adapter.test.ts",
+    "test:e2e:api": "AILF_E2E_API=1 tsx --test src/__tests__/api-tier2-tenant-integration.test.ts src/__tests__/run-remote-tier2.test.ts",
     "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts src/adapters/**/__tests__/*.adapter.test.ts",
     "pr-comment": "tsx src/cli.ts pr-comment",
     "coverage-audit": "tsx src/cli.ts report coverage",

package/tasks/knowledge-probe/groq-projections.task.ts

@@ -41,22 +41,40 @@ export default defineTask({
   assertions: [
     { type: "contains", value: "->" },
     { type: "contains", value: "select(" },
+    // Templated rubrics so the compiled assertions carry `metadata.dimension`
+    // and the scoring engine can populate per-dimension scores from the KP
+    // profile (factual-correctness 0.45 / completeness 0.35 / currency 0.20).
     {
       type: "llm-rubric",
-      value:
-        "The response should demonstrate accurate knowledge of GROQ " +
-        "projection syntax with working code examples. Check that the " +
-        "dereference operator, spread syntax, and select() are correctly " +
-        "explained with valid GROQ code.",
-      weight: 0.6,
+      template: "factual-correctness",
+      criteria: [
+        "The dereference operator `->` is correctly explained for following references",
+        "The spread operator `...` is shown in a valid projection example",
+        "`select()` is used with valid syntax for conditional projections",
+        'Computed field names (e.g., `"label": title`) are demonstrated correctly',
+        "Code examples use valid GROQ — no fabricated operators or deprecated syntax",
+      ],
     },
     {
       type: "llm-rubric",
-      value:
-        "Evaluate whether the response reflects current GROQ syntax " +
-        "(post-2023). Check for deprecated patterns or outdated " +
-        "recommendations.",
-      weight: 0.4,
+      template: "completeness",
+      criteria: [
+        "Basic object projection with `{}` is covered",
+        "Nested projections and the spread operator are both addressed",
+        "Computed/aliased field names are demonstrated",
+        "The dereference operator `->` is included with a worked example",
+        "Both inclusive (`[0..5]`) and exclusive (`[0...5]`) array slicing are explained",
+        "Conditional projections via `select()` are covered",
+      ],
+    },
+    {
+      type: "llm-rubric",
+      template: "currency",
+      criteria: [
+        "Examples reflect current GROQ syntax (post-2023) — no deprecated patterns",
+        "Recommendations don't reference removed or legacy query forms",
+        "Modern projection idioms are used (e.g., spread + override)",
+      ],
     },
   ],
 })