@sanity/ailf 3.8.1 → 4.0.0

package/config/canary-tasks.ts

@@ -0,0 +1,64 @@
+/**
+ * canary-tasks.ts — The Tier 3 canary set.
+ *
+ * Five tasks the Tier 3 nightly workflow runs against live LLMs every day.
+ * Composition follows the design doc's "weighted toward modes/areas with
+ * the most production usage and the highest historical regression rates"
+ * recommendation: GROQ and Content Lake (foundational consumer surfaces),
+ * Portable Text (historically drift-prone), Studio schema authoring (the
+ * second-most-used surface after queries), and a knowledge-probe pairing
+ * for cross-mode coverage.
+ *
+ * Each entry's `rationale` is the canary's load-bearing field — without it,
+ * future maintainers can't reason about whether a regression is meaningful
+ * or whether the slot has lost value. Update the rationale when you swap a
+ * canary entry; never silently replace one.
+ *
+ * Validated against the live task inventory by `scripts/check-canary-tasks.ts`
+ * (`pnpm check`). Dangling task IDs fail the build.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see .github/workflows/tier-3-nightly.yml — consumer
+ */
+
+import { defineCanaryTasks } from "@sanity/ailf-core"
+
+export default defineCanaryTasks({
+  tasks: [
+    {
+      taskId: "groq-blog-queries",
+      mode: "literacy",
+      rationale:
+        "Canonical first-use path for Sanity's most-used API. GROQ is the largest doc surface and the highest-leverage canary slot — drift here means drift in the most-consumed documentation. Filtering and pagination together exercise the largest cross-section of GROQ syntax in a single task.",
+    },
+    {
+      taskId: "content-lake-mutations",
+      mode: "literacy",
+      rationale:
+        "Foundational client API. CRUD is structurally distinct from query reasoning, so this catches regressions in mutation/transaction documentation that GROQ canary slots cannot reach. Every Sanity consumer eventually writes to the Content Lake.",
+    },
+    {
+      taskId: "portable-text-rendering",
+      mode: "literacy",
+      rationale:
+        "Major doc surface flagged as historically drift-prone in the testing audit. React-rendering of Portable Text mixes documentation, type definitions, and worked examples — a regression on any axis surfaces here first.",
+    },
+    {
+      taskId: "studio-blog-schema",
+      mode: "literacy",
+      rationale:
+        "Schema authoring (`defineType` / `defineField`) is the second-most-used surface after queries. Tests structural Studio docs that change shape across versions; pairs naturally with the GROQ canary because consumers typically author schemas before querying them.",
+    },
+    {
+      taskId: "kp-groq-projections",
+      mode: "knowledge-probe",
+      rationale:
+        "Cross-mode coverage. Pairs with `groq-blog-queries` (literacy) so we catch GROQ drift in both implementation (write code) and recall (explain syntax) modes. Knowledge-probe is the only non-literacy mode in the canary today; expand once mcp-server tasks land in the repo.",
+    },
+    // mcp-server canary slot — add a third mode here when a committed
+    // mcp-server task lands under packages/eval/tasks/mcp-server/. Today
+    // there are no production mcp-server tasks (only fixtures); the trigger
+    // is upstream and adding a placeholder slot would dangle. Surfaced at
+    // Phase 5 close (2026-04-27) — see W0116 retrospective.
+  ],
+})

package/config/models.ts

@@ -35,16 +35,23 @@ export default defineModels({
 
     // ── OpenAI ─────────────────────────────────────────────────
     {
+      // gpt-5.2 routes through chat completions (and through the in-house
+      // agentic provider for naive/optimized variants). `verbosity` is a
+      // Responses-API-only field — it would be silently dropped here, so
+      // it isn't configured. See W0131.
       id: "openai:chat:gpt-5.2",
       label: "GPT 5.2",
       config: {
         max_completion_tokens: 8192,
-        verbosity: "medium",
       },
       modes: ["literacy", "knowledge-probe"],
       // All literacy variants included by default
     },
     {
+      // GPT 5.4 evaluated only on the baseline literacy variant. Promptfoo's
+      // native handling of `openai:responses:` honors reasoning / verbosity /
+      // summary; the in-house agentic provider does not (W0131). MCP-server
+      // and knowledge-probe routes go through Promptfoo native too.
       id: "openai:responses:gpt-5.4",
       label: "GPT 5.4",
       config: {
@@ -55,7 +62,9 @@ export default defineModels({
       },
       timeoutMs: 600_000, // 10 min — reasoning model needs more headroom
       modes: ["literacy", "mcp-server", "knowledge-probe"],
-      // All literacy variants included by default
+      variants: {
+        literacy: ["baseline"],
+      },
     },
 
     // ── Disabled models (uncomment to enable) ──────────────────
@@ -93,12 +102,31 @@ export default defineModels({
   defaults: {
     temperature: 0.2,
     max_tokens: 4096,
-    maxToolRounds: 5, // for agentic modes
+    // Global default round budget for agentic modes. Per-mode overrides
+    // below give naive more headroom (W0134) since it spends rounds on
+    // retries when fetches fail. Per-model `config.maxToolRounds` still
+    // wins over both values.
+    maxToolRounds: 5,
+    modeMaxToolRounds: {
+      "agentic-naive": 8,
+      "agentic-optimized": 5,
+    },
     observerOptions: {
-      maxPreviewBytes: 2048,
+      // Per-class preview caps (W0133): default 4 KB, but search responses
+      // get 16 KB and llms.txt gets 128 KB so trace audits can resolve
+      // which result the model actually saw.
+      maxPreviewBytes: 4096,
+      previewLimits: {
+        default: 4096,
+        llmsTxt: 131072,
+        search: 16384,
+      },
       captureResponsePreview: true,
       includePatterns: ["sanity.io", "sanity.dev", "cdn.sanity.io"],
       sensitiveHeaders: ["authorization", "cookie", "x-api-key"],
+      // statusOnlyForUnmatched defaults to true (W0132) — model-side
+      // traffic to api.openai.com / api.anthropic.com / googleapis.com
+      // surfaces in run artifacts as slim status-only entries.
     },
   },
 })

package/config/test-budgets.ts

@@ -0,0 +1,24 @@
+/**
+ * test-budgets.ts — Per-provider daily USD spend caps for Tier 3 CI runs.
+ *
+ * Each cap is the maximum cost a single Tier 3 nightly run may incur for
+ * that provider. The Tier 3 workflow (`.github/workflows/tier-3-nightly.yml`)
+ * fails loudly if any provider's actual spend exceeds its cap.
+ *
+ * The design doc names a $30–60/day envelope across all providers. Caps
+ * here divide that envelope per-provider; tighten as baseline canary spend
+ * becomes measurable.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see scripts/tier-3-budget-check.mjs — enforcement
+ */
+
+import { defineTestBudgets } from "@sanity/ailf-core"
+
+export default defineTestBudgets({
+  perProviderDaily: {
+    anthropic: 30,
+    openai: 30,
+  },
+  warnFraction: 0.8,
+})

package/dist/_vendor/ailf-core/config-helpers.d.ts

@@ -26,9 +26,11 @@
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md (canonical)
  */
 import type { EvalConfig } from "./schemas/eval-config.js";
+import type { CanaryTaskSetConfig } from "./schemas/canary-tasks.js";
 import type { FeatureRegistry, RubricConfig, ThresholdConfig } from "./schemas/pipeline.js";
 import type { SchedulesFile } from "./schemas/schedules.js";
 import type { SinksFile } from "./schemas/sinks.js";
+import type { TestBudgetConfig } from "./schemas/test-budgets.js";
 import type { ModelsConfig } from "./types/index.js";
 import type { GeneralizedTaskDefinition } from "./types/generalized-task.js";
 import type { ModeBase, PresetDefinition } from "./types/plugin-registry.js";
@@ -55,8 +57,14 @@ export declare function defineTask(task: GeneralizedTaskDefinition): Generalized
  * Validates:
  * - Every `modes` entry is a canonical eval mode name
  * - Every `variants` key is a mode the model is enrolled in
+ * - `openai:responses:` model ids are not used for agentic literacy variants
+ *   (the in-house agentic loop dispatches to chat completions only)
+ * - Responses-API-only fields (`reasoning`, `summary`, `verbosity`) are not
+ *   set on a model that routes through the agentic provider — they would be
+ *   silently dropped.
  *
- * @throws {Error} On invalid mode names or mismatched variant keys
+ * @throws {Error} On invalid mode names, mismatched variant keys, or
+ *                 misconfigured OpenAI Responses-API fields.
  */
 export declare function defineModels(models: ModelsConfig): ModelsConfig;
 /**
@@ -89,6 +97,23 @@ export declare function defineSinks(sinks: SinksFile): SinksFile;
  * Used in `config/schedules.ts` for typed schedule configuration.
  */
 export declare function defineSchedules(schedules: SchedulesFile): SchedulesFile;
+/**
+ * Define per-provider daily USD spend caps for Tier 3 (live-LLM) CI runs.
+ *
+ * Used in `config/test-budgets.ts` for typed budget configuration.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export declare function defineTestBudgets(budgets: TestBudgetConfig): TestBudgetConfig;
+/**
+ * Define the curated canary task set for the Tier 3 nightly workflow.
+ *
+ * Used in `config/canary-tasks.ts`. Validation against the live task
+ * inventory happens in `scripts/check-canary-tasks.ts`, not here.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export declare function defineCanaryTasks(canary: CanaryTaskSetConfig): CanaryTaskSetConfig;
 /**
  * Source configuration — typed inline until a dedicated schema exists.
  *

package/dist/_vendor/ailf-core/config-helpers.js

@@ -54,6 +54,33 @@ export function defineTask(task) {
 // ---------------------------------------------------------------------------
 // Model registry helpers
 // ---------------------------------------------------------------------------
+/**
+ * OpenAI Responses-API-only fields. The agentic provider's OpenAI loop
+ * routes everything through `/v1/chat/completions` and would silently drop
+ * these. We surface the misconfiguration at config-load time instead.
+ *
+ * @see docs/work-items/W0131-honor-openai-responses-provider.json
+ */
+const RESPONSES_ONLY_FIELDS = ["reasoning", "summary", "verbosity"];
+/**
+ * Whether a model would be assembled into agentic-naive or agentic-optimized
+ * literacy variants. These are the variants that route through the in-house
+ * agentic provider (which speaks chat completions only); baseline routes
+ * through Promptfoo's native handling, which honors `openai:responses:` ids.
+ *
+ * Note: variant names mirror the literacy mode base in
+ * `packages/eval/src/pipeline/compiler/mode-bases/literacy.ts`.
+ */
+function participatesInAgenticLiteracy(model) {
+    const enrolledInLiteracy = !model.modes || model.modes.includes("literacy");
+    if (!enrolledInLiteracy)
+        return false;
+    const literacyVariants = model.variants?.literacy;
+    if (!literacyVariants)
+        return true;
+    return (literacyVariants.includes("agentic-naive") ||
+        literacyVariants.includes("agentic-optimized"));
+}
 /**
  * Define the model registry (models to evaluate and grader model).
  *
@@ -62,8 +89,14 @@ export function defineTask(task) {
  * Validates:
  * - Every `modes` entry is a canonical eval mode name
  * - Every `variants` key is a mode the model is enrolled in
+ * - `openai:responses:` model ids are not used for agentic literacy variants
+ *   (the in-house agentic loop dispatches to chat completions only)
+ * - Responses-API-only fields (`reasoning`, `summary`, `verbosity`) are not
+ *   set on a model that routes through the agentic provider — they would be
+ *   silently dropped.
  *
- * @throws {Error} On invalid mode names or mismatched variant keys
+ * @throws {Error} On invalid mode names, mismatched variant keys, or
+ *                 misconfigured OpenAI Responses-API fields.
  */
 export function defineModels(models) {
     const validModes = new Set(CANONICAL_EVAL_MODES);
@@ -87,6 +120,26 @@ export function defineModels(models) {
                 }
             }
         }
+        const usesAgentic = participatesInAgenticLiteracy(model);
+        if (usesAgentic && model.id.startsWith("openai:responses:")) {
+            throw new Error(`Model "${model.label ?? model.id}": the in-house agentic provider ` +
+                `does not implement the OpenAI Responses API endpoint — requests would ` +
+                `be silently downgraded to chat completions. Either restrict variants to ` +
+                `["baseline"] (Promptfoo's native handling honors openai:responses:) or ` +
+                `change the id to "openai:chat:..." for agentic evaluation. ` +
+                `See W0131 for context.`);
+        }
+        if (usesAgentic && model.config) {
+            const droppedFields = RESPONSES_ONLY_FIELDS.filter((f) => f in model.config);
+            if (droppedFields.length > 0) {
+                throw new Error(`Model "${model.label ?? model.id}": configured fields ` +
+                    `${droppedFields.map((f) => `"${f}"`).join(", ")} are only honored ` +
+                    `by the OpenAI Responses API. The agentic provider's chat-completions ` +
+                    `path would silently drop them. Either remove these fields or restrict ` +
+                    `variants to ["baseline"] so the model is evaluated only through ` +
+                    `Promptfoo's native Responses-API handler. See W0131 for context.`);
+            }
+        }
     }
     return models;
 }
@@ -145,6 +198,33 @@ export function defineSinks(sinks) {
 export function defineSchedules(schedules) {
     return schedules;
 }
+// ---------------------------------------------------------------------------
+// Test-budget helpers
+// ---------------------------------------------------------------------------
+/**
+ * Define per-provider daily USD spend caps for Tier 3 (live-LLM) CI runs.
+ *
+ * Used in `config/test-budgets.ts` for typed budget configuration.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export function defineTestBudgets(budgets) {
+    return budgets;
+}
+// ---------------------------------------------------------------------------
+// Canary task-set helpers
+// ---------------------------------------------------------------------------
+/**
+ * Define the curated canary task set for the Tier 3 nightly workflow.
+ *
+ * Used in `config/canary-tasks.ts`. Validation against the live task
+ * inventory happens in `scripts/check-canary-tasks.ts`, not here.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ */
+export function defineCanaryTasks(canary) {
+    return canary;
+}
 /**
  * Define documentation source configurations.
  *

package/dist/_vendor/ailf-core/index.d.ts

@@ -17,7 +17,7 @@ export * from "./services/index.js";
 export * from "./examples/index.js";
 export * from "./artifact-registry.js";
 export * from "./batch-signing.js";
-export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
+export { defineCanaryTasks, defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineTestBudgets, defineThresholds, } from "./config-helpers.js";
 export type { PricingEntry, PromptEntry, SourceEntry, } from "./config-helpers.js";
 export { env } from "./env-helper.js";
 export { NoOpArtifactWriter, NotImplementedError, } from "./ports/artifact-writer.js";

package/dist/_vendor/ailf-core/index.js

@@ -20,7 +20,7 @@ export * from "./batch-signing.js";
 // ---------------------------------------------------------------------------
 // Architecture overhaul — Phase 0 helpers
 // ---------------------------------------------------------------------------
-export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
+export { defineCanaryTasks, defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineTestBudgets, defineThresholds, } from "./config-helpers.js";
 export { env } from "./env-helper.js";
 export { NoOpArtifactWriter, NotImplementedError, } from "./ports/artifact-writer.js";
 export { assoc, resolveVariantMode, splitTaskVariant, } from "./artifact-capture/association.js";

package/dist/_vendor/ailf-core/schemas/canary-tasks.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @sanity/ailf-core — Canary task-set schemas.
+ *
+ * The canary task set is the curated subset of evaluation tasks the Tier 3
+ * nightly workflow runs against live LLMs. Each entry pins a `taskId` and
+ * `mode` together with a one-paragraph rationale documenting why the task
+ * earned a slot — the rationale is the canary set's single most important
+ * field; without it, future maintainers can't reason about whether a
+ * regression is meaningful or whether the slot has lost value.
+ *
+ * Validation that canary IDs map to real tasks lives in
+ * `scripts/check-canary-tasks.ts` (run by `pnpm check`); it can't live in
+ * Zod because the inventory comes from the repo task glob, not the schema.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/canary-tasks.ts — authored config
+ */
+import { z } from "zod";
+export declare const CanaryTaskEntrySchema: z.ZodObject<{
+    taskId: z.ZodString;
+    mode: z.ZodEnum<{
+        custom: "custom";
+        agentic: "agentic";
+        literacy: "literacy";
+        "mcp-server": "mcp-server";
+        "agent-harness": "agent-harness";
+        "knowledge-probe": "knowledge-probe";
+        baseline: "baseline";
+        observed: "observed";
+        full: "full";
+    }>;
+    rationale: z.ZodString;
+}, z.core.$strip>;
+export type CanaryTaskEntry = z.infer<typeof CanaryTaskEntrySchema>;
+export declare const CanaryTaskSetConfigSchema: z.ZodObject<{
+    tasks: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        mode: z.ZodEnum<{
+            custom: "custom";
+            agentic: "agentic";
+            literacy: "literacy";
+            "mcp-server": "mcp-server";
+            "agent-harness": "agent-harness";
+            "knowledge-probe": "knowledge-probe";
+            baseline: "baseline";
+            observed: "observed";
+            full: "full";
+        }>;
+        rationale: z.ZodString;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type CanaryTaskSetConfig = z.infer<typeof CanaryTaskSetConfigSchema>;

package/dist/_vendor/ailf-core/schemas/canary-tasks.js

@@ -0,0 +1,46 @@
+/**
+ * @sanity/ailf-core — Canary task-set schemas.
+ *
+ * The canary task set is the curated subset of evaluation tasks the Tier 3
+ * nightly workflow runs against live LLMs. Each entry pins a `taskId` and
+ * `mode` together with a one-paragraph rationale documenting why the task
+ * earned a slot — the rationale is the canary set's single most important
+ * field; without it, future maintainers can't reason about whether a
+ * regression is meaningful or whether the slot has lost value.
+ *
+ * Validation that canary IDs map to real tasks lives in
+ * `scripts/check-canary-tasks.ts` (run by `pnpm check`); it can't live in
+ * Zod because the inventory comes from the repo task glob, not the schema.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/canary-tasks.ts — authored config
+ */
+import { z } from "zod";
+import { RAW_EVAL_MODES } from "../../ailf-shared/index.js";
+// ---------------------------------------------------------------------------
+// Canary entry
+// ---------------------------------------------------------------------------
+export const CanaryTaskEntrySchema = z.object({
+    taskId: z.string().min(1, "canary entries must have a non-empty taskId"),
+    mode: z.enum(RAW_EVAL_MODES),
+    rationale: z
+        .string()
+        .min(40, "canary rationale must be at least one informative sentence"),
+});
+// ---------------------------------------------------------------------------
+// Canary set config
+// ---------------------------------------------------------------------------
+export const CanaryTaskSetConfigSchema = z
+    .object({
+    tasks: z.array(CanaryTaskEntrySchema),
+})
+    .refine((config) => {
+    const seen = new Set();
+    for (const entry of config.tasks) {
+        const key = `${entry.mode}:${entry.taskId}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+    }
+    return true;
+}, { message: "duplicate (mode, taskId) entries in canary set" });

package/dist/_vendor/ailf-core/schemas/index.d.ts

@@ -9,8 +9,10 @@
  * (Phase 0d). Original files are now re-export barrels.
  */
 export * from "./callback-payload.js";
+export * from "./canary-tasks.js";
 export * from "./eval-config.js";
 export * from "./pipeline-request.js";
 export * from "./pipeline.js";
 export * from "./schedules.js";
 export * from "./sinks.js";
+export * from "./test-budgets.js";

package/dist/_vendor/ailf-core/schemas/index.js

@@ -9,8 +9,10 @@
  * (Phase 0d). Original files are now re-export barrels.
  */
 export * from "./callback-payload.js";
+export * from "./canary-tasks.js";
 export * from "./eval-config.js";
 export * from "./pipeline-request.js";
 export * from "./pipeline.js";
 export * from "./schedules.js";
 export * from "./sinks.js";
+export * from "./test-budgets.js";

package/dist/_vendor/ailf-core/schemas/test-budgets.d.ts

@@ -0,0 +1,19 @@
+/**
+ * @sanity/ailf-core — Test-budget schemas.
+ *
+ * Per-provider daily USD spend caps for Tier 3 (live-LLM) CI workflows.
+ * The cap is the maximum cost a single Tier 3 nightly run may incur for
+ * a given provider; the workflow fails loudly if any provider's actual
+ * spend exceeds its cap.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/test-budgets.ts — authored config
+ */
+import { z } from "zod";
+export declare const ProviderBudgetCapsSchema: z.ZodRecord<z.ZodString, z.ZodNumber>;
+export type ProviderBudgetCaps = z.infer<typeof ProviderBudgetCapsSchema>;
+export declare const TestBudgetConfigSchema: z.ZodObject<{
+    perProviderDaily: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    warnFraction: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+export type TestBudgetConfig = z.infer<typeof TestBudgetConfigSchema>;

package/dist/_vendor/ailf-core/schemas/test-budgets.js

@@ -0,0 +1,34 @@
+/**
+ * @sanity/ailf-core — Test-budget schemas.
+ *
+ * Per-provider daily USD spend caps for Tier 3 (live-LLM) CI workflows.
+ * The cap is the maximum cost a single Tier 3 nightly run may incur for
+ * a given provider; the workflow fails loudly if any provider's actual
+ * spend exceeds its cap.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/eval/config/test-budgets.ts — authored config
+ */
+import { z } from "zod";
+// ---------------------------------------------------------------------------
+// Provider cap entry
+// ---------------------------------------------------------------------------
+const NonNegativeUsd = z
+    .number()
+    .nonnegative("budget caps must be non-negative USD amounts");
+export const ProviderBudgetCapsSchema = z
+    .record(z.string().min(1), NonNegativeUsd)
+    .refine((caps) => Object.keys(caps).length > 0, {
+    message: "perProviderDaily must declare at least one provider",
+});
+// ---------------------------------------------------------------------------
+// Test budget config
+// ---------------------------------------------------------------------------
+export const TestBudgetConfigSchema = z.object({
+    perProviderDaily: ProviderBudgetCapsSchema,
+    warnFraction: z
+        .number()
+        .gt(0)
+        .lte(1, "warnFraction must be in (0, 1]")
+        .default(0.8),
+});

package/dist/_vendor/ailf-shared/canary-drift.d.ts

@@ -0,0 +1,84 @@
+/**
+ * canary/drift.ts — Pure drift-statistic computation for the Tier 3
+ * framework-tests-framework loop.
+ *
+ * Consumes the projection shape returned by Studio's `latestReportsQuery`
+ * (we accept a slim subset so the function stays a pure-domain dependency
+ * with no Studio-package import). Computes per-area Δscore between the
+ * most-recent canary run and the trailing-N median, plus an overall
+ * Δscore for the run as a whole. Output classifies each delta as `ok`,
+ * `warn`, or `regression` against caller-provided thresholds.
+ *
+ * The function is total — it never throws. Edge cases (empty trailing
+ * window, missing scores) surface as `verdict: "no-baseline"` so the
+ * caller can decide whether to treat the missing baseline as a fail.
+ *
+ * @see docs/design-docs/testing-strategy.md — "Tier 3 — Live LLMs"
+ * @see packages/studio/src/queries.ts — `latestReportsQuery`
+ */
+/** Slim projection of a canary run report — subset of `ReportListItem`. */
+export interface CanaryReportSlim {
+    reportId: string;
+    completedAt: string;
+    overall: number;
+    scores: {
+        feature: string;
+        totalScore: number;
+    }[];
+}
+/** Verdict for a single Δ computation. */
+export type DriftVerdict = "ok" | "warn" | "regression" | "no-baseline";
+/** Δ between the most-recent run and the trailing-N median. */
+export interface DriftEntry {
+    /** "overall" for the run-level avg, or the area slug for a per-area Δ. */
+    feature: string;
+    /** Score in the most-recent run. */
+    current: number;
+    /** Median of trailing-N runs (excluding the most-recent). Null when no baseline. */
+    trailingMedian: number | null;
+    /** current − trailingMedian. Null when no baseline. */
+    delta: number | null;
+    verdict: DriftVerdict;
+}
+/** Tunable thresholds — caller decides what counts as warn vs regression. */
+export interface DriftThresholds {
+    /**
+     * How many prior runs (excluding the most-recent) form the trailing
+     * baseline. Sensible defaults sit between 5 and 10 for a daily canary.
+     */
+    trailingN: number;
+    /** Drop ≥ this magnitude (and < failDelta) → `warn`. */
+    warnDelta: number;
+    /** Drop ≥ this magnitude → `regression`. */
+    failDelta: number;
+    /**
+     * Minimum trailing-window size required to compute a delta. When the
+     * window is smaller, the entry's verdict is `no-baseline`. Defaults to
+     * 1 — a single prior run is enough to detect *some* movement.
+     */
+    minBaselineRuns?: number;
+}
+/** Aggregate result of `computeCanaryDrift`. */
+export interface CanaryDriftReport {
+    /** ID + timestamp of the most-recent run. */
+    reportId: string;
+    completedAt: string;
+    /** Run-level Δscore (overall avg). */
+    overall: DriftEntry;
+    /** Per-area Δscores. One entry per area present in the most-recent run. */
+    byArea: DriftEntry[];
+    /** True when any verdict is `regression`. */
+    hasRegression: boolean;
+    /** True when any verdict is `warn` or `regression`. */
+    hasMovement: boolean;
+}
+/**
+ * Compute per-area + overall drift for a sequence of canary runs.
+ *
+ * `reports` must be ordered **newest-first** (matching `latestReportsQuery`'s
+ * `order(completedAt desc)`). The most-recent run is `reports[0]`; the
+ * trailing window is `reports.slice(1, 1 + trailingN)`.
+ *
+ * @throws never — all error states surface as `no-baseline` verdicts.
+ */
+export declare function computeCanaryDrift(reports: CanaryReportSlim[], thresholds: DriftThresholds): CanaryDriftReport | null;