@sanity/ailf 7.0.1 → 7.1.2

package/dist/_vendor/ailf-shared/event-types.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Known notification event types and soft-enum helpers.
+ *
+ * Event types are free-form strings by design — teams can wire new events
+ * without a code change. This module seeds Studio comboboxes with canonical
+ * values and provides a narrowing predicate, without closing the enum.
+ *
+ * Parallel to the same type aliases in `@sanity/ailf-core`'s team module:
+ * `shared` is the leaf of the dependency graph, so the studio schema can
+ * import the runtime tuple without pulling in core.
+ */
+export declare const KNOWN_EVENT_TYPES: readonly ["eval.failed", "eval.completed", "eval.threshold-breached", "eval.score-regressed", "task.created", "task.archived", "area.unowned-tasks"];
+export type KnownEventType = (typeof KNOWN_EVENT_TYPES)[number];
+export type EventType = KnownEventType | (string & {});
+export declare function isKnownEventType(value: string): value is KnownEventType;

package/dist/_vendor/ailf-shared/event-types.js

@@ -0,0 +1,23 @@
+/**
+ * Known notification event types and soft-enum helpers.
+ *
+ * Event types are free-form strings by design — teams can wire new events
+ * without a code change. This module seeds Studio comboboxes with canonical
+ * values and provides a narrowing predicate, without closing the enum.
+ *
+ * Parallel to the same type aliases in `@sanity/ailf-core`'s team module:
+ * `shared` is the leaf of the dependency graph, so the studio schema can
+ * import the runtime tuple without pulling in core.
+ */
+export const KNOWN_EVENT_TYPES = [
+    "eval.failed",
+    "eval.completed",
+    "eval.threshold-breached",
+    "eval.score-regressed",
+    "task.created",
+    "task.archived",
+    "area.unowned-tasks",
+];
+export function isKnownEventType(value) {
+    return KNOWN_EVENT_TYPES.includes(value);
+}

package/dist/_vendor/ailf-shared/generated/help-content.js

@@ -88,7 +88,7 @@ export const HELP_TOPICS = [
     {
         "id": "scoring-model",
         "title": "Understanding Scores",
-        "body": "## The three dimensions\n\nEvery evaluation task is scored on three dimensions, each graded 0–100:\n\n- **Task Completion (50% weight)** — Can the AI implement the requested feature?\n  Does the output actually do what was asked?\n- **Code Correctness (25% weight)** — Is the generated code idiomatic, correct,\n  and following best practices?\n- **Doc Coverage (25% weight)** — Did the documentation provide the information\n  needed to implement the feature?\n\n## How the overall score is calculated\n\nThe three dimensions combine into a single **AI Literacy Score** per task using\nnamed scoring profiles from `config/rubrics.yaml`:\n\n```\nGold (with docs):    Total = Task × 0.50 + Code × 0.25 + Docs × 0.25\nBaseline (no docs):  Total = Task × 0.60 + Code × 0.40\n```\n\nThe gold profile includes all three dimensions. The baseline profile excludes\nDoc Coverage because it is undefined when no documentation is provided. This\nensures Doc Lift (ceiling − floor) is a clean structural measurement of\ndocumentation value.\n\nThe weighted composite produces a score from 0–100. Scores are then averaged\nacross all tasks in a feature area to produce a **per-area score**, and across\nall areas to produce the **overall score**.\n\n## What the numbers mean\n\n| Score range  | Interpretation                                                    |\n| ------------ | ----------------------------------------------------------------- |\n| **80–100**   | Docs are working well — AI agents produce correct implementations |\n| **70–79**    | Needs attention — there may be gaps in specific dimensions        |\n| **Below 70** | Weak — AI agents consistently struggle with this area             |\n\n## Ceiling decomposition (baseline mode)\n\nWhen running in baseline mode, each task is evaluated twice — with and without\ndocumentation. This produces:\n\n- **Floor score** — Score without docs (what the model knows from training data\n  alone)\n- **Ceiling score** — Score with gold-standard docs injected directly into the\n  prompt\n- **Doc Lift** — Ceiling minus floor. Positive means docs help; negative means\n  docs hurt.\n- **Doc Quality Gap** — 100 minus ceiling. Room for documentation improvement.\n\n## Three-layer decomposition (full mode)\n\nFull mode adds a third measurement — what happens when AI agents find docs on\ntheir own:\n\n- **Floor** — No docs (parametric knowledge only)\n- **Ceiling** — Gold-standard docs injected (best the docs can do)\n- **Actual** — Agent-retrieved docs (real-world performance)\n- **Retrieval Gap** — Ceiling minus actual (quality lost to findability)\n- **Infrastructure Efficiency** — Actual ÷ ceiling (what fraction of doc quality\n  reaches agents)\n\n## Cost tracking\n\nEach evaluation also tracks token costs:\n\n- **Provider cost** — Token usage for generating implementations\n- **Grader cost** — Token usage for the grading model's assessments\n- **Total cost** — Both combined, reported in the score summary",
+        "body": "## The three dimensions\n\nEvery evaluation task is scored on three dimensions, each graded 0–100:\n\n- **Task Completion (50% weight)** — Can the AI implement the requested feature?\n  Does the output actually do what was asked?\n- **Code Correctness (25% weight)** — Is the generated code idiomatic, correct,\n  and following best practices?\n- **Doc Coverage (25% weight)** — Did the documentation provide the information\n  needed to implement the feature?\n\n## How the overall score is calculated\n\nThe three dimensions combine into a single **AI Literacy Score** per task using\nnamed scoring profiles from `packages/eval/config/rubrics.ts`:\n\n```\nGold (with docs):    Total = Task × 0.50 + Code × 0.25 + Docs × 0.25\nBaseline (no docs):  Total = Task × 0.60 + Code × 0.40\n```\n\nThe gold profile includes all three dimensions. The baseline profile excludes\nDoc Coverage because it is undefined when no documentation is provided. This\nensures Doc Lift (ceiling − floor) is a clean structural measurement of\ndocumentation value.\n\nThe weighted composite produces a score from 0–100. Scores are then averaged\nacross all tasks in a feature area to produce a **per-area score**, and across\nall areas to produce the **overall score**.\n\n## What the numbers mean\n\n| Score range  | Interpretation                                                    |\n| ------------ | ----------------------------------------------------------------- |\n| **80–100**   | Docs are working well — AI agents produce correct implementations |\n| **70–79**    | Needs attention — there may be gaps in specific dimensions        |\n| **Below 70** | Weak — AI agents consistently struggle with this area             |\n\n## Ceiling decomposition (baseline mode)\n\nWhen running in baseline mode, each task is evaluated twice — with and without\ndocumentation. This produces:\n\n- **Floor score** — Score without docs (what the model knows from training data\n  alone)\n- **Ceiling score** — Score with gold-standard docs injected directly into the\n  prompt\n- **Doc Lift** — Ceiling minus floor. Positive means docs help; negative means\n  docs hurt.\n- **Doc Quality Gap** — 100 minus ceiling. Room for documentation improvement.\n\n## Three-layer decomposition (full mode)\n\nFull mode adds a third measurement — what happens when AI agents find docs on\ntheir own:\n\n- **Floor** — No docs (parametric knowledge only)\n- **Ceiling** — Gold-standard docs injected (best the docs can do)\n- **Actual** — Agent-retrieved docs (real-world performance)\n- **Retrieval Gap** — Ceiling minus actual (quality lost to findability)\n- **Infrastructure Efficiency** — Actual ÷ ceiling (what fraction of doc quality\n  reaches agents)\n\n## Cost tracking\n\nEach evaluation also tracks token costs:\n\n- **Provider cost** — Token usage for generating implementations\n- **Grader cost** — Token usage for the grading model's assessments\n- **Total cost** — Both combined, reported in the score summary",
         "source": "docs/help/scoring-model.md",
         "related": [
             "three-layer",
@@ -99,7 +99,7 @@ export const HELP_TOPICS = [
     {
         "id": "weaknesses-recommendations",
         "title": "Weaknesses & Recommendations",
-        "body": "## Understanding weaknesses\n\nThe Issues sub-tab in Diagnostics lists every area or dimension that scored\nbelow threshold. Each weakness entry shows:\n\n- **The feature area** — Which product feature is affected (e.g., GROQ,\n  Functions, Webhooks).\n- **The bottleneck dimension** — Which scoring dimension is dragging the area\n  down: task completion, code correctness, or doc coverage.\n- **The score** — How far below threshold the dimension scored.\n\n## Gap analysis recommendations\n\nWhen an evaluation runs with gap analysis enabled, the dashboard shows\n**prioritized recommendations** — specific actions ranked by estimated impact.\n\nEach recommendation includes:\n\n- **Failure mode** — The type of doc problem identified:\n  - `missing-docs` — The functionality isn't documented at all.\n  - `incorrect-docs` — The docs contain factual errors.\n  - `outdated-docs` — The docs describe an old API version or pattern.\n  - `poor-structure` — The docs exist but are hard to find or understand.\n- **Estimated lift** — How many score points fixing this gap would add. Based on\n  raising the bottleneck dimension to the median of non-bottleneck dimensions.\n  Conservative estimate — actual improvement may be higher.\n- **Confidence** — How sure the analysis is about this diagnosis (high, medium,\n  or low).\n- **Affected tasks** — Which specific evaluation tasks exposed this gap.\n\n## Low-scoring judgments\n\nBelow the recommendations, you'll find the **grader's explanations** for tests\nthat scored below 70. These are the raw assessments from the grading model\nexplaining exactly what went wrong — missing API calls, incorrect patterns,\nhallucinated features, etc.\n\nEach judgment shows the task, the dimension, the score, and the grader's natural\nlanguage reason. These are the most granular diagnostic signal available and\noften point directly to the doc section that needs fixing.",
+        "body": "## Understanding weaknesses\n\nThe Issues sub-tab in Diagnostics lists every area or dimension that scored\nbelow threshold. Each weakness entry shows:\n\n- **The feature area** — Which product feature is affected (e.g., GROQ,\n  Functions, Webhooks).\n- **The bottleneck dimension** — Which scoring dimension is dragging the area\n  down: task completion, code correctness, or doc coverage.\n- **The score** — How far below threshold the dimension scored.\n\n## Gap analysis recommendations\n\nWhen an evaluation runs with gap analysis enabled, the dashboard shows\n**prioritized recommendations** — specific actions ranked by estimated impact.\n\nEach recommendation includes:\n\n- **Failure mode** — The type of doc problem identified:\n  - `missing-docs` — The functionality isn't documented at all.\n  - `incorrect-docs` — The docs contain factual errors.\n  - `outdated-docs` — The docs describe an old API version or pattern.\n  - `poor-structure` — The docs exist but are hard to find or understand.\n- **Estimated lift** — How many score points fixing this gap would add. Based on\n  raising the bottleneck dimension to the median of non-bottleneck dimensions.\n  Conservative estimate — actual improvement may be higher.\n- **Confidence** — How sure the analysis is about this diagnosis (high, medium,\n  or low).\n- **Affected tasks** — Which specific evaluation tasks exposed this gap.\n\n## Diagnosis cards\n\nEvery published report now carries a **diagnosis artifact** — a set of cards\nproduced by the post-pipeline hook (`ailf interpret`). The Studio diagnosis\npanel renders these cards directly; the dashboard's Recommendations and\nFailure-modes panels migrate to the same source in a follow-up.\n\nThe hook runs by default for every pipeline invocation. To opt out for a single\nrun, pass `--no-summary`; to opt out in CI, set `AILF_INTERPRET_ON_RUN=0` in the\nworkflow env block; to opt out project-wide, set `summary.onRun: never` in\n`.ailf/config.yaml`.\n\n## Low-scoring judgments\n\nBelow the recommendations, you'll find the **grader's explanations** for tests\nthat scored below 70. These are the raw assessments from the grading model\nexplaining exactly what went wrong — missing API calls, incorrect patterns,\nhallucinated features, etc.\n\nEach judgment shows the task, the dimension, the score, and the grader's natural\nlanguage reason. These are the most granular diagnostic signal available and\noften point directly to the doc section that needs fixing.",
         "source": "docs/help/weaknesses-recommendations.md",
         "related": [
             "interpreting-diagnostics",

package/dist/_vendor/ailf-shared/index.d.ts

@@ -18,17 +18,19 @@
  * surface against future regressions.
  */
 export { computeCanaryDrift, type CanaryDriftReport, type CanaryReportSlim, type DriftEntry, type DriftThresholds, type DriftVerdict, } from "./canary-drift.js";
-export { type DocumentRef } from "./document-ref.js";
+export { buildContextDocPath, type DocumentRef } from "./document-ref.js";
 export { makeEditorialReference, type EditorialReference, type MakeEditorialReferenceArgs, } from "./editorial-reference.js";
+export { isKnownEventType, KNOWN_EVENT_TYPES, type EventType, type KnownEventType, } from "./event-types.js";
 export { FEATURE_FLAGS, type FeatureFlag, type FeatureFlagKey, } from "./feature-flags.js";
 export { DEFAULT_GCS_ARTIFACT_BUCKET } from "./gcs-defaults.js";
 export { GLOSSARY, type GlossaryEntry, type GlossarySlug } from "./glossary.js";
 export { HELP_TOPICS } from "./help-content.js";
 export { type HelpTopic } from "./help-topics.js";
+export { isKnownMemberRole, KNOWN_MEMBER_ROLES, type KnownMemberRole, type MemberRole, } from "./member-roles.js";
 export { GRADE_BOUNDARIES, scoreGrade, type ScoreGrade, } from "./score-grades.js";
 export { NOISE_THRESHOLD } from "./noise-threshold.js";
-export { CANONICAL_EVAL_MODES, LEGACY_EVAL_MODE_ALIASES, LITERACY_VARIANTS, RAW_EVAL_MODES, type EvalMode, type LiteracyVariant, type RawEvalMode, } from "./eval-modes.js";
-export { isKnownOwnerTeam, KNOWN_OWNER_TEAMS, normalizeOwnerTeam, } from "./owner-teams.js";
+export { CANONICAL_EVAL_MODES, isLiteracyVariant, LEGACY_EVAL_MODE_ALIASES, LITERACY_VARIANTS, RAW_EVAL_MODES, type EvalMode, type LiteracyVariant, type RawEvalMode, } from "./eval-modes.js";
+export { isKnownOwnerTeam, KNOWN_OWNER_TEAMS, normalizeOwnerTeam, resolveTeamRef, type SlugLike, } from "./owner-teams.js";
 export { isRunClassification, RUN_CLASSIFICATIONS, RUN_EXECUTOR_SURFACES, type RunClassification, type RunExecutor, type RunExecutorSurface, type RunExecutorSystem, type RunExecutorUser, type RunHost, type RunLineage, type RunOwner, type RunTool, } from "./run-classification.js";
 export { type RunTrigger } from "./run-trigger.js";
 export { type RunContext } from "./run-context.js";

package/dist/_vendor/ailf-shared/index.js

@@ -18,13 +18,16 @@
  * surface against future regressions.
  */
 export { computeCanaryDrift, } from "./canary-drift.js";
+export { buildContextDocPath } from "./document-ref.js";
 export { makeEditorialReference, } from "./editorial-reference.js";
+export { isKnownEventType, KNOWN_EVENT_TYPES, } from "./event-types.js";
 export { FEATURE_FLAGS, } from "./feature-flags.js";
 export { DEFAULT_GCS_ARTIFACT_BUCKET } from "./gcs-defaults.js";
 export { GLOSSARY } from "./glossary.js";
 export { HELP_TOPICS } from "./help-content.js";
+export { isKnownMemberRole, KNOWN_MEMBER_ROLES, } from "./member-roles.js";
 export { GRADE_BOUNDARIES, scoreGrade, } from "./score-grades.js";
 export { NOISE_THRESHOLD } from "./noise-threshold.js";
-export { CANONICAL_EVAL_MODES, LEGACY_EVAL_MODE_ALIASES, LITERACY_VARIANTS, RAW_EVAL_MODES, } from "./eval-modes.js";
-export { isKnownOwnerTeam, KNOWN_OWNER_TEAMS, normalizeOwnerTeam, } from "./owner-teams.js";
+export { CANONICAL_EVAL_MODES, isLiteracyVariant, LEGACY_EVAL_MODE_ALIASES, LITERACY_VARIANTS, RAW_EVAL_MODES, } from "./eval-modes.js";
+export { isKnownOwnerTeam, KNOWN_OWNER_TEAMS, normalizeOwnerTeam, resolveTeamRef, } from "./owner-teams.js";
 export { isRunClassification, RUN_CLASSIFICATIONS, RUN_EXECUTOR_SURFACES, } from "./run-classification.js";

package/dist/_vendor/ailf-shared/member-roles.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Known team-member roles and soft-enum helpers.
+ *
+ * Roles are free-form strings — teams can introduce custom roles (e.g.
+ * `"reviewer"`, `"sme"`) without a code change. This module seeds Studio
+ * comboboxes with canonical values and exposes a narrowing predicate
+ * without closing the enum.
+ *
+ * Parallel to the same type aliases in `@sanity/ailf-core`'s team module:
+ * `shared` is the leaf of the dependency graph, so the studio schema can
+ * import the runtime tuple without pulling in core.
+ */
+export declare const KNOWN_MEMBER_ROLES: readonly ["lead", "member", "oncall"];
+export type KnownMemberRole = (typeof KNOWN_MEMBER_ROLES)[number];
+export type MemberRole = KnownMemberRole | (string & {});
+export declare function isKnownMemberRole(value: string): value is KnownMemberRole;

package/dist/_vendor/ailf-shared/member-roles.js

@@ -0,0 +1,16 @@
+/**
+ * Known team-member roles and soft-enum helpers.
+ *
+ * Roles are free-form strings — teams can introduce custom roles (e.g.
+ * `"reviewer"`, `"sme"`) without a code change. This module seeds Studio
+ * comboboxes with canonical values and exposes a narrowing predicate
+ * without closing the enum.
+ *
+ * Parallel to the same type aliases in `@sanity/ailf-core`'s team module:
+ * `shared` is the leaf of the dependency graph, so the studio schema can
+ * import the runtime tuple without pulling in core.
+ */
+export const KNOWN_MEMBER_ROLES = ["lead", "member", "oncall"];
+export function isKnownMemberRole(value) {
+    return KNOWN_MEMBER_ROLES.includes(value);
+}

package/dist/_vendor/ailf-shared/owner-teams.d.ts

@@ -24,3 +24,22 @@ export declare const KNOWN_OWNER_TEAMS: readonly string[];
  */
 export declare function normalizeOwnerTeam(value: string | undefined | null): string;
 export declare function isKnownOwnerTeam(value: string): boolean;
+/**
+ * Lightweight team lookup against an in-memory team list.
+ *
+ * Consumers fetch team docs via GROQ then call this helper to resolve a
+ * freeform `owner.team` string (D0037) into a Sanity reference. Unknown
+ * strings return null. Known aliases (via OWNER_TEAM_ALIASES) are honored.
+ *
+ * Returning null is the cue for the UI to render an "unresolved team"
+ * badge — not an error condition.
+ */
+export interface SlugLike {
+    _id: string;
+    slug: {
+        current: string;
+    };
+}
+export declare function resolveTeamRef(value: string | null | undefined, teams: readonly SlugLike[]): {
+    _ref: string;
+} | null;

package/dist/_vendor/ailf-shared/owner-teams.js

@@ -14,11 +14,21 @@
  * @see docs/decisions/D0037-run-classification-and-ownership-taxonomy.md
  */
 export const KNOWN_OWNER_TEAMS = [
+    "ai-growth",
+    "billing-and-integrations",
+    "content-agent",
     "content-lake",
-    "core-docs",
-    "growth",
-    "media",
-    "platform",
+    "data",
+    "design-and-research",
+    "docs",
+    "editorial-experience",
+    "engineering",
+    "identity",
+    "media-library",
+    "product",
+    "runtime",
+    "sdk",
+    "ssi",
     "studio",
 ];
 /**
@@ -26,8 +36,11 @@ export const KNOWN_OWNER_TEAMS = [
  * drift has been observed belong here. Unknown values pass through.
  */
 const OWNER_TEAM_ALIASES = {
-    coredocs: "core-docs",
-    docs: "core-docs",
+    "core-docs": "docs",
+    coredocs: "docs",
+    documentation: "docs",
+    growth: "ai-growth",
+    media: "media-library",
     studio_team: "studio",
     "studio-team": "studio",
 };
@@ -50,3 +63,10 @@ export function normalizeOwnerTeam(value) {
 export function isKnownOwnerTeam(value) {
     return KNOWN_OWNER_TEAMS.includes(value);
 }
+export function resolveTeamRef(value, teams) {
+    const normalized = normalizeOwnerTeam(value ?? undefined);
+    if (!normalized || normalized === "unknown")
+        return null;
+    const match = teams.find((t) => t.slug?.current === normalized);
+    return match ? { _ref: match._id } : null;
+}

package/dist/_vendor/ailf-shared/run-context.d.ts

@@ -14,7 +14,7 @@
  * @see docs/decisions/D0032-run-anchored-artifact-store.md
  * @see docs/design-docs/run-artifact-store.md (§ Drift Prevention)
  */
-import type { EvalMode } from "./eval-modes.js";
+import type { EvalMode, LiteracyVariant } from "./eval-modes.js";
 import type { RunClassification, RunExecutor, RunHost, RunLineage, RunOwner, RunTool } from "./run-classification.js";
 import type { RunTrigger } from "./run-trigger.js";
 export interface RunContext {
@@ -75,4 +75,11 @@ export interface RunContext {
     tool?: RunTool;
     /** What initiated this run */
     trigger: RunTrigger;
+    /**
+     * Literacy mode variant — `baseline`, `agentic`, `observed`, or `full`.
+     * Only meaningful when `mode === "literacy"`; absent for other modes.
+     * Surfaced on `ReportProvenance` so dashboards can disambiguate which
+     * variant produced a given report.
+     */
+    variant?: LiteracyVariant;
 }

package/dist/adapters/doc-fetchers/sanity-doc-fetcher.js

@@ -16,6 +16,7 @@
 import { mkdirSync, writeFileSync } from "fs";
 import { join } from "path";
 import { canonicalDocRefLabel, isIdRef, isPathRef, isPerspectiveRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
+import { buildContextDocPath } from "../../_vendor/ailf-shared/index.js";
 import { fetchUrlContent, } from "../../pipeline/fetch-url-content.js";
 import { createPerspectiveClient, createPublishedClient, getSanityClient, } from "../../sanity/client.js";
 import { extractSymbolsForDoc, renderDocument, } from "../../sanity/document-renderers.js";
@@ -376,7 +377,20 @@ export class SanityDocFetcher {
             : getSanityClient(toSanityOverrides(source));
         const allMetadata = await client.fetch(ARTICLES_METADATA_BY_SLUGS_QUERY, { slugs: [...allSlugs] });
         return allMetadata
-            .map((m) => ({ _id: m._id, _rev: m._rev, slug: m.slug, title: m.title }))
+            .map((m) => {
+            const path = buildContextDocPath({
+                sectionSlug: m.sectionSlug,
+                slug: m.slug,
+            });
+            return {
+                _id: m._id,
+                _rev: m._rev,
+                slug: m.slug,
+                ...(m.sectionSlug ? { sectionSlug: m.sectionSlug } : {}),
+                ...(path ? { path } : {}),
+                title: m.title,
+            };
+        })
             .sort((a, b) => a.slug.localeCompare(b.slug));
     }
     // -----------------------------------------------------------------------

package/dist/adapters/grader-outputs/promptfoo-grader-output.d.ts

@@ -99,4 +99,68 @@ export declare const GraderJudgmentSchema: z.ZodObject<{
         graderJudgmentsVersion: z.ZodString;
     }, z.core.$strip>;
 }, z.core.$strict>;
-export type { GraderJudgment } from "../../_vendor/ailf-core/index.d.ts";
+/**
+ * Wire-format schema — what the grader LLM is asked to emit.
+ *
+ * This is the subset of {@link GraderJudgmentSchema} that contains only
+ * fields the LLM can correctly produce. The pipeline parses untrusted
+ * grader output against this shape, then synthesizes the four
+ * pipeline-owned fields (`judgmentId`, `metadata.graderModel`,
+ * `metadata.graderJudgmentsVersion`, `hallucinationCheckedAgainst`) plus
+ * the three result-context fields (`taskId`, `modelId`, `dimension`) to
+ * build the full {@link GraderJudgment} storage shape.
+ *
+ * `.strict()` is retained — the LLM is told exactly which keys to emit,
+ * so any extras are either prompt-injection attempts or noise we want
+ * to surface as parse failures (which then drop to the
+ * `synthesizeUnparsedJudgment` fallback). The bar to hit that fallback
+ * is much lower than before W0273 — previously a missing pipeline-owned
+ * field tripped it on every emission.
+ *
+ * Asserts `satisfies z.ZodType<GraderEmittedJudgment>` against the
+ * independently-authored core type (D0045). The trust-boundary CI gate
+ * (`pnpm check-trust-boundary-satisfies`) covers this file.
+ *
+ * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md
+ */
+export declare const GraderEmittedJudgmentSchema: z.ZodObject<{
+    score: z.ZodNumber;
+    reason: z.ZodString;
+    failureMode: z.ZodString;
+    subJudgments: z.ZodArray<z.ZodObject<{
+        criterionId: z.ZodString;
+        met: z.ZodBoolean;
+        evidence: z.ZodString;
+        confidence: z.ZodObject<{
+            level: z.ZodEnum<{
+                low: "low";
+                medium: "medium";
+                high: "high";
+            }>;
+            signalsPresent: z.ZodNumber;
+            derivation: z.ZodString;
+        }, z.core.$strip>;
+    }, z.core.$strip>>;
+    docCitations: z.ZodArray<z.ZodObject<{
+        documentId: z.ZodString;
+        slug: z.ZodOptional<z.ZodString>;
+        role: z.ZodEnum<{
+            supports: "supports";
+            contradicts: "contradicts";
+            missing: "missing";
+            irrelevant: "irrelevant";
+        }>;
+        hallucinated: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strip>>;
+    confidence: z.ZodObject<{
+        level: z.ZodEnum<{
+            low: "low";
+            medium: "medium";
+            high: "high";
+        }>;
+        signalsPresent: z.ZodNumber;
+        derivation: z.ZodString;
+    }, z.core.$strip>;
+    outputFailure: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>;
+export type { GraderEmittedJudgment, GraderJudgment } from "../../_vendor/ailf-core/index.d.ts";

package/dist/adapters/grader-outputs/promptfoo-grader-output.js

@@ -91,3 +91,38 @@ export const GraderJudgmentSchema = z
     }),
 })
     .strict();
+/**
+ * Wire-format schema — what the grader LLM is asked to emit.
+ *
+ * This is the subset of {@link GraderJudgmentSchema} that contains only
+ * fields the LLM can correctly produce. The pipeline parses untrusted
+ * grader output against this shape, then synthesizes the four
+ * pipeline-owned fields (`judgmentId`, `metadata.graderModel`,
+ * `metadata.graderJudgmentsVersion`, `hallucinationCheckedAgainst`) plus
+ * the three result-context fields (`taskId`, `modelId`, `dimension`) to
+ * build the full {@link GraderJudgment} storage shape.
+ *
+ * `.strict()` is retained — the LLM is told exactly which keys to emit,
+ * so any extras are either prompt-injection attempts or noise we want
+ * to surface as parse failures (which then drop to the
+ * `synthesizeUnparsedJudgment` fallback). The bar to hit that fallback
+ * is much lower than before W0273 — previously a missing pipeline-owned
+ * field tripped it on every emission.
+ *
+ * Asserts `satisfies z.ZodType<GraderEmittedJudgment>` against the
+ * independently-authored core type (D0045). The trust-boundary CI gate
+ * (`pnpm check-trust-boundary-satisfies`) covers this file.
+ *
+ * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md
+ */
+export const GraderEmittedJudgmentSchema = z
+    .object({
+    score: z.number(),
+    reason: z.string(),
+    failureMode: z.string(),
+    subJudgments: z.array(CriterionSubJudgmentSchema),
+    docCitations: z.array(DocCitationSchema),
+    confidence: ConfidenceSchema,
+    outputFailure: z.boolean().optional(),
+})
+    .strict();

package/dist/adapters/task-sources/changed-docs-filter.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Filter tasks by changed doc slugs.
+ *
+ * When `changedDocs` is set and non-empty, returns only tasks whose
+ * `context.docs[*].slug` intersects the provided list. Tasks without a
+ * `context.docs` array (e.g. knowledge-probe, mcp-server, agent-harness
+ * modes) are excluded — there's no way for them to "touch" a doc slug.
+ *
+ * An empty or undefined `changedDocs` is a no-op (returns input).
+ */
+import type { GeneralizedTaskDefinition } from "../../_vendor/ailf-core/index.d.ts";
+export declare function filterByChangedDocs(tasks: readonly GeneralizedTaskDefinition[], changedDocs: readonly string[] | undefined): GeneralizedTaskDefinition[];

package/dist/adapters/task-sources/changed-docs-filter.js

@@ -0,0 +1,30 @@
+/**
+ * Every variant of `GeneralizedTaskDefinition` declares an optional
+ * `context?: { docs?: GeneralizedDocRef[]; ... }`, so structural access
+ * narrows correctly without an `as` cast. Returns `undefined` when the
+ * task carries no doc refs.
+ */
+function taskContextDocs(task) {
+    return task.context?.docs;
+}
+/**
+ * `GeneralizedDocRef` is a 4-way union — only `SlugDocRef` and `IdDocRef`
+ * carry a `slug`. Returns `undefined` for path / perspective refs.
+ */
+function docSlug(ref) {
+    return "slug" in ref ? ref.slug : undefined;
+}
+export function filterByChangedDocs(tasks, changedDocs) {
+    if (!changedDocs || changedDocs.length === 0)
+        return [...tasks];
+    const wanted = new Set(changedDocs);
+    return tasks.filter((task) => {
+        const docs = taskContextDocs(task);
+        if (!docs || docs.length === 0)
+            return false;
+        return docs.some((d) => {
+            const slug = docSlug(d);
+            return slug != null && wanted.has(slug);
+        });
+    });
+}

package/dist/adapters/task-sources/content-lake-task-source.js

@@ -15,6 +15,8 @@
  * @see packages/core/src/ports/task-source.ts — TaskSource port
  * @see docs/decisions/D0038-content-lake-authorable-task-modes.md
  */
+import { buildContextDocPath } from "../../_vendor/ailf-shared/index.js";
+import { filterByChangedDocs } from "./changed-docs-filter.js";
 import { ContentLakeAuthorableTaskSchema } from "./repo-schemas.js";
 // ---------------------------------------------------------------------------
 // GROQ query — fetches ailf.task documents with resolved references
@@ -127,7 +129,7 @@ export class ContentLakeTaskSource {
             console.warn("  ⚠️  ContentLakeTaskSource: no ailf.task documents found in the Content Lake. " +
                 "Have you run the migration (Phase 3) or created tasks in Studio?");
         }
-        return definitions;
+        return filterByChangedDocs(definitions, filter?.changedDocs);
     }
 }
 // ---------------------------------------------------------------------------
@@ -222,9 +224,11 @@ function mapCanonicalDocRef(raw) {
         case "slug":
             return raw.slug ? { slug: raw.slug, reason } : null;
         case "path": {
-            // Prefer explicit path field; fall back to deriving from doc reference
-            const path = raw.path ||
-                (raw.sectionSlug && raw.slug ? `${raw.sectionSlug}/${raw.slug}` : null);
+            const path = buildContextDocPath({
+                path: raw.path,
+                sectionSlug: raw.sectionSlug,
+                slug: raw.slug,
+            });
             return path ? { path, reason } : null;
         }
         case "id": {
@@ -232,10 +236,12 @@ function mapCanonicalDocRef(raw) {
             const id = raw.docId || raw.docRefId || null;
             if (!id)
                 return null;
-            // Carry slug and derived path as optional DX annotations
-            const derivedPath = raw.sectionSlug && raw.slug
-                ? `${raw.sectionSlug}/${raw.slug}`
-                : undefined;
+            // Carry slug and derived path as optional DX annotations — single
+            // source of truth in `buildContextDocPath` (@sanity/ailf-shared).
+            const derivedPath = buildContextDocPath({
+                sectionSlug: raw.sectionSlug,
+                slug: raw.slug,
+            });
             return {
                 id,
                 reason,

package/dist/adapters/task-sources/repo-task-source.js

@@ -22,6 +22,7 @@ import { existsSync, readdirSync, readFileSync } from "fs";
 import { resolve } from "path";
 import { load } from "js-yaml";
 import { CANONICAL_EVAL_MODES } from "../../_vendor/ailf-shared/index.js";
+import { filterByChangedDocs } from "./changed-docs-filter.js";
 import { detectLegacyFieldNames, migratePromptShape, parseCanonicalTaskFile, } from "./repo-schemas.js";
 import { discoverTsTaskFiles, loadTsTaskFile } from "./task-file-loader.js";
 /** Set of canonical mode names for O(1) lookup */
@@ -111,7 +112,7 @@ export class RepoTaskSource {
                 }
             }
         }
-        return definitions;
+        return filterByChangedDocs(definitions, filter?.changedDocs);
     }
 }
 // ---------------------------------------------------------------------------

package/dist/commands/pipeline-action.d.ts

@@ -98,13 +98,14 @@ export declare function computeResolvedOptions(opts: PipelineCliOptions): Resolv
 /**
  * Determine whether the post-run diagnosis summary hook should fire.
  *
- * 4-level precedence chain (D6-20):
+ * 4-level precedence chain (D0054):
  *   Level 1 — CLI flag (absolute): if `cliOpts.summary` is boolean, use it.
  *   Level 2 — AILF_INTERPRET_ON_RUN env var (absolute): strict "1"/"0" parse;
  *             anything else falls through (T-06-11 spoofing mitigation).
  *   Level 3 — config `summary.onRun` (absolute): "always" → true; "never" → false;
  *             "auto" or absent falls through to level 4.
- *   Level 4 — default auto: TTY && !CI (SC1 default-off in CI).
+ *   Level 4 — default always-on: diagnosis is a first-class output produced
+ *             for every pipeline run unless explicitly opted out at levels 1–3.
  */
 export declare function shouldRunPostSummary(cliOpts: PipelineCliOptions, resolvedOnRun: "auto" | "always" | "never" | undefined): boolean;
 export declare function buildSynthesisTelemetry(diagnosis: Diagnosis): SynthesisCostTelemetry;
@@ -113,7 +114,7 @@ export declare function buildSynthesisTelemetry(diagnosis: Diagnosis): Synthesis
  *
  * Fires after orchestratePipeline() + writePipelineResult() (D6-02).
  * Hook failure prints to stderr but does NOT change exit code (D6-03).
- * CI default-off: fires only when shouldRunPostSummary returns true (D6-20).
+ * Fires whenever shouldRunPostSummary returns true (D0054 — default on).
  *
  * @param ctx - App context (composition root wiring)
  * @param result - Pipeline result (includes reportId when published)

package/dist/commands/pipeline-action.js

@@ -388,13 +388,14 @@ function resolvePublishAuto(repoValue) {
 /**
  * Determine whether the post-run diagnosis summary hook should fire.
  *
- * 4-level precedence chain (D6-20):
+ * 4-level precedence chain (D0054):
  *   Level 1 — CLI flag (absolute): if `cliOpts.summary` is boolean, use it.
  *   Level 2 — AILF_INTERPRET_ON_RUN env var (absolute): strict "1"/"0" parse;
  *             anything else falls through (T-06-11 spoofing mitigation).
  *   Level 3 — config `summary.onRun` (absolute): "always" → true; "never" → false;
  *             "auto" or absent falls through to level 4.
- *   Level 4 — default auto: TTY && !CI (SC1 default-off in CI).
+ *   Level 4 — default always-on: diagnosis is a first-class output produced
+ *             for every pipeline run unless explicitly opted out at levels 1–3.
  */
 export function shouldRunPostSummary(cliOpts, resolvedOnRun) {
     // Level 1: CLI flag wins absolutely
@@ -415,8 +416,9 @@ export function shouldRunPostSummary(cliOpts, resolvedOnRun) {
     if (resolvedOnRun === "never")
         return false;
     // "auto" or undefined falls through
-    // Level 4: default auto — fire only when stdout is interactive and not in CI
-    return Boolean(process.stdout.isTTY) && process.env.CI !== "true";
+    // Level 4: diagnosis is on by default — emit a diagnosis artifact for every
+    // pipeline run unless an upstream level explicitly opted out (D0054).
+    return true;
 }
 /**
  * Build a SynthesisCostTelemetry payload from a completed Diagnosis.
@@ -479,7 +481,7 @@ export function buildSynthesisTelemetry(diagnosis) {
  *
  * Fires after orchestratePipeline() + writePipelineResult() (D6-02).
  * Hook failure prints to stderr but does NOT change exit code (D6-03).
- * CI default-off: fires only when shouldRunPostSummary returns true (D6-20).
+ * Fires whenever shouldRunPostSummary returns true (D0054 — default on).
  *
  * @param ctx - App context (composition root wiring)
  * @param result - Pipeline result (includes reportId when published)

package/dist/commands/run.js

@@ -43,8 +43,8 @@ export function createRunCommand() {
         .option("-p, --publish", "Write report to Sanity + fan out to sinks (auto-enabled for full runs when report store is configured)")
         .option("--no-publish", "Suppress auto-publishing")
         .option("--publish-tag <tag>", "Label for published report")
-        .option("--summary", "Force post-run diagnosis summary (overrides config and CI default-off)")
-        .option("--no-summary", "Suppress post-run diagnosis summary")
+        .option("--summary", "Force post-run diagnosis summary (overrides config and env var)")
+        .option("--no-summary", "Suppress post-run diagnosis summary (default is on)")
         .option("--config <path>", "Load pipeline config from a TS/JS/YAML/JSON file (overrides most CLI flags)")
         .option("-o, --output <path>", "Write PR comment markdown to file")
         .option("--promptfoo-url <url>", "Promptfoo share URL for report")

package/dist/config/rubrics.ts

@@ -15,10 +15,6 @@ import { defineRubrics } from "../_vendor/ailf-core/index.js"
 // template entry below. Source of truth lives in packages/eval/src/grader/;
 // the helper picks the right list by dimension family.
 import { failureModesForDimension } from "../grader/index.js"
-// Single source of truth for the wire-format version stamped into the
-// grader-prompt footer (VER-01 D-02). Interpolated below so the
-// announced version cannot drift from the schema's expected value.
-import { graderJudgmentsVersion } from "../adapters/grader-outputs/index.js"
 
 export default defineRubrics({
   templates: {
@@ -242,20 +238,23 @@ export default defineRubrics({
     "agent-harness": { gold: "agent-harness" },
   },
 
-  // Phase 3 GRAD-05 (Plan 03-01) — structured GraderJudgment JSON sketch.
-  // Documents the target wire format the grader emits. The strict schema's
-  // GRAD-02 additive fields stay optional in this plan; Plan 03-04 flips
-  // them to required and bumps graderJudgmentsVersion to 1.0.0.
+  // W0273 — the footer documents the wire-format subset of GraderJudgment
+  // that the grader LLM actually controls. The pipeline parses this against
+  // GraderEmittedJudgmentSchema and then synthesizes the four pipeline-owned
+  // fields (judgmentId, metadata.{graderModel,graderJudgmentsVersion},
+  // hallucinationCheckedAgainst) to build the storage GraderJudgment.
+  //
+  // See docs/audits/2026-05-22-empty-gap-analysis-regression.md for the
+  // rationale (Phase 3 GRAD-05 made these fields required + .strict(),
+  // and asking the LLM for pipeline-owned values caused 100% parse
+  // failures starting 2026-05-11).
   footer: `Return ONLY a JSON object with this exact shape:
 {
-  "judgmentId": "<string>",
   "score": <number 0-100>,
   "reason": "<explanation, ≤500 chars>",
+  "failureMode": "<one of the declared modes for this dimension or 'unclassified'>",
   "subJudgments": [{ "criterionId": "<id>", "met": <bool>, "evidence": "<≤280 chars>", "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" } }],
   "docCitations": [{ "documentId": "<id>", "slug": "<optional slug>", "role": "supports|contradicts|missing|irrelevant", "hallucinated": <bool> }],
-  "failureMode": "<one of the declared modes for this dimension or 'unclassified'>",
-  "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" },
-  "hallucinationCheckedAgainst": ["<doc id>"],
-  "metadata": { "graderModel": "<string>", "graderJudgmentsVersion": "${graderJudgmentsVersion}" }
+  "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" }
 }`,
 })