@sanity/ailf 4.6.0 → 5.0.0

package/canonical/grader-references/agent-harness-tools.yaml

@@ -0,0 +1,42 @@
+# agent-harness-tools.yaml — Human-graded reference sample for agent-harness tasks
+#
+# Phase 3 GRAD-03 calibration fixture (Plan 03-02). Top-level
+# `expectedFailureMode` + `dimension` are read by the calibration test in
+# packages/eval/src/grader/__tests__/calibration.test.ts.
+
+- taskId: agent-multi-step-schema-edit
+  area: agent-harness
+  dimension: agent-harness
+  expectedFailureMode: tool-misuse
+  response: |
+    To add a `coverImage` field to the `post` schema and migrate existing
+    documents, the agent ran the following tool calls:
+
+    1. `read_file({ path: "schemas/post.ts" })` — read the schema.
+    2. `write_file({ path: "schemas/post.ts", content: "<new schema>" })` — wrote the updated schema.
+    3. `run_command({ command: "sanity migrate add-cover-image" })` — attempted to run a migration.
+
+    The `sanity migrate` command failed because no migration script exists at
+    that name. The agent then stopped without creating the migration file.
+  rubrics:
+    - dimension: tool-usage
+      rubricText: |
+        Score the agent's tool usage from 0 to 100:
+        - 0: Completely wrong tool usage — called wrong tools or with invalid inputs
+        - 25: Poor tool usage — correct tools but wrong parameters or sequencing
+        - 50: Adequate — correct tools and basic parameters, some inefficiency
+        - 75: Good — efficient tool usage with proper error handling
+        - 100: Excellent — optimal tool selection, correct inputs, minimal redundancy
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      humanScore: 25
+      notes: |
+        The agent invoked `run_command` with a migration name that did not
+        exist in the project, instead of first listing available migrations
+        with `list_migrations` (or equivalent) or scaffolding a new migration
+        file with the correct tool. The migration step was a tool-misuse:
+        the right tool exists, but the agent supplied invalid arguments
+        (a non-existent migration name) and then failed to recover by
+        scaffolding a new migration. Failure mode: tool-misuse.
+  gradedBy: "senior-engineer"
+  gradedAt: "2026-05-10"

package/canonical/grader-references/knowledge-probe-recall.yaml

@@ -0,0 +1,36 @@
+# knowledge-probe-recall.yaml — Human-graded reference sample for KP tasks
+#
+# Phase 3 GRAD-03 calibration fixture (Plan 03-02). Top-level
+# `expectedFailureMode` + `dimension` are read by the calibration test in
+# packages/eval/src/grader/__tests__/calibration.test.ts.
+
+- taskId: kp-default-dataset
+  area: knowledge-probe
+  dimension: knowledge-probe
+  expectedFailureMode: factual-error
+  response: |
+    The default dataset for new Sanity Studio projects is `production`. When
+    you run `npm create sanity@latest` and accept the defaults, the CLI
+    creates a dataset named `production`.
+  rubrics:
+    - dimension: factual-correctness
+      rubricText: |
+        Score factual correctness from 0 to 100:
+        - 0: Completely wrong — fabricated facts, hallucinated APIs, incorrect syntax
+        - 25: Mostly wrong — some real concepts but significant factual errors
+        - 50: Mixed — some correct facts alongside notable inaccuracies
+        - 75: Mostly correct — minor inaccuracies or imprecisions
+        - 100: Fully correct — all statements are accurate and verifiable
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      humanScore: 30
+      notes: |
+        Per current Sanity CLI docs the default dataset chosen by
+        `npm create sanity@latest` is selectable, with `production` being
+        the conventional default name surfaced in the prompt — but the
+        candidate phrases this as a deterministic default rather than a
+        prompt-driven choice. The factual claim "the default is production"
+        is wrong as stated; the CLI prompts for a dataset name and offers
+        `production` as a suggested value. Failure mode: factual-error.
+  gradedBy: "senior-engineer"
+  gradedAt: "2026-05-10"

package/canonical/grader-references/mcp-server-spec.yaml

@@ -0,0 +1,51 @@
+# mcp-server-spec.yaml — Human-graded reference sample for MCP server tasks
+#
+# Phase 3 GRAD-03 calibration fixture (Plan 03-02). Top-level
+# `expectedFailureMode` + `dimension` are read by the calibration test in
+# packages/eval/src/grader/__tests__/calibration.test.ts.
+
+- taskId: mcp-search-tool-pagination
+  area: mcp-server
+  dimension: mcp-behavior
+  expectedFailureMode: spec-mismatch
+  response: |
+    Here's an MCP `search` tool that returns paginated results:
+
+    ```typescript
+    import { Server } from "@modelcontextprotocol/sdk/server/index.js"
+
+    server.setRequestHandler({ method: "tools/call" }, async (request) => {
+      if (request.params.name === "search") {
+        const { query, page = 1, pageSize = 10 } = request.params.arguments
+        const results = await searchIndex(query, page, pageSize)
+        return {
+          content: [
+            { type: "text", text: JSON.stringify(results) },
+          ],
+          nextPageToken: page + 1,
+        }
+      }
+    })
+    ```
+  rubrics:
+    - dimension: output-correctness
+      rubricText: |
+        Score MCP tool output handling from 0 to 100:
+        - 0: Completely failed to use tool outputs
+        - 25: Used outputs but misinterpreted the data
+        - 50: Partially correct interpretation of tool outputs
+        - 75: Mostly correct — minor misuse of returned data
+        - 100: Perfect output handling — correctly interpreted all tool responses
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      humanScore: 30
+      notes: |
+        The candidate adds a `nextPageToken` field at the top level of the
+        tool-call response. The published MCP `tools/call` response spec only
+        defines `content`, `isError`, and `_meta` at the top level; arbitrary
+        top-level fields like `nextPageToken` are not part of the spec and
+        are ignored by spec-compliant clients. Pagination metadata belongs
+        inside the `content` block (e.g., as a structured JSON payload) or
+        in `_meta`. Failure mode: spec-mismatch.
+  gradedBy: "senior-engineer"
+  gradedAt: "2026-05-10"

package/canonical/grader-references/portable-text.yaml

@@ -0,0 +1,48 @@
+# portable-text.yaml — Human-graded reference sample for Portable Text tasks
+#
+# Phase 3 GRAD-03 calibration fixture (Plan 03-02). Top-level
+# `expectedFailureMode` + `dimension` are read by the calibration test in
+# packages/eval/src/grader/__tests__/calibration.test.ts.
+
+- taskId: portable-text-react-marks
+  area: portable-text
+  dimension: doc-coverage
+  expectedFailureMode: outdated-docs
+  response: |
+    To customize the marks renderer in @portabletext/react, you can pass a `serializers` prop:
+
+    ```tsx
+    import { PortableText } from "@portabletext/react"
+
+    const serializers = {
+      marks: {
+        link: ({ children, mark }) => (
+          <a href={mark.href} target="_blank" rel="noopener">{children}</a>
+        ),
+      },
+    }
+
+    export function Body({ value }: { value: any }) {
+      return <PortableText blocks={value} serializers={serializers} />
+    }
+    ```
+  rubrics:
+    - dimension: doc-coverage
+      rubricText: |
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      humanScore: 25
+      notes: |
+        The candidate uses the v2 `serializers` prop and `blocks` prop, both of
+        which were renamed in @portabletext/react v3+ to `components` and
+        `value`. The doc the candidate is following is outdated — current
+        docs document the v3 component-based API. Failure mode:
+        outdated-docs (the doc reflects an older API).
+  gradedBy: "senior-engineer"
+  gradedAt: "2026-05-10"

package/config/rubrics.ts

@@ -11,6 +11,15 @@
 
 import { defineRubrics } from "@sanity/ailf-core"
 
+// Plan 03-02 — per-dimension failure-mode taxonomies stamped onto each
+// template entry below. Source of truth lives in packages/eval/src/grader/;
+// the helper picks the right list by dimension family.
+import { failureModesForDimension } from "../src/grader/index.js"
+// Single source of truth for the wire-format version stamped into the
+// grader-prompt footer (VER-01 D-02). Interpolated below so the
+// announced version cannot drift from the schema's expected value.
+import { graderJudgmentsVersion } from "../src/adapters/grader-outputs/index.js"
+
 export default defineRubrics({
   templates: {
     // ── Core literacy dimensions ────────────────────────────
@@ -25,6 +34,7 @@ export default defineRubrics({
         "100: Fully functional code — works as expected",
       ],
       criteria_label: "Must demonstrate:",
+      failureModes: failureModesForDimension("task-completion"),
     },
     "code-correctness": {
       dimension: "code-correctness",
@@ -37,6 +47,7 @@ export default defineRubrics({
         "100: Follows all best practices, idiomatic implementation",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("code-correctness"),
     },
     "doc-coverage": {
       dimension: "doc-coverage",
@@ -48,6 +59,7 @@ export default defineRubrics({
         "80: Minor gaps — almost everything was documented",
         "100: Complete coverage — all necessary info was in docs",
       ],
+      failureModes: failureModesForDimension("doc-coverage"),
     },
 
     // ── MCP server dimensions ───────────────────────────────
@@ -62,6 +74,7 @@ export default defineRubrics({
         "100: Perfect tool inputs — all parameters correct and well-formed",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("input-validation"),
     },
     "mcp-output-correctness": {
       dimension: "output-correctness",
@@ -74,6 +87,7 @@ export default defineRubrics({
         "100: Perfect output handling — correctly interpreted all tool responses",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("output-correctness"),
     },
     "mcp-error-handling": {
       dimension: "error-handling",
@@ -86,6 +100,7 @@ export default defineRubrics({
         "100: Excellent — handled all errors appropriately with clear messaging",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("error-handling"),
     },
     "mcp-security": {
       dimension: "security",
@@ -98,6 +113,7 @@ export default defineRubrics({
         "100: Perfect security — only used authorized tools with safe inputs",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("security"),
     },
 
     // ── Knowledge probe dimensions ──────────────────────────
@@ -112,6 +128,7 @@ export default defineRubrics({
         "100: Fully correct — all statements are accurate and verifiable",
       ],
       criteria_label: "Verify:",
+      failureModes: failureModesForDimension("factual-correctness"),
     },
     completeness: {
       dimension: "completeness",
@@ -124,6 +141,7 @@ export default defineRubrics({
         "100: Comprehensive — thorough coverage of all important aspects",
       ],
       criteria_label: "Check coverage of:",
+      failureModes: failureModesForDimension("completeness"),
     },
     currency: {
       dimension: "currency",
@@ -136,6 +154,7 @@ export default defineRubrics({
         "100: Fully current — references latest APIs, patterns, and best practices",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("currency"),
     },
 
     // ── Agent harness dimensions ────────────────────────────
@@ -151,6 +170,7 @@ export default defineRubrics({
         "100: Excellent process — optimal tool usage, clear planning, graceful recovery",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("process-quality"),
     },
     "agent-output": {
       dimension: "agent-output",
@@ -163,6 +183,7 @@ export default defineRubrics({
         "100: Excellent output — fully correct, clean, and complete",
       ],
       criteria_label: "Check for:",
+      failureModes: failureModesForDimension("agent-output"),
     },
     "agent-tool-usage": {
       dimension: "tool-usage",
@@ -175,6 +196,7 @@ export default defineRubrics({
         "100: Excellent — optimal tool selection, correct inputs, minimal redundancy",
       ],
       criteria_label: "Evaluate:",
+      failureModes: failureModesForDimension("tool-usage"),
     },
   },
 
@@ -220,6 +242,20 @@ export default defineRubrics({
     "agent-harness": { gold: "agent-harness" },
   },
 
-  footer:
-    'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+  // Phase 3 GRAD-05 (Plan 03-01) — structured GraderJudgment JSON sketch.
+  // Documents the target wire format the grader emits. The strict schema's
+  // GRAD-02 additive fields stay optional in this plan; Plan 03-04 flips
+  // them to required and bumps graderJudgmentsVersion to 1.0.0.
+  footer: `Return ONLY a JSON object with this exact shape:
+{
+  "judgmentId": "<string>",
+  "score": <number 0-100>,
+  "reason": "<explanation, ≤500 chars>",
+  "subJudgments": [{ "criterionId": "<id>", "met": <bool>, "evidence": "<≤280 chars>", "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" } }],
+  "docCitations": [{ "documentId": "<id>", "slug": "<optional slug>", "role": "supports|contradicts|missing|irrelevant", "hallucinated": <bool> }],
+  "failureMode": "<one of the declared modes for this dimension or 'unclassified'>",
+  "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" },
+  "hallucinationCheckedAgainst": ["<doc id>"],
+  "metadata": { "graderModel": "<string>", "graderJudgmentsVersion": "${graderJudgmentsVersion}" }
+}`,
 })

package/dist/_vendor/ailf-core/artifact-registry.d.ts

@@ -79,7 +79,7 @@ export type WriteSource = WritePolicy;
  */
 export type ArtifactTruncationPolicy = "reject" | "trailing-truncate" | "fielded-truncate" | "trial-oversize";
 /** The union of every artifact type known to AILF. */
-export type ArtifactType = "runManifest" | "scoreSummary" | "pipelineResult" | "pipelineContext" | "documentManifest" | "prComment" | "readinessReport" | "reportSnapshot" | "autoComparison" | "gapReport" | "sinkResults" | "callbackRequest" | "callbackResponse" | "configSnapshot" | "evalConfigGenerated" | "comparisonReport" | "discoveryReport" | "failureModes" | "renderedPrompts" | "rawResults" | "testOutputs" | "graderPrompts" | "graderJudgments" | "symbolPreflight" | "traces";
+export type ArtifactType = "runManifest" | "scoreSummary" | "pipelineResult" | "pipelineContext" | "documentManifest" | "prComment" | "readinessReport" | "reportSnapshot" | "autoComparison" | "gapReport" | "sinkResults" | "callbackRequest" | "callbackResponse" | "configSnapshot" | "evalConfigGenerated" | "comparisonReport" | "discoveryReport" | "failureModes" | "renderedPrompts" | "rawResults" | "testOutputs" | "graderPrompts" | "graderJudgments" | "symbolPreflight" | "traces" | "diagnosis" | "perEntryAttribution" | "attributionMeta";
 /**
  * Result of parsing a per-entry key into a sanitized filename component.
  * Success carries the sanitized value; failure carries a reason for 4xx responses.
@@ -114,6 +114,32 @@ export interface ManifestPreviewDeclaration<TPreview = unknown> {
     readonly extract: (entry: unknown) => TPreview;
     readonly capBytes: number;
 }
+/**
+ * Callback signature for `ArtifactDescriptor.objectPath`. Aliased so
+ * external path builders (e.g. `diagnosisPathBuilder`) declare a named
+ * return type rather than re-typing the inline arrow signature.
+ */
+export type ArtifactObjectPath = (runId: RunId, entryKey?: string, version?: string) => string;
+/**
+ * Opt-in marker for the bulk-versioned-with-report-axis carve-out
+ * (Plan 01-03 / ITER2-BLOCKER-01). Any descriptor that needs to combine
+ * `layout: "bulk"` with a `report` axis (e.g. the post-hoc diagnosis
+ * artifact) must set
+ * `pathSafetyMarker: BULK_VERSIONED_WITH_REPORT_AXIS` to opt in to the
+ * carve-out in `assertValidArtifactDescriptor`.
+ *
+ * `Symbol.for(...)` is registry-keyed: it survives minification (the
+ * key is a string literal that minifiers cannot rename) and yields
+ * cross-module-instance equality by spec. This makes it bundler-stable
+ * — a deliberate replacement for any source-text reflection on the
+ * descriptor's `objectPath` closure body.
+ *
+ * The marker alone does NOT trigger the carve-out — the four structural
+ * conditions (bulk + versionedBy + run+report axes) must also hold.
+ * Conversely, the structural conditions alone do NOT trigger the
+ * carve-out without the marker. This is an explicit opt-in.
+ */
+export declare const BULK_VERSIONED_WITH_REPORT_AXIS: unique symbol;
 /**
  * Per-type declaration consumed by writers, signers, and readers.
  *
@@ -157,6 +183,14 @@ export interface ArtifactDescriptor<TEntry = unknown, TPreview = unknown> {
      * When unset, the path is unversioned (legacy behavior).
      */
     readonly versionedBy?: VersionedBy;
+    /**
+     * Optional opt-in marker for the bulk-versioned-with-report-axis
+     * carve-out in `assertValidArtifactDescriptor` (Plan 01-03 /
+     * ITER2-BLOCKER-01). See `BULK_VERSIONED_WITH_REPORT_AXIS` for full
+     * semantics. Descriptors without this marker are subject to the
+     * standard bounded-axis rule.
+     */
+    readonly pathSafetyMarker?: typeof BULK_VERSIONED_WITH_REPORT_AXIS;
     /**
      * Build the GCS object path for this artifact.
      * - bulk: returns `runs/{runId}/{slug}.{ext}`; `entryKey` is ignored.
@@ -165,7 +199,7 @@ export interface ArtifactDescriptor<TEntry = unknown, TPreview = unknown> {
      *   `runs/{runId}/{slug}-{version}.{ext}` for bulk-shaped versioned
      *   descriptors. `entryKey` is ignored on versioned bulk paths.
      */
-    readonly objectPath: (runId: RunId, entryKey?: string, version?: string) => string;
+    readonly objectPath: ArtifactObjectPath;
     /**
      * Build a filename-safe entry key from association values. Only meaningful
      * for `layout === "per-entry"` — bulk descriptors omit it.
@@ -201,6 +235,30 @@ export interface ArtifactDescriptor<TEntry = unknown, TPreview = unknown> {
  * collapsed to an unversioned path.
  */
 export declare function versionedPathBuilder(slug: string, mime: ArtifactMime): (runId: RunId, _entryKey?: string, version?: string) => string;
+/**
+ * Two-axis post-hoc path builder for the diagnosis descriptor (Plan 01-03,
+ * D-09). Filename: `diagnosis-{diagnosisVersion}-{cardSetShortHash}.json`
+ * where `cardSetShortHash = sha256(cardVersion).slice(0, 12)`.
+ *
+ * The compound version argument is `${diagnosisVersion}|${cardVersion}`
+ * to fit the existing 3-arg `objectPath` signature without growing the
+ * shared `ArtifactObjectPath` shape. Callers should use
+ * `encodeDiagnosisPathVersion()` rather than building the compound
+ * string by hand.
+ *
+ * The diagnosis descriptor opts into the bulk-versioned-with-report-axis
+ * carve-out via `pathSafetyMarker: BULK_VERSIONED_WITH_REPORT_AXIS`
+ * (ITER2-BLOCKER-01). The builder itself does not need to know about
+ * the marker — it just builds the path; the carve-out is a registration-
+ * time concern in `assertValidArtifactDescriptor`.
+ */
+export declare function diagnosisPathBuilder(): ArtifactObjectPath;
+/**
+ * Pack the diagnosis descriptor's two version segments into the existing
+ * single-string `version` slot of `ArtifactObjectPath`. Separator is `|`;
+ * `diagnosisVersion` MUST NOT contain `|` (the function rejects that case).
+ */
+export declare function encodeDiagnosisPathVersion(diagnosisVersion: string, cardVersion: string): string;
 /** Test-only reset for the legacy-key warning flag. Not exported publicly. */
 export declare function __resetLegacyTestOutputsWarning(): void;
 /**