@sanity/ailf-studio 1.1.0 → 1.2.0

package/README.md

@@ -138,7 +138,7 @@ registration step needed. There are four ways to execute tasks:
 | **Scheduled pipeline**         | GitHub Actions cron (daily + weekly)                              | All enabled tasks             |
 
 See the
-[CONTRIBUTING_TASKS](https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/CONTRIBUTING_TASKS.md#running-your-task)
+[CONTRIBUTING_TASKS](https://github.com/sanity-labs/ai-literacy-framework/blob/main/docs/contributing-tasks.md#running-your-task)
 guide for the full execution flow and details on each method.
 
 ## Dashboard Views

package/dist/index.d.ts

@@ -26,7 +26,7 @@ import { DocumentRef } from './document-ref.js';
  * `ownership: "repo"` (i.e., active mirrors). Native tasks and
  * already-graduated tasks never see it.
  *
- * @see docs/exec-plans/task-lifecycle/phase-1-ownership.md
+ * @see docs/archive/exec-plans/task-lifecycle/phase-1-ownership.md
  */
 
 declare const GraduateToNativeAction: DocumentActionComponent;
@@ -90,7 +90,7 @@ declare function ReleasePicker(props: StringInputProps): react_jsx_runtime.JSX.E
  *
  * Paired with `SyncStatusBadge` to show sync freshness for active mirrors.
  *
- * @see docs/exec-plans/task-lifecycle/phase-1-ownership.md
+ * @see docs/archive/exec-plans/task-lifecycle/phase-1-ownership.md
  */
 interface GitAuthorInfo {
     gitName?: string;
@@ -130,7 +130,7 @@ declare function MirrorBanner({ origin, ownership }: MirrorBannerProps): react_j
  * Uses the `origin.lastSyncedAt` timestamp that the pipeline sets
  * on every mirror upsert (Phase 5a).
  *
- * @see docs/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
  */
 interface SyncStatusBadgeProps {
     /** ISO 8601 timestamp from origin.lastSyncedAt */
@@ -399,10 +399,25 @@ interface ReportListItem {
 interface ScoreItem {
     codeCorrectness: number;
     docCoverage: number;
+    /**
+     * Generic dimension scores map — all dimensions by camelCase key (0–100).
+     *
+     * Non-literacy modes (agent-harness, mcp-server) store their actual
+     * dimensions here (e.g., agentOutput, toolUsage). Literacy mode may
+     * also populate this alongside the three legacy named fields above.
+     *
+     * UI components should read from this map via `resolveDimensions()` in
+     * `lib/dimensions.ts` rather than hardcoding the three named fields.
+     */
+    dimensions?: Record<string, number>;
     docLift: number;
     /** Sanity documents used for this feature area's evaluation */
     documents?: DocumentRef[];
     feature: string;
+    /** Grouping strategy — "task" for agent-harness, "feature" for literacy */
+    groupType?: "aggregate" | "feature" | "task";
+    /** True when floor > ceiling (docs hurt performance) */
+    negativeDocLift?: boolean;
     taskCompletion: number;
     testCount: number;
     totalScore: number;
@@ -611,24 +626,24 @@ declare function HelpDrawer(): react_jsx_runtime.JSX.Element | null;
  * stat cards, table headers, and comparison views.
  *
  * @see docs/design-docs/scenario-matrix/evaluation-ceiling.md (three reference points)
- * @see docs/ARCHITECTURE.md (scoring model)
+ * @see docs/architecture.md (scoring model)
  */
 declare const GLOSSARY: {
-    readonly overallScore: "A weighted average across all feature areas: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).";
-    readonly docLift: "How much the docs help, compared to the model's training data alone. This is the score with docs minus the score without. Higher is better.";
+    readonly overallScore: "A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).";
+    readonly docLift: "How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.";
     readonly actualScore: "How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.";
     readonly retrievalGap: "The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.";
     readonly infraEfficiency: "What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.";
-    readonly floor: "Score without any documentation. This tells you what the model already knows from its training data.";
+    readonly floor: "Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.";
     readonly ceiling: "Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.";
     readonly actual: "Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.";
     readonly retGap: "Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.";
     readonly efficiency: "What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).";
     readonly invertedRetGap: "⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.";
-    readonly score: "Weighted score for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.";
+    readonly score: "Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.";
     readonly taskCompletion: "Can the LLM implement the requested feature? Graded 0–100.";
     readonly codeCorrectness: "Is the generated code idiomatic, correct, and following best practices? Graded 0–100.";
-    readonly docCoverage: "Did the docs provide the information needed to implement the feature? Graded 0–100.";
+    readonly docCoverage: "Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.";
     readonly tests: "Number of test cases in this feature area.";
     readonly overallDelta: "Change in overall score between the two runs. Positive means the experiment scored higher.";
     readonly actualDelta: "Change in actual (agent-retrieved) score between runs. Positive means agents did better.";
@@ -645,7 +660,7 @@ declare const GLOSSARY: {
     readonly healthWeak: "Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.";
     readonly negativeDocLiftMetric: "Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.";
     readonly weakAreas: "Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.";
-    readonly docsHurt: "Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation is actively misleading the model. These docs need to be rewritten or removed.";
+    readonly docsHurt: "Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.";
     readonly retrievalIssues: "Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.";
     readonly dimWeaknesses: "Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).";
     readonly efficiencyAnomalies: "Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.";
@@ -674,7 +689,7 @@ declare const GLOSSARY: {
     readonly sourceProduction: "Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.";
     readonly sourceBranch: "Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.";
     readonly sourceLocal: "Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.";
-    readonly reportScore: "The overall weighted score for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.";
+    readonly reportScore: "The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.";
     readonly reportMode: "The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.";
     readonly reportTrigger: "What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.";
     readonly modeBaseline: "Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).";
@@ -899,7 +914,7 @@ declare const reportSchema: {
  * repositories. Mirrored tasks have a read-only `origin` block that
  * tracks their source repo provenance.
  *
- * @see docs/CONTRIBUTING_TASKS.md#running-your-task — full execution guide
+ * @see docs/contributing-tasks.md#running-your-task — full execution guide
  * @see docs/design-docs/tasks-as-content.md
  * @see docs/design-docs/tasks-as-content.md#decision-8-domain-specific-assertion-types-not-a-promptfoo-subset
  */
@@ -909,11 +924,11 @@ declare const taskSchema: {
 } & Omit<sanity.DocumentDefinition, "preview"> & {
     preview?: sanity.PreviewConfig<{
         area: string;
-        description: string;
         id: string;
         origin: string;
         ownership: string;
         status: string;
+        title: string;
     }, Record<string, unknown>> | undefined;
 };
 

package/dist/index.js

@@ -117,12 +117,24 @@ function scoreGrade(score) {
 // ../shared/dist/noise-threshold.js
 var NOISE_THRESHOLD = 2;
 
-// ../shared/dist/dimension-names.js
-var DIMENSION_LABELS_KEBAB = {
-  "code-correctness": "Code Correctness",
-  "doc-coverage": "Doc Coverage",
-  "task-completion": "Task Completion"
-};
+// ../shared/dist/eval-modes.js
+var CANONICAL_EVAL_MODES = [
+  "literacy",
+  "mcp-server",
+  "agent-harness",
+  "knowledge-probe",
+  "custom"
+];
+var LEGACY_EVAL_MODE_ALIASES = [
+  "baseline",
+  "agentic",
+  "observed",
+  "full"
+];
+var RAW_EVAL_MODES = [
+  ...CANONICAL_EVAL_MODES,
+  ...LEGACY_EVAL_MODE_ALIASES
+];
 
 // src/types.ts
 function formatPercent(n) {
@@ -2218,8 +2230,8 @@ var taskSchema = defineType5({
     defineField5({
       description: "Human-readable label shown in reports and Studio",
       group: ["main", "all-fields"],
-      name: "description",
-      title: "Description",
+      name: "title",
+      title: "Title",
       type: "string",
       validation: (rule) => rule.required()
     }),
@@ -2280,9 +2292,9 @@ var taskSchema = defineType5({
     defineField5({
       description: "The implementation prompt sent to the evaluated LLM. This is the user-facing request (e.g., 'Write GROQ queries for a blog app...'). Plain text or Markdown.",
       group: ["main", "all-fields"],
-      name: "taskPrompt",
+      name: "promptText",
       rows: 10,
-      title: "Task Prompt",
+      title: "Prompt Text",
       type: "text",
       validation: (rule) => rule.required()
     }),
@@ -2292,7 +2304,7 @@ var taskSchema = defineType5({
     defineField5({
       description: "Feature area this task belongs to. Used for score aggregation and --area CLI filtering.",
       group: ["main", "all-fields"],
-      name: "featureArea",
+      name: "area",
       title: "Feature Area",
       to: [{ type: "ailf.featureArea" }],
       type: "reference",
@@ -2312,7 +2324,7 @@ var taskSchema = defineType5({
     defineField5({
       description: "Documentation that the LLM should use to complete this task. These become the gold-standard doc context injected in baseline mode. Each entry is either a single article document or a content release (whose articles are all included automatically).",
       group: ["main", "all-fields"],
-      name: "canonicalDocs",
+      name: "contextDocs",
       of: [
         {
           components: {
@@ -2437,7 +2449,7 @@ var taskSchema = defineType5({
     defineField5({
       description: "Grading criteria for evaluating the LLM's output. At least one assertion is required. Use llm-rubric with a template for scored evaluation, or value-based assertions for exact checks.",
       group: ["main", "all-fields"],
-      name: "assert",
+      name: "assertions",
       of: [
         {
           fields: [
@@ -2585,8 +2597,8 @@ var taskSchema = defineType5({
           type: "boolean"
         }),
         defineField5({
-          description: 'Rubric mode for baseline. "abbreviated" uses a shorter rubric, "full" uses the same rubric as gold, "none" skips rubric grading.',
-          initialValue: "abbreviated",
+          description: 'Rubric mode for baseline. "full" uses the same rubric as gold, "abbreviated" uses a shorter rubric, "none" skips rubric grading.',
+          initialValue: "full",
           name: "rubric",
           options: {
             list: [
@@ -2916,11 +2928,11 @@ var taskSchema = defineType5({
   preview: {
     prepare({
       area,
-      description,
       id,
       origin,
       ownership,
-      status
+      status,
+      title
     }) {
       const taskId = id !== null && typeof id === "object" && "current" in id ? id.current : void 0;
       const isMirror = ownership === "repo" || !ownership && origin !== null && typeof origin === "object" && "repo" in origin;
@@ -2944,16 +2956,16 @@ var taskSchema = defineType5({
       const statusIcon = status === "archived" ? "\u{1F4E6} " : status === "draft" ? "\u{1F9EA} " : status === "paused" ? "\u23F8\uFE0F " : "";
       return {
         subtitle: `${areaStr}${typeof taskId === "string" ? taskId : ""}${syncInfo}`,
-        title: `${prefix}${statusIcon}${typeof description === "string" ? description : "Task"}`
+        title: `${prefix}${statusIcon}${typeof title === "string" ? title : "Task"}`
       };
     },
     select: {
-      area: "featureArea.areaId",
-      description: "description",
+      area: "area.areaId",
       id: "id",
       origin: "origin",
       ownership: "ownership",
-      status: "status"
+      status: "status",
+      title: "title"
     }
   },
   // Document-level read-only when owned by a repo.
@@ -3138,16 +3150,6 @@ function useHelp() {
 
 // src/generated/help-content.ts
 var HELP_TOPICS = [
-  {
-    "id": "eval-modes",
-    "title": "Evaluation Modes",
-    "body": "The framework runs each task under different conditions to isolate what helps AI\nagents succeed:\n\n**Full** (default) \u2014 Runs baseline + agentic sequentially in a single pipeline\nrun, then merges results into a **three-layer decomposition**: floor (model\nknowledge alone), ceiling (gold-standard docs injected), and actual\n(agent-retrieved docs). The retrieval gap (ceiling \u2212 actual) measures how much\ndocumentation quality is lost to discoverability.\n\n**Baseline** \u2014 Docs are fetched from Sanity via GROQ and injected directly into\nthe prompt. Each task runs twice: once with docs, once without. The LLM never\ntouches the internet. The score difference is the **Doc Lift** \u2014 how much\ndocumentation helps vs. training data alone.\n\n**Agentic** \u2014 The LLM gets no docs in the prompt. Instead it receives tools\n(`web_search`, `fetch_page`, `list_docs`) and must find relevant documentation\non its own. Two sub-modes test different discovery strategies:\n\n- **Naive** \u2014 web search \u2192 reads pages via Jina Reader (simulates a human)\n- **Optimized** \u2014 starts from `llms.txt` \u2192 fetches `.md` versions directly\n\n**Observed** \u2014 Same as baseline (docs injected in prompt), but the LLM's HTTP\nactivity is recorded through an instrumented proxy (`InstrumentedProvider`).\nCaptures which API endpoints the model references without changing its behavior.",
-    "source": "docs/ARCHITECTURE.md",
-    "related": [
-      "scoring-model",
-      "three-layer"
-    ]
-  },
   {
     "id": "negative-doc-lift",
     "title": "When Docs Hurt: Negative Doc Lift",
@@ -3305,7 +3307,7 @@ Click into any report for the full breakdown: per-area scores, diagnostics, and
   {
     "id": "scoring-model",
     "title": "Understanding Scores",
-    "body": "## The three dimensions\n\nEvery evaluation task is scored on three dimensions, each graded 0\u2013100:\n\n- **Task Completion (50% weight)** \u2014 Can the AI implement the requested feature?\n  Does the output actually do what was asked?\n- **Code Correctness (25% weight)** \u2014 Is the generated code idiomatic, correct,\n  and following best practices?\n- **Doc Coverage (25% weight)** \u2014 Did the documentation provide the information\n  needed to implement the feature?\n\n## How the overall score is calculated\n\nThe three dimensions combine into a single **AI Literacy Score** per task:\n\n```\nTotal = Task Completion \xD7 0.50 + Code Correctness \xD7 0.25 + Doc Coverage \xD7 0.25\n```\n\nThis weighted composite produces a score from 0\u2013100. Scores are then averaged\nacross all tasks in a feature area to produce a **per-area score**, and across\nall areas to produce the **overall score**.\n\n## What the numbers mean\n\n| Score range  | Interpretation                                                    |\n| ------------ | ----------------------------------------------------------------- |\n| **80\u2013100**   | Docs are working well \u2014 AI agents produce correct implementations |\n| **70\u201379**    | Needs attention \u2014 there may be gaps in specific dimensions        |\n| **Below 70** | Weak \u2014 AI agents consistently struggle with this area             |\n\n## Ceiling decomposition (baseline mode)\n\nWhen running in baseline mode, each task is evaluated twice \u2014 with and without\ndocumentation. This produces:\n\n- **Floor score** \u2014 Score without docs (what the model knows from training data\n  alone)\n- **Ceiling score** \u2014 Score with gold-standard docs injected directly into the\n  prompt\n- **Doc Lift** \u2014 Ceiling minus floor. Positive means docs help; negative means\n  docs hurt.\n- **Doc Quality Gap** \u2014 100 minus ceiling. Room for documentation improvement.\n\n## Three-layer decomposition (full mode)\n\nFull mode adds a third measurement \u2014 what happens when AI agents find docs on\ntheir own:\n\n- **Floor** \u2014 No docs (parametric knowledge only)\n- **Ceiling** \u2014 Gold-standard docs injected (best the docs can do)\n- **Actual** \u2014 Agent-retrieved docs (real-world performance)\n- **Retrieval Gap** \u2014 Ceiling minus actual (quality lost to findability)\n- **Infrastructure Efficiency** \u2014 Actual \xF7 ceiling (what fraction of doc quality\n  reaches agents)\n\n## Cost tracking\n\nEach evaluation also tracks token costs:\n\n- **Provider cost** \u2014 Token usage for generating implementations\n- **Grader cost** \u2014 Token usage for the grading model's assessments\n- **Total cost** \u2014 Both combined, reported in the score summary",
+    "body": "## The three dimensions\n\nEvery evaluation task is scored on three dimensions, each graded 0\u2013100:\n\n- **Task Completion (50% weight)** \u2014 Can the AI implement the requested feature?\n  Does the output actually do what was asked?\n- **Code Correctness (25% weight)** \u2014 Is the generated code idiomatic, correct,\n  and following best practices?\n- **Doc Coverage (25% weight)** \u2014 Did the documentation provide the information\n  needed to implement the feature?\n\n## How the overall score is calculated\n\nThe three dimensions combine into a single **AI Literacy Score** per task using\nnamed scoring profiles from `config/rubrics.yaml`:\n\n```\nGold (with docs):    Total = Task \xD7 0.50 + Code \xD7 0.25 + Docs \xD7 0.25\nBaseline (no docs):  Total = Task \xD7 0.60 + Code \xD7 0.40\n```\n\nThe gold profile includes all three dimensions. The baseline profile excludes\nDoc Coverage because it is undefined when no documentation is provided. This\nensures Doc Lift (ceiling \u2212 floor) is a clean structural measurement of\ndocumentation value.\n\nThe weighted composite produces a score from 0\u2013100. Scores are then averaged\nacross all tasks in a feature area to produce a **per-area score**, and across\nall areas to produce the **overall score**.\n\n## What the numbers mean\n\n| Score range  | Interpretation                                                    |\n| ------------ | ----------------------------------------------------------------- |\n| **80\u2013100**   | Docs are working well \u2014 AI agents produce correct implementations |\n| **70\u201379**    | Needs attention \u2014 there may be gaps in specific dimensions        |\n| **Below 70** | Weak \u2014 AI agents consistently struggle with this area             |\n\n## Ceiling decomposition (baseline mode)\n\nWhen running in baseline mode, each task is evaluated twice \u2014 with and without\ndocumentation. This produces:\n\n- **Floor score** \u2014 Score without docs (what the model knows from training data\n  alone)\n- **Ceiling score** \u2014 Score with gold-standard docs injected directly into the\n  prompt\n- **Doc Lift** \u2014 Ceiling minus floor. Positive means docs help; negative means\n  docs hurt.\n- **Doc Quality Gap** \u2014 100 minus ceiling. Room for documentation improvement.\n\n## Three-layer decomposition (full mode)\n\nFull mode adds a third measurement \u2014 what happens when AI agents find docs on\ntheir own:\n\n- **Floor** \u2014 No docs (parametric knowledge only)\n- **Ceiling** \u2014 Gold-standard docs injected (best the docs can do)\n- **Actual** \u2014 Agent-retrieved docs (real-world performance)\n- **Retrieval Gap** \u2014 Ceiling minus actual (quality lost to findability)\n- **Infrastructure Efficiency** \u2014 Actual \xF7 ceiling (what fraction of doc quality\n  reaches agents)\n\n## Cost tracking\n\nEach evaluation also tracks token costs:\n\n- **Provider cost** \u2014 Token usage for generating implementations\n- **Grader cost** \u2014 Token usage for the grading model's assessments\n- **Total cost** \u2014 Both combined, reported in the score summary",
     "source": "docs/help/scoring-model.md",
     "related": [
       "three-layer",
@@ -3327,17 +3329,27 @@ Click into any report for the full breakdown: per-area scores, diagnostics, and
   {
     "id": "how-agents-work",
     "title": "How AI Agents Find Documentation",
-    "body": "Understanding how popular AI coding agents retrieve and use documentation is\ncentral to the ai-literacy-framework evaluation framework. This document\nexplains the mechanisms used by common agents and how our test modes simulate\nthem.\n\n## The Documentation Access Problem\n\nWhen a developer asks an AI coding assistant \"Set up a Sanity Studio with a\ncustom blog schema,\" the agent needs to find and read the relevant Sanity\ndocumentation. But different agents do this in fundamentally different ways, and\nthose differences directly impact the quality of the response.\n\nThe framework measures this impact through four evaluation modes: **full**\n(default \u2014 runs baseline + agentic together), **baseline** (docs in prompt),\n**agentic** (tool-calling with real web access), and **observed** (instrumented\nsingle-call).\n\n## How Popular Agents Work\n\n### Claude Code (Anthropic)\n\nClaude Code has built-in tools including `WebSearchTool` and `WebFetchTool`.\nWhen a user asks a Sanity question:\n\n1. The model decides whether to search the web\n2. If so, it calls `WebSearchTool` with a query string\n3. Search results come back as structured data (titles, URLs, snippets)\n4. The model may call `WebFetchTool` to read specific pages\n5. The fetched content is returned as **rendered text** \u2014 Claude Code's fetch\n   tool handles JavaScript rendering internally, so even SPA pages return\n   readable content\n6. The model synthesizes the fetched docs with its training data to produce an\n   answer\n\n**Key characteristic**: Claude Code sees the web as rendered, readable text. It\ndoesn't get raw HTML soup. But it also doesn't know about agent-friendly\nendpoints like `.md` files or `llms.txt` \u2014 it fetches the same HTML pages a\nbrowser would load.\n\n### ChatGPT (OpenAI)\n\nChatGPT's browsing capability uses Bing search under the hood:\n\n1. The model decides to search (users can also explicitly ask it to browse)\n2. It searches via Bing, getting ranked results\n3. It can \"click\" on results to read page content\n4. Pages are rendered server-side and returned as text\n5. The model reads relevant sections and synthesizes an answer\n\n**Key characteristic**: ChatGPT's browsing is similar to Claude Code \u2014 it gets\nrendered content. The URLs visited are returned in citations. It also has no\nawareness of `llms.txt` or `.md` endpoints.\n\n### Cursor\n\nCursor takes a different approach:\n\n1. It maintains a pre-built index of popular documentation sites (`@docs`)\n2. Users can manually add documentation sources\n3. It also has web search capability for unknown topics\n4. Codebase context is injected automatically from the project\n\n**Key characteristic**: Cursor's `@docs` feature means it may have indexed\nSanity docs already, but the index may be outdated. For unknown topics, it falls\nback to web search like the other agents.\n\n### GitHub Copilot\n\nCopilot primarily relies on:\n\n1. The model's training data (parametric knowledge)\n2. Codebase context from the current project\n3. Bing search for `@workspace` queries in newer versions\n\n**Key characteristic**: Copilot historically had no web access, relying entirely\non training data. Newer versions can search, but the experience is similar to\nChatGPT.\n\n## The JavaScript SPA Problem\n\nSanity's documentation site (`sanity.io/docs`) is built with Next.js \u2014 a\nJavaScript single-page application. When an agent makes a raw HTTP request:\n\n```\nGET https://www.sanity.io/docs/schema-types\n\u2192 Returns ~125KB of HTML that is mostly:\n  - <script> tags for Next.js bundles\n  - React hydration data\n  - Navigation chrome\n  - Very little actual documentation text\n```\n\nReal agents handle this differently than a raw `fetch()`:\n\n| Agent         | Raw fetch? | Gets readable content? | How?                               |\n| ------------- | ---------- | ---------------------- | ---------------------------------- |\n| Claude Code   | No         | Yes                    | Built-in rendering in WebFetchTool |\n| ChatGPT       | No         | Yes                    | Server-side rendering via Bing     |\n| Cursor        | No         | Yes                    | Pre-built doc index                |\n| Raw `fetch()` | Yes        | **No**                 | Gets HTML soup                     |\n\nThis is why the agentic provider uses **Jina Reader** (`r.jina.ai`) as a\nreadability proxy in \"naive\" mode \u2014 it simulates the rendering capability that\nreal agents have built in.\n\n## Sanity's Agent-Friendly Endpoints\n\nSanity has invested in making their documentation accessible to AI agents\nthrough special endpoints:\n\n### `.md` endpoint\n\nAppending `.md` to any docs URL returns pure markdown:\n\n```\nGET https://www.sanity.io/docs/schema-types.md\nContent-Type: text/markdown;charset=UTF-8\n\n# Schema types\nSchema types are used to define the shape of your content...\n```\n\nThis returns **clean markdown** \u2014 no HTML, no JavaScript, no navigation. Just\nthe documentation content. Typical response size: 2-10KB (vs 125KB for the HTML\npage).\n\n### `llms.txt`\n\nSanity provides an `llms.txt` file at `https://www.sanity.io/docs/llms.txt` \u2014 a\nstructured listing of all documentation pages designed for AI agent consumption:\n\n```\n# Sanity\n## Docs\n- [Manage Sanity with code](https://www.sanity.io/docs/blueprints)\n- [Introduction](https://www.sanity.io/docs/blueprints-introduction)\n- [Deploy with GitHub Actions](https://www.sanity.io/docs/blueprints/blueprint-action)\n...\n```\n\nThis follows the emerging [llms.txt standard](https://llmstxt.org/) \u2014 a\nmachine-readable table of contents that tells agents what documentation is\navailable and where to find it.\n\n### Impact on Agent Performance\n\nOur smoke tests show the dramatic difference these endpoints make:\n\n| Metric           | Naive Agent (Jina) | Optimized Agent (.md) |\n| ---------------- | ------------------ | --------------------- |\n| Result           | \u274C FAIL            | \u2705 PASS               |\n| Latency          | 57.9s              | 15.2s (3.8\xD7 faster)   |\n| Bytes downloaded | 108 KB             | 59 KB (45% less)      |\n| Total requests   | 9                  | 6 (33% fewer)         |\n| Search queries   | 3                  | 0 (used llms.txt)     |\n\nThe optimized agent skips search entirely \u2014 it calls `list_docs(\"sanity.io\")` to\nget the `llms.txt` table of contents, identifies the relevant pages, and fetches\nthem directly as `.md`. No search round-trips, no proxy overhead, no content\ncleaning needed.\n\n## How Test Modes Map to Real Agents\n\n| Mode                       | Config                          | Simulates                  | Documentation Access                                                                                                       |\n| -------------------------- | ------------------------------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- |\n| `eval` (baseline)          | `promptfooconfig.yaml`          | No agent \u2014 docs in prompt  | Docs are injected directly into the prompt context, with and without variants                                              |\n| `eval:observed`            | `promptfooconfig.observed.yaml` | Non-agentic API call       | Single OpenAI API call, records the HTTP request but model doesn't browse                                                  |\n| `eval:agentic` (naive)     | `promptfooconfig.agentic.yaml`  | Claude Code, ChatGPT today | Model has `web_search` + `fetch_page` tools; pages fetched via Jina Reader (simulates JS rendering)                        |\n| `eval:agentic` (optimized) | `promptfooconfig.agentic.yaml`  | Ideal future agent         | Model has `web_search` + `fetch_page` + `list_docs` tools; fetches `.md` endpoints directly, uses `llms.txt` for discovery |\n\n### Why Both Naive and Optimized?\n\nThe comparison between naive and optimized modes answers a critical business\nquestion:\n\n> **\"How much does investing in agent-friendly documentation endpoints (`.md`,\n> `llms.txt`) improve the AI developer experience?\"**\n\nIf the optimized agent significantly outperforms the naive agent, it validates\nthe investment in these endpoints. The data from our tests provides concrete\nevidence for this.\n\n## Limitations of the Simulation\n\nWhile the agentic provider faithfully simulates agent behavior, there are\ndifferences from real agents:\n\n1. **Search quality**: We use DuckDuckGo via Jina as a search fallback. Real\n   agents use Bing (ChatGPT) or their own search (Claude Code). Search result\n   quality varies.\n\n2. **Page rendering**: Jina Reader is a good proxy for JS rendering, but may\n   produce slightly different output than what Claude Code or ChatGPT's internal\n   renderers produce.\n\n3. **Context window management**: Real agents have sophisticated context\n   management \u2014 they may truncate long pages, summarize content, or use sliding\n   windows. Our provider returns content up to a fixed limit (12KB).\n\n4. **Codebase context**: Real agents (especially Cursor and Copilot) inject the\n   developer's current codebase into context. Our eval doesn't simulate this \u2014\n   it only tests documentation retrieval.\n\n5. **Multi-turn interactions**: A real developer might have a conversation with\n   their agent, refining the request. Our eval tests single-turn interactions.\n\n## Future Directions\n\n- **Subprocess agents**: Run actual `claude` CLI or other agent CLIs as\n  subprocesses and capture their real network traffic via HTTP proxy\n  interception\n- **Anthropic/OpenAI native tools**: Use Claude's built-in `web_search` tool or\n  OpenAI's `web_search_preview` in the Responses API for more faithful\n  simulation\n- **Agent-specific configs**: Separate configs that match each agent's exact\n  tool set and system prompt\n- **Codebase context injection**: Simulate a project workspace to test how\n  agents combine docs with code context",
+    "body": "Understanding how popular AI coding agents retrieve and use documentation is\ncentral to the ai-literacy-framework evaluation framework. This document\nexplains the mechanisms used by common agents and how our test modes simulate\nthem.\n\n## The Documentation Access Problem\n\nWhen a developer asks an AI coding assistant \"Set up a Sanity Studio with a\ncustom blog schema,\" the agent needs to find and read the relevant Sanity\ndocumentation. But different agents do this in fundamentally different ways, and\nthose differences directly impact the quality of the response.\n\nThe framework measures this impact through four evaluation modes: **full**\n(default \u2014 runs baseline + agentic together), **baseline** (docs in prompt),\n**agentic** (tool-calling with real web access), and **observed** (instrumented\nsingle-call).\n\n## How Popular Agents Work\n\n### Claude Code (Anthropic)\n\nClaude Code has built-in tools including `WebSearchTool` and `WebFetchTool`.\nWhen a user asks a Sanity question:\n\n1. The model decides whether to search the web\n2. If so, it calls `WebSearchTool` with a query string\n3. Search results come back as structured data (titles, URLs, snippets)\n4. The model may call `WebFetchTool` to read specific pages\n5. The fetched content is returned as **rendered text** \u2014 Claude Code's fetch\n   tool handles JavaScript rendering internally, so even SPA pages return\n   readable content\n6. The model synthesizes the fetched docs with its training data to produce an\n   answer\n\n**Key characteristic**: Claude Code sees the web as rendered, readable text. It\ndoesn't get raw HTML soup. But it also doesn't know about agent-friendly\nendpoints like `.md` files or `llms.txt` \u2014 it fetches the same HTML pages a\nbrowser would load.\n\n### ChatGPT (OpenAI)\n\nChatGPT's browsing capability uses Bing search under the hood:\n\n1. The model decides to search (users can also explicitly ask it to browse)\n2. It searches via Bing, getting ranked results\n3. It can \"click\" on results to read page content\n4. Pages are rendered server-side and returned as text\n5. The model reads relevant sections and synthesizes an answer\n\n**Key characteristic**: ChatGPT's browsing is similar to Claude Code \u2014 it gets\nrendered content. The URLs visited are returned in citations. It also has no\nawareness of `llms.txt` or `.md` endpoints.\n\n### Cursor\n\nCursor takes a different approach:\n\n1. It maintains a pre-built index of popular documentation sites (`@docs`)\n2. Users can manually add documentation sources\n3. It also has web search capability for unknown topics\n4. Codebase context is injected automatically from the project\n\n**Key characteristic**: Cursor's `@docs` feature means it may have indexed\nSanity docs already, but the index may be outdated. For unknown topics, it falls\nback to web search like the other agents.\n\n### GitHub Copilot\n\nCopilot primarily relies on:\n\n1. The model's training data (parametric knowledge)\n2. Codebase context from the current project\n3. Bing search for `@workspace` queries in newer versions\n\n**Key characteristic**: Copilot historically had no web access, relying entirely\non training data. Newer versions can search, but the experience is similar to\nChatGPT.\n\n## The JavaScript SPA Problem\n\nSanity's documentation site (`sanity.io/docs`) is built with Next.js \u2014 a\nJavaScript single-page application. When an agent makes a raw HTTP request:\n\n```\nGET https://www.sanity.io/docs/schema-types\n\u2192 Returns ~125KB of HTML that is mostly:\n  - <script> tags for Next.js bundles\n  - React hydration data\n  - Navigation chrome\n  - Very little actual documentation text\n```\n\nReal agents handle this differently than a raw `fetch()`:\n\n| Agent         | Raw fetch? | Gets readable content? | How?                               |\n| ------------- | ---------- | ---------------------- | ---------------------------------- |\n| Claude Code   | No         | Yes                    | Built-in rendering in WebFetchTool |\n| ChatGPT       | No         | Yes                    | Server-side rendering via Bing     |\n| Cursor        | No         | Yes                    | Pre-built doc index                |\n| Raw `fetch()` | Yes        | **No**                 | Gets HTML soup                     |\n\nThis is why the agentic provider uses **Jina Reader** (`r.jina.ai`) as a\nreadability proxy in \"naive\" mode \u2014 it simulates the rendering capability that\nreal agents have built in.\n\n## Sanity's Agent-Friendly Endpoints\n\nSanity has invested in making their documentation accessible to AI agents\nthrough special endpoints:\n\n### `.md` endpoint\n\nAppending `.md` to any docs URL returns pure markdown:\n\n```\nGET https://www.sanity.io/docs/schema-types.md\nContent-Type: text/markdown;charset=UTF-8\n\n# Schema types\nSchema types are used to define the shape of your content...\n```\n\nThis returns **clean markdown** \u2014 no HTML, no JavaScript, no navigation. Just\nthe documentation content. Typical response size: 2-10KB (vs 125KB for the HTML\npage).\n\n### `llms.txt`\n\nSanity provides an `llms.txt` file at `https://www.sanity.io/docs/llms.txt` \u2014 a\nstructured listing of all documentation pages designed for AI agent consumption:\n\n```\n# Sanity\n## Docs\n- [Manage Sanity with code](https://www.sanity.io/docs/blueprints)\n- [Introduction](https://www.sanity.io/docs/blueprints-introduction)\n- [Deploy with GitHub Actions](https://www.sanity.io/docs/blueprints/blueprint-action)\n...\n```\n\nThis follows the emerging [llms.txt standard](https://llmstxt.org/) \u2014 a\nmachine-readable table of contents that tells agents what documentation is\navailable and where to find it.\n\n### Impact on Agent Performance\n\nOur smoke tests show the dramatic difference these endpoints make:\n\n| Metric           | Naive Agent (Jina) | Optimized Agent (.md) |\n| ---------------- | ------------------ | --------------------- |\n| Result           | \u274C FAIL            | \u2705 PASS               |\n| Latency          | 57.9s              | 15.2s (3.8\xD7 faster)   |\n| Bytes downloaded | 108 KB             | 59 KB (45% less)      |\n| Total requests   | 9                  | 6 (33% fewer)         |\n| Search queries   | 3                  | 0 (used llms.txt)     |\n\nThe optimized agent skips search entirely \u2014 it calls `list_docs(\"sanity.io\")` to\nget the `llms.txt` table of contents, identifies the relevant pages, and fetches\nthem directly as `.md`. No search round-trips, no proxy overhead, no content\ncleaning needed.\n\n## How Test Modes Map to Real Agents\n\n| Mode                       | Config                          | Simulates                  | Documentation Access                                                                                                       |\n| -------------------------- | ------------------------------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------- |\n| `eval` (baseline)          | `promptfooconfig.yaml`          | No agent \u2014 docs in prompt  | Docs are injected directly into the prompt context, with and without variants                                              |\n| `eval:observed`            | `promptfooconfig.observed.yaml` | Non-agentic API call       | Single OpenAI API call, records the HTTP request but model doesn't browse                                                  |\n| `eval:agentic` (naive)     | `promptfooconfig.agentic.yaml`  | Claude Code, ChatGPT today | Model has `web_search` + `fetch_page` tools; pages fetched via Jina Reader (simulates JS rendering)                        |\n| `eval:agentic` (optimized) | `promptfooconfig.agentic.yaml`  | Ideal future agent         | Model has `web_search` + `fetch_page` + `list_docs` tools; fetches `.md` endpoints directly, uses `llms.txt` for discovery |\n| `agent-harness`            | compiled via compiler           | Real agent in sandbox      | Agent harness mode evaluates real agent behavior in a sandboxed environment (Docker, tempdir, git-worktree)                |\n\n### Why Both Naive and Optimized?\n\nThe comparison between naive and optimized modes answers a critical business\nquestion:\n\n> **\"How much does investing in agent-friendly documentation endpoints (`.md`,\n> `llms.txt`) improve the AI developer experience?\"**\n\nIf the optimized agent significantly outperforms the naive agent, it validates\nthe investment in these endpoints. The data from our tests provides concrete\nevidence for this.\n\n## Limitations of the Simulation\n\nWhile the agentic provider faithfully simulates agent behavior, there are\ndifferences from real agents:\n\n1. **Search quality**: We use DuckDuckGo via Jina as a search fallback. Real\n   agents use Bing (ChatGPT) or their own search (Claude Code). Search result\n   quality varies.\n\n2. **Page rendering**: Jina Reader is a good proxy for JS rendering, but may\n   produce slightly different output than what Claude Code or ChatGPT's internal\n   renderers produce.\n\n3. **Context window management**: Real agents have sophisticated context\n   management \u2014 they may truncate long pages, summarize content, or use sliding\n   windows. Our provider returns content up to a fixed limit (12KB).\n\n4. **Codebase context**: Real agents (especially Cursor and Copilot) inject the\n   developer's current codebase into context. Our eval doesn't simulate this \u2014\n   it only tests documentation retrieval.\n\n5. **Multi-turn interactions**: A real developer might have a conversation with\n   their agent, refining the request. Our eval tests single-turn interactions.\n\n## Future Directions\n\nThe architecture overhaul (Phase 4: agent harness mode) addressed several of\nthese goals \u2014 real agents can now be evaluated in sandboxed environments with\nfixture provisioning, tool manifests, and process-quality scoring. Remaining\ndirections:\n\n- **Subprocess agents** _(partially addressed by agent harness mode)_: The\n  harness supports running agents via entrypoints in Docker, tempdir, or\n  git-worktree sandboxes. Real `claude` CLI or other agent CLIs can be\n  configured as harness entrypoints.\n- **Anthropic/OpenAI native tools**: Use Claude's built-in `web_search` tool or\n  OpenAI's `web_search_preview` in the Responses API for more faithful\n  simulation of the agentic mode\n- **Agent-specific configs**: The compiler's mode handler system makes it\n  straightforward to create per-agent configurations\n- **Codebase context injection** _(partially addressed by fixture\n  provisioning)_: The agent harness fixture provisioner can inject project\n  workspaces, dependency manifests, and code contexts into sandbox environments",
     "source": "docs/how-agents-work.md",
     "related": [
       "eval-modes",
       "retrieval-gap"
     ]
   },
+  {
+    "id": "eval-modes",
+    "title": "Evaluation Modes",
+    "body": '> **This guide is for:** Anyone using AILF who wants to understand what modes\n> exist and when to use each one.\n\nAILF supports five canonical evaluation modes. Each mode measures a different\naspect of AI tool effectiveness.\n\n## Mode overview\n\n| Mode                | What it measures                                     | When to use it                        |\n| ------------------- | ---------------------------------------------------- | ------------------------------------- |\n| **literacy**        | Can AI agents implement features using your docs?    | Testing documentation quality         |\n| **mcp-server**      | Can an LLM correctly use your MCP server\'s tools?    | Testing MCP server implementations    |\n| **knowledge-probe** | What does the model know without any docs?           | Measuring baseline model knowledge    |\n| **agent-harness**   | Can an autonomous agent complete tasks in a sandbox? | Testing agent capabilities end-to-end |\n| **custom**          | Whatever you define                                  | Building your own evaluation type     |\n\n## Choosing a mode\n\n```\nWhat do you want to test?\n    \u2502\n    \u251C\u2500\u2500 "Are our docs helping AI agents?" \u2500\u2500\u2500\u2500\u2500\u2500\u2192 literacy\n    \u251C\u2500\u2500 "Does our MCP server work correctly?" \u2500\u2500\u2192 mcp-server\n    \u251C\u2500\u2500 "What does the model already know?" \u2500\u2500\u2500\u2500\u2192 knowledge-probe\n    \u251C\u2500\u2500 "Can an agent complete real tasks?" \u2500\u2500\u2500\u2500\u2192 agent-harness\n    \u2514\u2500\u2500 "Something else entirely" \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2192 custom\n```',
+    "source": "docs/modes.md",
+    "related": [
+      "scoring-model",
+      "three-layer"
+    ]
+  },
   {
     "id": "glossary",
     "title": "Glossary",
-    "body": "**Overall Score**\n: A weighted average across all feature areas: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).\n\n**Doc Lift**\n: How much the docs help, compared to the model's training data alone. This is the score with docs minus the score without. Higher is better.\n\n**Actual Score**\n: How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.\n\n**Retrieval Gap**\n: The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.\n\n**Infra Efficiency**\n: What percentage of the docs' potential quality actually reaches agents (actual \xF7 ceiling). 100% means agents find and use all relevant docs perfectly.\n\n**Floor**\n: Score without any documentation. This tells you what the model already knows from its training data.\n\n**Ceiling**\n: Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.\n\n**Actual**\n: Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.\n\n**Ret Gap**\n: Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.\n\n**Efficiency**\n: What fraction of the docs' quality reaches agents in practice (actual \xF7 ceiling, shown as a percentage).\n\n**Inverted Ret Gap**\n: \u26A0\uFE0F Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.\n\n**Score**\n: Weighted score for this feature area: Task Completion \xD7 50% + Code Correctness \xD7 25% + Doc Coverage \xD7 25%.\n\n**Task Completion**\n: Can the LLM implement the requested feature? Graded 0\u2013100.\n\n**Code Correctness**\n: Is the generated code idiomatic, correct, and following best practices? Graded 0\u2013100.\n\n**Doc Coverage**\n: Did the docs provide the information needed to implement the feature? Graded 0\u2013100.\n\n**Tests**\n: Number of test cases in this feature area.\n\n**Overall Delta**\n: Change in overall score between the two runs. Positive means the experiment scored higher.\n\n**Actual Delta**\n: Change in actual (agent-retrieved) score between runs. Positive means agents did better.\n\n**Ret Gap Delta**\n: Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.\n\n**Efficiency Delta**\n: Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.\n\n**Baseline**\n: The reference run you're comparing against.\n\n**Experiment**\n: The new run you're evaluating.\n\n**Delta**\n: Difference between experiment and baseline. Positive means improvement, negative means regression.\n\n**Change**\n: Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).\n\n**Low Scoring Judgments**\n: The grading model's explanations for tests that scored below 70/100.\n\n**Judgment Reason**\n: The grading model's natural language explanation of what went wrong.\n\n**Health Strong**\n: Feature areas scoring 80 or above. The docs are working well for these features \u2014 AI agents produce correct, complete implementations.\n\n**Health Attention**\n: Feature areas scoring 70\u201379. These are okay but could be improved \u2014 there may be gaps in specific dimensions like doc coverage or code correctness.\n\n**Health Weak**\n: Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.\n\n**Negative Doc Lift Metric**\n: Number of areas where the documentation actually hurts AI performance \u2014 the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.\n\n**Weak Areas**\n: Feature areas where the overall score is below 70. These need the most attention \u2014 low scores mean AI agents consistently struggle to implement these features.\n\n**Docs Hurt**\n: Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation is actively misleading the model. These docs need to be rewritten or removed.\n\n**Retrieval Issues**\n: Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.\n\n**Dim Weaknesses**\n: Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most \u2014 task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).\n\n**Efficiency Anomalies**\n: Areas where agent efficiency exceeds 100% \u2014 meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.\n\n**Doc Lift Wins**\n: Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.\n\n**Retrieval Excellence**\n: Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.\n\n**Model Breakdown**\n: Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently \u2014 useful for spotting models that struggle with specific feature areas.\n\n**Strengths**\n: What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.\n\n**Recommendations**\n: Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.\n\n**Total Potential Lift**\n: Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate \u2014 each gap targets the median of non-bottlenecked dimensions, not 100.\n\n**Failure Mode**\n: The type of documentation problem: missing-docs (functionality not covered), incorrect-docs (factual errors), outdated-docs (stale API/patterns), or poor-structure (hard to find/understand).\n\n**Estimated Lift**\n: Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.\n\n**Confidence**\n: How confident the classifier is in this diagnosis. High = strong keyword + structural signal agreement. Medium = partial agreement. Low = weak signals only.\n\n**Agent Behavior Overview**\n: How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.\n\n**Search Queries**\n: The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.\n\n**Doc Slugs Visited**\n: Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.\n\n**External Domains**\n: Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.\n\n**Avg Doc Pages Visited**\n: Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.\n\n**Avg Searches Performed**\n: Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.\n\n**Avg Network Time Ms**\n: Average time spent on network requests per test. Includes page fetches, search queries, and API calls.\n\n**Total Requests**\n: Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.\n\n**Total Bytes Downloaded**\n: Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.\n\n**Dim Task Completion**\n: Change in task completion between runs. Positive means implementations are more complete.\n\n**Dim Code Correctness**\n: Change in code correctness between runs. Positive means better code quality.\n\n**Dim Doc Coverage**\n: Change in doc coverage between runs. Positive means the docs are providing more useful information.\n\n**Area Delta**\n: Score change for this area compared to the previous evaluation run.\n\n**Source Production**\n: Production source \u2014 docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.\n\n**Source Branch**\n: Branch source \u2014 docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.\n\n**Source Local**\n: Local source \u2014 docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.\n\n**Report Score**\n: The overall weighted score for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.\n\n**Report Mode**\n: The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.\n\n**Report Trigger**\n: What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.\n\n**Mode Baseline**\n: Baseline mode \u2014 tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).\n\n**Mode Full**\n: Full mode \u2014 runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.\n\n**Mode Agentic**\n: Agentic mode \u2014 the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?\n\n**Mode Observed**\n: Observed mode \u2014 records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.\n\n**Mode Debug**\n: Debug mode \u2014 a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.\n\n**Trigger Manual**\n: Manually triggered \u2014 someone ran the evaluation pipeline by hand, either locally or via the Studio UI.\n\n**Trigger Ci**\n: CI-triggered \u2014 the evaluation ran automatically as part of a pull request or merge pipeline.\n\n**Trigger Schedule**\n: Scheduled \u2014 the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.\n\n**Trigger Webhook**\n: Webhook-triggered \u2014 a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.\n\n**Trigger Cross Repo**\n: Cross-repo \u2014 triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
+    "body": "**Overall Score**\n: A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).\n\n**Doc Lift**\n: How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.\n\n**Actual Score**\n: How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.\n\n**Retrieval Gap**\n: The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.\n\n**Infra Efficiency**\n: What percentage of the docs' potential quality actually reaches agents (actual \xF7 ceiling). 100% means agents find and use all relevant docs perfectly.\n\n**Floor**\n: Output-quality composite without documentation \u2014 Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.\n\n**Ceiling**\n: Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.\n\n**Actual**\n: Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.\n\n**Ret Gap**\n: Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.\n\n**Efficiency**\n: What fraction of the docs' quality reaches agents in practice (actual \xF7 ceiling, shown as a percentage).\n\n**Inverted Ret Gap**\n: \u26A0\uFE0F Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.\n\n**Score**\n: Ceiling composite for this feature area: Task Completion \xD7 50% + Code Correctness \xD7 25% + Doc Coverage \xD7 25%.\n\n**Task Completion**\n: Can the LLM implement the requested feature? Graded 0\u2013100.\n\n**Code Correctness**\n: Is the generated code idiomatic, correct, and following best practices? Graded 0\u2013100.\n\n**Doc Coverage**\n: Did the docs provide the information needed to implement the feature? Graded 0\u2013100. This dimension only contributes to the ceiling composite (with docs) \u2014 it's excluded from the floor composite because it's undefined without documentation.\n\n**Tests**\n: Number of test cases in this feature area.\n\n**Overall Delta**\n: Change in overall score between the two runs. Positive means the experiment scored higher.\n\n**Actual Delta**\n: Change in actual (agent-retrieved) score between runs. Positive means agents did better.\n\n**Ret Gap Delta**\n: Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.\n\n**Efficiency Delta**\n: Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.\n\n**Baseline**\n: The reference run you're comparing against.\n\n**Experiment**\n: The new run you're evaluating.\n\n**Delta**\n: Difference between experiment and baseline. Positive means improvement, negative means regression.\n\n**Change**\n: Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).\n\n**Low Scoring Judgments**\n: The grading model's explanations for tests that scored below 70/100.\n\n**Judgment Reason**\n: The grading model's natural language explanation of what went wrong.\n\n**Health Strong**\n: Feature areas scoring 80 or above. The docs are working well for these features \u2014 AI agents produce correct, complete implementations.\n\n**Health Attention**\n: Feature areas scoring 70\u201379. These are okay but could be improved \u2014 there may be gaps in specific dimensions like doc coverage or code correctness.\n\n**Health Weak**\n: Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.\n\n**Negative Doc Lift Metric**\n: Number of areas where the documentation actually hurts AI performance \u2014 the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.\n\n**Weak Areas**\n: Feature areas where the overall score is below 70. These need the most attention \u2014 low scores mean AI agents consistently struggle to implement these features.\n\n**Docs Hurt**\n: Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.\n\n**Retrieval Issues**\n: Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.\n\n**Dim Weaknesses**\n: Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most \u2014 task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).\n\n**Efficiency Anomalies**\n: Areas where agent efficiency exceeds 100% \u2014 meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.\n\n**Doc Lift Wins**\n: Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.\n\n**Retrieval Excellence**\n: Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.\n\n**Model Breakdown**\n: Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently \u2014 useful for spotting models that struggle with specific feature areas.\n\n**Strengths**\n: What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.\n\n**Recommendations**\n: Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.\n\n**Total Potential Lift**\n: Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate \u2014 each gap targets the median of non-bottlenecked dimensions, not 100.\n\n**Failure Mode**\n: The type of documentation problem: missing-docs (functionality not covered), incorrect-docs (factual errors), outdated-docs (stale API/patterns), or poor-structure (hard to find/understand).\n\n**Estimated Lift**\n: Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.\n\n**Confidence**\n: How confident the classifier is in this diagnosis. High = strong keyword + structural signal agreement. Medium = partial agreement. Low = weak signals only.\n\n**Agent Behavior Overview**\n: How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.\n\n**Search Queries**\n: The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.\n\n**Doc Slugs Visited**\n: Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.\n\n**External Domains**\n: Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.\n\n**Avg Doc Pages Visited**\n: Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.\n\n**Avg Searches Performed**\n: Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.\n\n**Avg Network Time Ms**\n: Average time spent on network requests per test. Includes page fetches, search queries, and API calls.\n\n**Total Requests**\n: Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.\n\n**Total Bytes Downloaded**\n: Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.\n\n**Dim Task Completion**\n: Change in task completion between runs. Positive means implementations are more complete.\n\n**Dim Code Correctness**\n: Change in code correctness between runs. Positive means better code quality.\n\n**Dim Doc Coverage**\n: Change in doc coverage between runs. Positive means the docs are providing more useful information.\n\n**Area Delta**\n: Score change for this area compared to the previous evaluation run.\n\n**Source Production**\n: Production source \u2014 docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.\n\n**Source Branch**\n: Branch source \u2014 docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.\n\n**Source Local**\n: Local source \u2014 docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.\n\n**Report Score**\n: The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.\n\n**Report Mode**\n: The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.\n\n**Report Trigger**\n: What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.\n\n**Mode Baseline**\n: Baseline mode \u2014 tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).\n\n**Mode Full**\n: Full mode \u2014 runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.\n\n**Mode Agentic**\n: Agentic mode \u2014 the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?\n\n**Mode Observed**\n: Observed mode \u2014 records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.\n\n**Mode Debug**\n: Debug mode \u2014 a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.\n\n**Trigger Manual**\n: Manually triggered \u2014 someone ran the evaluation pipeline by hand, either locally or via the Studio UI.\n\n**Trigger Ci**\n: CI-triggered \u2014 the evaluation ran automatically as part of a pull request or merge pipeline.\n\n**Trigger Schedule**\n: Scheduled \u2014 the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.\n\n**Trigger Webhook**\n: Webhook-triggered \u2014 a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.\n\n**Trigger Cross Repo**\n: Cross-repo \u2014 triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
     "source": "packages/studio/src/glossary.ts",
     "tags": [
       "reference",
@@ -3386,23 +3398,23 @@ import { useClient as useClient3 } from "sanity";
 // src/glossary.ts
 var GLOSSARY = {
   // -- Overview stats -------------------------------------------------------
-  overallScore: "A weighted average across all feature areas: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).",
-  docLift: "How much the docs help, compared to the model's training data alone. This is the score with docs minus the score without. Higher is better.",
+  overallScore: "A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).",
+  docLift: "How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.",
   actualScore: "How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.",
   retrievalGap: "The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.",
   infraEfficiency: "What percentage of the docs' potential quality actually reaches agents (actual \xF7 ceiling). 100% means agents find and use all relevant docs perfectly.",
   // -- Three-layer decomposition columns ------------------------------------
-  floor: "Score without any documentation. This tells you what the model already knows from its training data.",
+  floor: "Output-quality composite without documentation \u2014 Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.",
   ceiling: "Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.",
   actual: "Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.",
   retGap: "Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.",
   efficiency: "What fraction of the docs' quality reaches agents in practice (actual \xF7 ceiling, shown as a percentage).",
   invertedRetGap: "\u26A0\uFE0F Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.",
   // -- Per-area score columns -----------------------------------------------
-  score: "Weighted score for this feature area: Task Completion \xD7 50% + Code Correctness \xD7 25% + Doc Coverage \xD7 25%.",
+  score: "Ceiling composite for this feature area: Task Completion \xD7 50% + Code Correctness \xD7 25% + Doc Coverage \xD7 25%.",
   taskCompletion: "Can the LLM implement the requested feature? Graded 0\u2013100.",
   codeCorrectness: "Is the generated code idiomatic, correct, and following best practices? Graded 0\u2013100.",
-  docCoverage: "Did the docs provide the information needed to implement the feature? Graded 0\u2013100.",
+  docCoverage: "Did the docs provide the information needed to implement the feature? Graded 0\u2013100. This dimension only contributes to the ceiling composite (with docs) \u2014 it's excluded from the floor composite because it's undefined without documentation.",
   tests: "Number of test cases in this feature area.",
   // -- Comparison deltas ----------------------------------------------------
   overallDelta: "Change in overall score between the two runs. Positive means the experiment scored higher.",
@@ -3423,7 +3435,7 @@ var GLOSSARY = {
   healthWeak: "Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.",
   negativeDocLiftMetric: "Number of areas where the documentation actually hurts AI performance \u2014 the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.",
   weakAreas: "Feature areas where the overall score is below 70. These need the most attention \u2014 low scores mean AI agents consistently struggle to implement these features.",
-  docsHurt: "Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation is actively misleading the model. These docs need to be rewritten or removed.",
+  docsHurt: "Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.",
   retrievalIssues: "Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.",
   dimWeaknesses: "Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most \u2014 task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).",
   efficiencyAnomalies: "Areas where agent efficiency exceeds 100% \u2014 meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.",
@@ -3460,7 +3472,7 @@ var GLOSSARY = {
   sourceBranch: "Branch source \u2014 docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.",
   sourceLocal: "Local source \u2014 docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.",
   // -- Report list columns ----------------------------------------------------
-  reportScore: "The overall weighted score for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.",
+  reportScore: "The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.",
   reportMode: "The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.",
   reportTrigger: "What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.",
   // -- Mode values -----------------------------------------------------------
@@ -3477,6 +3489,82 @@ var GLOSSARY = {
   triggerCrossRepo: "Cross-repo \u2014 triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks."
 };
 
+// src/lib/dimensions.ts
+var DIMENSION_TOOLTIPS = {
+  agentOutput: "Quality and completeness of the agent's output. Graded 0\u2013100.",
+  assertionPassRate: "Fraction of structural assertions that passed. Graded 0\u2013100.",
+  codeCorrectness: GLOSSARY.codeCorrectness,
+  docCoverage: GLOSSARY.docCoverage,
+  taskCompletion: GLOSSARY.taskCompletion,
+  toolUsage: "How effectively the agent used available tools (file read/write, shell, etc.). Graded 0\u2013100."
+};
+function dimensionLabel(key) {
+  return key.replace(/([A-Z])/g, " $1").replace(/^./, (c) => c.toUpperCase()).trim();
+}
+var SHORT_LABELS = {
+  agentOutput: "Agent",
+  assertionPassRate: "Assert",
+  codeCorrectness: "Code",
+  docCoverage: "Docs",
+  taskCompletion: "Task",
+  toolUsage: "Tools"
+};
+function dimensionShortLabel(key) {
+  return SHORT_LABELS[key] ?? dimensionLabel(key).split(" ")[0];
+}
+var LITERACY_DEFAULTS = [
+  { key: "taskCompletion", field: "taskCompletion" },
+  { key: "codeCorrectness", field: "codeCorrectness" },
+  { key: "docCoverage", field: "docCoverage" }
+];
+function resolveDimensions(score) {
+  const dims = score.dimensions;
+  if (dims && Object.keys(dims).length > 0) {
+    return Object.entries(dims).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => ({
+      key,
+      label: dimensionLabel(key),
+      tooltip: DIMENSION_TOOLTIPS[key] ?? "",
+      value
+    }));
+  }
+  return LITERACY_DEFAULTS.map(({ key, field }) => ({
+    key,
+    label: dimensionLabel(key),
+    tooltip: DIMENSION_TOOLTIPS[key] ?? "",
+    value: score[field] ?? 0
+  }));
+}
+function isLiteracyMode(mode) {
+  if (mode) return mode === "literacy";
+  return true;
+}
+function collectDimensionKeys(scores) {
+  const keys = /* @__PURE__ */ new Set();
+  for (const score of scores) {
+    if (score.dimensions && Object.keys(score.dimensions).length > 0) {
+      for (const key of Object.keys(score.dimensions)) {
+        keys.add(key);
+      }
+    } else {
+      for (const { key } of LITERACY_DEFAULTS) {
+        keys.add(key);
+      }
+    }
+  }
+  return [...keys].sort();
+}
+function getDimensionValue(score, key) {
+  if (score.dimensions?.[key] != null) return score.dimensions[key];
+  const legacyMap = {
+    codeCorrectness: "codeCorrectness",
+    docCoverage: "docCoverage",
+    taskCompletion: "taskCompletion"
+  };
+  const field = legacyMap[key];
+  if (field) return score[field] ?? 0;
+  return 0;
+}
+
 // src/lib/comparison.ts
 function scoreMap(summary) {
   return new Map(summary.scores.map((s) => [s.feature, s]));
@@ -3543,14 +3631,26 @@ function computeThreeLayerDeltas(baseline, experiment) {
 function computeDimensionDeltas(baseline, experiment) {
   const bMap = scoreMap(baseline);
   const eMap = scoreMap(experiment);
+  const allDimKeys = collectDimensionKeys([
+    ...baseline.scores,
+    ...experiment.scores
+  ]);
   return allAreas(baseline, experiment).map((area) => {
     const bItem = bMap.get(area);
     const eItem = eMap.get(area);
+    const deltas = {};
+    for (const key of allDimKeys) {
+      const bVal = bItem ? getDimensionValue(bItem, key) : 0;
+      const eVal = eItem ? getDimensionValue(eItem, key) : 0;
+      deltas[key] = eVal - bVal;
+    }
     return {
       area,
-      taskDelta: (eItem?.taskCompletion ?? 0) - (bItem?.taskCompletion ?? 0),
-      codeDelta: (eItem?.codeCorrectness ?? 0) - (bItem?.codeCorrectness ?? 0),
-      docDelta: (eItem?.docCoverage ?? 0) - (bItem?.docCoverage ?? 0)
+      deltas,
+      // Legacy accessors for backward compatibility
+      taskDelta: deltas.taskCompletion ?? 0,
+      codeDelta: deltas.codeCorrectness ?? 0,
+      docDelta: deltas.docCoverage ?? 0
     };
   });
 }
@@ -3593,7 +3693,10 @@ var latestReportsQuery = (
       taskCompletion,
       codeCorrectness,
       docCoverage,
+      dimensions,
       docLift,
+      groupType,
+      negativeDocLift,
       testCount,
       actualScore,
       retrievalGap,
@@ -6033,6 +6136,7 @@ function DiagnosticsOverview({
   overall,
   comparison,
   durationMs,
+  mode,
   totalTests
 }) {
   if (scores.length === 0) return null;
@@ -6043,6 +6147,7 @@ function DiagnosticsOverview({
   const weak = scores.filter((s) => s.totalScore < SCORE_CAUTION);
   const negativeDocLiftCount = scores.filter((s) => s.docLift < 0).length;
   const hasAgenticData = overall.avgActualScore != null;
+  const showDocMetrics = !mode || mode === "literacy";
   const improved = comparison?.improved ?? [];
   const regressed = comparison?.regressed ?? [];
   const unchanged = comparison?.unchanged ?? [];
@@ -6057,7 +6162,7 @@ function DiagnosticsOverview({
             style: {
               display: "grid",
               gap: 12,
-              gridTemplateColumns: "repeat(3, 1fr)"
+              gridTemplateColumns: showDocMetrics ? "repeat(3, 1fr)" : "repeat(2, 1fr)"
             },
             children: [
               /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.overallScore, children: /* @__PURE__ */ jsx25(
@@ -6066,11 +6171,11 @@ function DiagnosticsOverview({
                   delta: comparison?.deltas.overall,
                   label: "AVG SCORE",
                   sentiment: scoreSentiment(overall.avgScore),
-                  subtitle: "With-docs ceiling",
+                  subtitle: showDocMetrics ? "With-docs ceiling" : "Weighted composite",
                   value: Math.round(overall.avgScore)
                 }
               ) }),
-              /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.docLift, children: /* @__PURE__ */ jsx25(
+              showDocMetrics && /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.docLift, children: /* @__PURE__ */ jsx25(
                 ScoreCard,
                 {
                   delta: comparison?.deltas.docLift,
@@ -6080,7 +6185,7 @@ function DiagnosticsOverview({
                   value: Math.round(overall.avgDocLift)
                 }
               ) }),
-              /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.floor, children: /* @__PURE__ */ jsx25(
+              showDocMetrics ? /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.floor, children: /* @__PURE__ */ jsx25(
                 ScoreCard,
                 {
                   label: "FLOOR",
@@ -6088,6 +6193,13 @@ function DiagnosticsOverview({
                   subtitle: "Without docs baseline",
                   value: Math.round(overall.avgFloorScore ?? 0)
                 }
+              ) }) : /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.tests, children: /* @__PURE__ */ jsx25(
+                ScoreCard,
+                {
+                  label: "TESTS",
+                  subtitle: "Total test cases",
+                  value: totalTests ?? 0
+                }
               ) })
             ]
           }
@@ -6098,10 +6210,10 @@ function DiagnosticsOverview({
             style: {
               display: "grid",
               gap: 12,
-              gridTemplateColumns: "repeat(3, 1fr)"
+              gridTemplateColumns: showDocMetrics ? "repeat(3, 1fr)" : "repeat(2, 1fr)"
             },
             children: [
-              /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.negativeDocLiftMetric, children: /* @__PURE__ */ jsx25(
+              showDocMetrics && /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.negativeDocLiftMetric, children: /* @__PURE__ */ jsx25(
                 MetricCard,
                 {
                   label: "Negative Doc Lift",
@@ -6109,7 +6221,7 @@ function DiagnosticsOverview({
                   value: `${negativeDocLiftCount} area${negativeDocLiftCount === 1 ? "" : "s"}`
                 }
               ) }),
-              /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.tests, children: /* @__PURE__ */ jsx25(MetricCard, { label: "Tests", value: String(totalTests ?? 0) }) }),
+              showDocMetrics && /* @__PURE__ */ jsx25(HoverTip, { display: "block", text: GLOSSARY.tests, children: /* @__PURE__ */ jsx25(MetricCard, { label: "Tests", value: String(totalTests ?? 0) }) }),
               durationMs != null && durationMs > 0 ? /* @__PURE__ */ jsx25(
                 HoverTip,
                 {
@@ -6649,7 +6761,9 @@ function groupByArea(judgments) {
   }
   return [...byArea.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([area, js]) => [area, js.sort((a, b) => a.score - b.score)]);
 }
-var DIMENSION_LABELS = DIMENSION_LABELS_KEBAB;
+function dimensionLabel2(dim) {
+  return dim.split("-").map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(" ");
+}
 function JudgmentList({
   focus,
   judgments,
@@ -6741,7 +6855,7 @@ function JudgmentCard({
   const [expanded, setExpanded] = useState12(focused);
   const cardRef = useRef5(null);
   const toast = useToast2();
-  const dimLabel = DIMENSION_LABELS[judgment.dimension] ?? judgment.dimension;
+  const dimLabel = dimensionLabel2(judgment.dimension);
   const sep = judgment.taskId.indexOf(" - ");
   const taskName = sep > 0 ? judgment.taskId.substring(sep + 3) : judgment.taskId;
   useEffect8(() => {
@@ -7762,22 +7876,40 @@ import React3, {
 import { WarningOutlineIcon as WarningOutlineIcon2 } from "@sanity/icons";
 import { Box as Box19, Flex as Flex22, Stack as Stack25, Text as Text30 } from "@sanity/ui";
 import { Fragment as Fragment11, jsx as jsx41, jsxs as jsxs29 } from "react/jsx-runtime";
+var DIMENSION_TOOLTIPS2 = {
+  agentOutput: "Quality and completeness of the agent's output. Graded 0\u2013100.",
+  assertionPassRate: "Fraction of structural assertions that passed. Graded 0\u2013100.",
+  codeCorrectness: GLOSSARY.codeCorrectness,
+  docCoverage: GLOSSARY.docCoverage,
+  taskCompletion: GLOSSARY.taskCompletion,
+  toolUsage: "How effectively the agent used available tools (file read/write, shell, etc.). Graded 0\u2013100."
+};
 function tableTier2(width) {
   if (width >= 900) return "full";
   if (width >= 600) return "compact";
   return "narrow";
 }
-function gridColumns(tier, hasActual) {
+function gridColumns(tier, dimCount, showLift, hasActual) {
+  const dims = Array.from({ length: dimCount }, () => "1fr").join(" ");
   switch (tier) {
-    case "full":
-      return hasActual ? "120px 1fr 1fr 1fr 1fr 80px 72px 72px" : "120px 1fr 1fr 1fr 1fr 80px 72px";
-    case "compact":
-      return "96px 1fr 1fr 1fr 1fr 80px";
+    case "full": {
+      const parts = ["120px", "1fr", dims];
+      if (showLift) parts.push("80px");
+      if (showLift) parts.push("72px");
+      if (hasActual) parts.push("72px");
+      return parts.join(" ");
+    }
+    case "compact": {
+      const parts = ["96px", "1fr", dims];
+      if (showLift) parts.push("80px");
+      return parts.join(" ");
+    }
     case "narrow":
-      return "56px 1fr 1fr 1fr 1fr";
+      return `56px 1fr ${dims}`;
   }
 }
 function AreaScoresGrid({
+  mode,
   scores,
   perArea,
   perModel
@@ -7788,6 +7920,8 @@ function AreaScoresGrid({
     () => scores.some((s) => s.actualScore != null),
     [scores]
   );
+  const showLift = isLiteracyMode(mode);
+  const dimKeys = useMemo6(() => collectDimensionKeys(scores), [scores]);
   const [sortField, setSortField] = useState17("score");
   const [sortDir, setSortDir] = useState17("desc");
   const handleSort = useCallback22(
@@ -7809,16 +7943,10 @@ function AreaScoresGrid({
           return (a.totalScore - b.totalScore) * dir;
         case "area":
           return a.feature.localeCompare(b.feature) * dir;
-        case "task":
-          return (a.taskCompletion - b.taskCompletion) * dir;
-        case "code":
-          return (a.codeCorrectness - b.codeCorrectness) * dir;
-        case "docs":
-          return (a.docCoverage - b.docCoverage) * dir;
         case "lift":
           return (a.docLift - b.docLift) * dir;
         default:
-          return 0;
+          return (getDimensionValue(a, sortField) - getDimensionValue(b, sortField)) * dir;
       }
     });
   }, [scores, sortField, sortDir]);
@@ -7848,7 +7976,12 @@ function AreaScoresGrid({
           borderBottom: "1px solid var(--card-border-color)",
           display: "grid",
           gap: "0 12px",
-          gridTemplateColumns: gridColumns(tier, hasActual),
+          gridTemplateColumns: gridColumns(
+            tier,
+            dimKeys.length,
+            showLift,
+            hasActual
+          ),
           padding: "12px 16px 8px"
         },
         children: [
@@ -7859,7 +7992,7 @@ function AreaScoresGrid({
               direction: sortDir,
               label: "Score",
               onClick: () => handleSort("score"),
-              tooltip: `${GLOSSARY.score} This is the ceiling score \u2014 with gold-standard docs injected.`
+              tooltip: GLOSSARY.score
             }
           ),
           /* @__PURE__ */ jsx41(
@@ -7871,37 +8004,18 @@ function AreaScoresGrid({
               onClick: () => handleSort("area")
             }
           ),
-          /* @__PURE__ */ jsx41(
+          dimKeys.map((key) => /* @__PURE__ */ jsx41(
             ColHeader3,
             {
-              active: sortField === "task",
+              active: sortField === key,
               direction: sortDir,
-              label: "Task",
-              onClick: () => handleSort("task"),
-              tooltip: GLOSSARY.taskCompletion
-            }
-          ),
-          /* @__PURE__ */ jsx41(
-            ColHeader3,
-            {
-              active: sortField === "code",
-              direction: sortDir,
-              label: "Code",
-              onClick: () => handleSort("code"),
-              tooltip: GLOSSARY.codeCorrectness
-            }
-          ),
-          /* @__PURE__ */ jsx41(
-            ColHeader3,
-            {
-              active: sortField === "docs",
-              direction: sortDir,
-              label: "Docs",
-              onClick: () => handleSort("docs"),
-              tooltip: GLOSSARY.docCoverage
-            }
-          ),
-          tier !== "narrow" && /* @__PURE__ */ jsx41(
+              label: dimensionShortLabel(key),
+              onClick: () => handleSort(key),
+              tooltip: DIMENSION_TOOLTIPS2[key] ?? `${dimensionShortLabel(key)} dimension score (0\u2013100).`
+            },
+            key
+          )),
+          tier !== "narrow" && showLift && /* @__PURE__ */ jsx41(
             ColHeader3,
             {
               active: sortField === "lift",
@@ -7911,7 +8025,7 @@ function AreaScoresGrid({
               tooltip: GLOSSARY.docLift
             }
           ),
-          tier === "full" && /* @__PURE__ */ jsx41(ColHeader3, { label: "Floor", tooltip: GLOSSARY.floor }),
+          tier === "full" && showLift && /* @__PURE__ */ jsx41(ColHeader3, { label: "Floor", tooltip: GLOSSARY.floor }),
           tier === "full" && hasActual && /* @__PURE__ */ jsx41(ColHeader3, { label: "Actual", tooltip: GLOSSARY.actualScore })
         ]
       }
@@ -7922,15 +8036,19 @@ function AreaScoresGrid({
         {
           area,
           delta: perArea?.[area.feature],
+          dimKeys,
           hasActual,
+          showLift,
           tier
         }
       ),
       modelScoresByFeature && /* @__PURE__ */ jsx41(
         ModelSubRows,
         {
+          dimKeys,
           hasActual,
           models: modelScoresByFeature.get(area.feature),
+          showLift,
           tier
         }
       )
@@ -7938,26 +8056,32 @@ function AreaScoresGrid({
   ] });
 }
 function ModelSubRows({
+  dimKeys,
   hasActual,
   models,
+  showLift,
   tier
 }) {
   if (!models || models.length === 0) return null;
   return /* @__PURE__ */ jsx41(Fragment11, { children: models.map((entry) => /* @__PURE__ */ jsx41(
     ModelRow,
     {
+      dimKeys,
       hasActual,
       label: entry.label,
       scores: entry.scores,
+      showLift,
       tier
     },
     entry.label
   )) });
 }
 function ModelRow({
+  dimKeys,
   hasActual,
   label,
   scores,
+  showLift,
   tier
 }) {
   const isNarrow = tier === "narrow";
@@ -7970,7 +8094,12 @@ function ModelRow({
         borderBottom: "1px solid var(--card-border-color)",
         display: "grid",
         gap: "0 12px",
-        gridTemplateColumns: gridColumns(tier, hasActual),
+        gridTemplateColumns: gridColumns(
+          tier,
+          dimKeys.length,
+          showLift,
+          hasActual
+        ),
         padding: isNarrow ? "6px 12px 6px 20px" : "6px 16px 6px 28px"
       },
       children: [
@@ -7987,34 +8116,17 @@ function ModelRow({
           }
         ) }),
         /* @__PURE__ */ jsx41(Flex22, { align: "center", gap: 2, children: /* @__PURE__ */ jsx41(Text30, { muted: true, size: 1, children: label }) }),
-        /* @__PURE__ */ jsx41(
-          DimCell,
-          {
-            area: label,
-            dim: "Task Completion",
-            size: "small",
-            value: scores.taskCompletion
-          }
-        ),
-        /* @__PURE__ */ jsx41(
+        dimKeys.map((key) => /* @__PURE__ */ jsx41(
           DimCell,
           {
             area: label,
-            dim: "Code Correctness",
+            dim: dimensionLabel(key),
             size: "small",
-            value: scores.codeCorrectness
-          }
-        ),
-        /* @__PURE__ */ jsx41(
-          DimCell,
-          {
-            area: label,
-            dim: "Doc Coverage",
-            size: "small",
-            value: scores.docCoverage
-          }
-        ),
-        !isNarrow && /* @__PURE__ */ jsxs29(
+            value: getDimensionValue(scores, key)
+          },
+          key
+        )),
+        !isNarrow && showLift && /* @__PURE__ */ jsxs29(
           Text30,
           {
             size: 1,
@@ -8029,7 +8141,7 @@ function ModelRow({
             ]
           }
         ),
-        tier === "full" && /* @__PURE__ */ jsx41(
+        tier === "full" && showLift && /* @__PURE__ */ jsx41(
           Text30,
           {
             muted: true,
@@ -8057,7 +8169,9 @@ function ModelRow({
 function AreaRow({
   area,
   delta,
+  dimKeys,
   hasActual,
+  showLift,
   tier
 }) {
   const isNarrow = tier === "narrow";
@@ -8069,7 +8183,12 @@ function AreaRow({
         borderBottom: "1px solid var(--card-border-color)",
         display: "grid",
         gap: "0 12px",
-        gridTemplateColumns: gridColumns(tier, hasActual),
+        gridTemplateColumns: gridColumns(
+          tier,
+          dimKeys.length,
+          showLift,
+          hasActual
+        ),
         padding: isNarrow ? "8px 12px" : "10px 16px"
       },
       children: [
@@ -8079,7 +8198,7 @@ function AreaRow({
             {
               text: /* @__PURE__ */ jsxs29(Text30, { size: 2, style: { lineHeight: 1.5 }, children: [
                 /* @__PURE__ */ jsx41("span", { style: { fontWeight: 600 }, children: area.feature }),
-                " ceiling score:",
+                " score:",
                 " ",
                 /* @__PURE__ */ jsx41(
                   "span",
@@ -8095,8 +8214,7 @@ function AreaRow({
                 /* @__PURE__ */ jsx41("span", { style: { color: "var(--card-muted-fg-color)" }, children: "/100" }),
                 ".",
                 " ",
-                GLOSSARY.score,
-                " This is the ceiling \u2014 with gold-standard docs injected."
+                GLOSSARY.score
               ] }),
               children: /* @__PURE__ */ jsx41(
                 "div",
@@ -8123,7 +8241,7 @@ function AreaRow({
         ] }),
         /* @__PURE__ */ jsxs29(Flex22, { align: "center", gap: 2, wrap: "wrap", children: [
           /* @__PURE__ */ jsx41(Text30, { size: 2, weight: "medium", children: area.feature }),
-          area.negativeDocLift && /* @__PURE__ */ jsx41(HoverTip, { text: GLOSSARY.docsHurt, children: /* @__PURE__ */ jsx41(
+          area.negativeDocLift && showLift && /* @__PURE__ */ jsx41(HoverTip, { text: GLOSSARY.docsHurt, children: /* @__PURE__ */ jsx41(
             "span",
             {
               style: {
@@ -8140,31 +8258,16 @@ function AreaRow({
             }
           ) })
         ] }),
-        /* @__PURE__ */ jsx41(
-          DimCell,
-          {
-            area: area.feature,
-            dim: "Task Completion",
-            value: area.taskCompletion
-          }
-        ),
-        /* @__PURE__ */ jsx41(
-          DimCell,
-          {
-            area: area.feature,
-            dim: "Code Correctness",
-            value: area.codeCorrectness
-          }
-        ),
-        /* @__PURE__ */ jsx41(
+        dimKeys.map((key) => /* @__PURE__ */ jsx41(
           DimCell,
           {
             area: area.feature,
-            dim: "Doc Coverage",
-            value: area.docCoverage
-          }
-        ),
-        !isNarrow && /* @__PURE__ */ jsx41(
+            dim: dimensionLabel(key),
+            value: getDimensionValue(area, key)
+          },
+          key
+        )),
+        !isNarrow && showLift && /* @__PURE__ */ jsx41(
           HoverTip,
           {
             text: /* @__PURE__ */ jsxs29(Text30, { size: 2, style: { lineHeight: 1.5 }, children: [
@@ -8206,7 +8309,7 @@ function AreaRow({
             )
           }
         ),
-        tier === "full" && /* @__PURE__ */ jsx41(
+        tier === "full" && showLift && /* @__PURE__ */ jsx41(
           Text30,
           {
             muted: true,
@@ -8243,11 +8346,10 @@ function DimCell({
   size = "normal",
   value
 }) {
-  const glossary = {
-    "Task Completion": GLOSSARY.taskCompletion,
-    "Code Correctness": GLOSSARY.codeCorrectness,
-    "Doc Coverage": GLOSSARY.docCoverage
-  };
+  const dimKey = Object.keys(DIMENSION_TOOLTIPS2).find(
+    (k) => dimensionLabel(k) === dim
+  );
+  const tooltip = dimKey ? DIMENSION_TOOLTIPS2[dimKey] : "";
   const textSize = size === "small" ? 0 : 1;
   const barHeight = size === "small" ? 3 : 4;
   return /* @__PURE__ */ jsx41(
@@ -8274,7 +8376,7 @@ function DimCell({
         /* @__PURE__ */ jsx41("span", { style: { color: "var(--card-muted-fg-color)" }, children: "/100" }),
         ".",
         " ",
-        glossary[dim] ?? ""
+        tooltip
       ] }),
       children: /* @__PURE__ */ jsxs29(Stack25, { space: 1, style: { width: "100%" }, children: [
         /* @__PURE__ */ jsx41(
@@ -8509,6 +8611,7 @@ function useModelSelection({
 // src/components/report-detail/StrengthsList.tsx
 import { jsx as jsx43, jsxs as jsxs31 } from "react/jsx-runtime";
 function StrengthsList({
+  mode,
   scores,
   comparison,
   perModel
@@ -8548,6 +8651,7 @@ function StrengthsList({
       /* @__PURE__ */ jsx43(
         AreaScoresGrid,
         {
+          mode,
           perArea: comparison?.deltas?.perArea,
           perModel: expandedPerModel,
           scores: displayedScores
@@ -8613,6 +8717,7 @@ import {
 import { Box as Box21, Flex as Flex25, Stack as Stack27, Text as Text33 } from "@sanity/ui";
 import { jsx as jsx44, jsxs as jsxs32 } from "react/jsx-runtime";
 function WeaknessesList({
+  mode,
   scores,
   comparison,
   perModel
@@ -8658,6 +8763,7 @@ function WeaknessesList({
       /* @__PURE__ */ jsx44(
         AreaScoresGrid,
         {
+          mode,
           perArea,
           perModel: expandedPerModel,
           scores: weakAreas
@@ -8947,40 +9053,22 @@ function dimTip(area, dim, score, description) {
   ] });
 }
 function getDimensionWeaknesses(area) {
+  const dims = resolveDimensions(area);
   const result = [];
-  if (area.taskCompletion < DIMENSION_WEAKNESS)
-    result.push({
-      dimension: "Task Completion",
-      tip: dimTip(
-        area.feature,
-        "Task Completion",
-        Math.round(area.taskCompletion),
-        GLOSSARY.taskCompletion
-      ),
-      value: Math.round(area.taskCompletion)
-    });
-  if (area.codeCorrectness < DIMENSION_WEAKNESS)
-    result.push({
-      dimension: "Code Correctness",
-      tip: dimTip(
-        area.feature,
-        "Code Correctness",
-        Math.round(area.codeCorrectness),
-        GLOSSARY.codeCorrectness
-      ),
-      value: Math.round(area.codeCorrectness)
-    });
-  if (area.docCoverage < DIMENSION_WEAKNESS)
-    result.push({
-      dimension: "Doc Coverage",
-      tip: dimTip(
-        area.feature,
-        "Doc Coverage",
-        Math.round(area.docCoverage),
-        GLOSSARY.docCoverage
-      ),
-      value: Math.round(area.docCoverage)
-    });
+  for (const dim of dims) {
+    if (dim.value < DIMENSION_WEAKNESS) {
+      result.push({
+        dimension: dim.label,
+        tip: dimTip(
+          area.feature,
+          dim.label,
+          Math.round(dim.value),
+          dim.tooltip
+        ),
+        value: Math.round(dim.value)
+      });
+    }
+  }
   return result;
 }
 
@@ -9137,6 +9225,7 @@ function ReportDetail({
             {
               comparison,
               durationMs: report.durationMs,
+              mode: provenance.mode,
               overall: summary.overall,
               scores: summary.scores,
               totalTests
@@ -9153,6 +9242,7 @@ function ReportDetail({
         comparison,
         focus,
         judgments: summary.lowScoringJudgments,
+        mode: provenance.mode,
         onNavigate: (newSubTab, newFocus) => onTabChange("diagnostics", newSubTab, newFocus),
         perModel: summary.perModel,
         recommendations: summary.recommendations,
@@ -9189,6 +9279,7 @@ function DiagnosticsPanel({
   comparison,
   focus,
   judgments,
+  mode,
   onNavigate,
   perModel,
   recommendations,
@@ -9254,6 +9345,7 @@ function DiagnosticsPanel({
       StrengthsList,
       {
         comparison,
+        mode,
         perModel,
         scores
       }
@@ -9263,6 +9355,7 @@ function DiagnosticsPanel({
         WeaknessesList,
         {
           comparison,
+          mode,
           perModel,
           scores
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf-studio",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "AI Literacy Framework — Sanity Studio dashboard plugin",
   "type": "module",
   "license": "MIT",