@sanity/ailf 6.1.1 → 7.0.0

package/dist/_vendor/ailf-shared/glossary.d.ts

@@ -0,0 +1,318 @@
+/**
+ * glossary.ts
+ *
+ * Centralized metric glossary used by Studio and dashboard alike. Each entry
+ * is keyed by a canonical slug. The value carries:
+ *
+ *   - `label`: the canonical user-facing label, e.g., "Overall Score". Used
+ *     by the dashboard's `<MetricLabel>` and may be reused as table-header
+ *     copy where space allows.
+ *   - `long`: full description for tooltips and help drawers. Copy was
+ *     ported verbatim from the Studio-era `packages/studio/src/glossary.ts`.
+ *
+ * A `short` field is a planned future addition for tight contexts where the
+ * `long` description is too verbose. Adding it later is a non-breaking
+ * schema change.
+ *
+ * @see docs/design-docs/scenario-matrix/evaluation-ceiling.md (three reference points)
+ * @see docs/architecture.md (scoring model)
+ */
+export interface GlossaryEntry {
+    label: string;
+    long: string;
+}
+export declare const GLOSSARY: {
+    readonly overallScore: {
+        readonly label: "Overall Score";
+        readonly long: "A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).";
+    };
+    readonly docLift: {
+        readonly label: "Doc Lift";
+        readonly long: "How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.";
+    };
+    readonly actualScore: {
+        readonly label: "Actual Score";
+        readonly long: "How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.";
+    };
+    readonly retrievalGap: {
+        readonly label: "Retrieval Gap";
+        readonly long: "The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.";
+    };
+    readonly infraEfficiency: {
+        readonly label: "Infra Efficiency";
+        readonly long: "What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.";
+    };
+    readonly floor: {
+        readonly label: "Floor";
+        readonly long: "Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.";
+    };
+    readonly ceiling: {
+        readonly label: "Ceiling";
+        readonly long: "Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.";
+    };
+    readonly actual: {
+        readonly label: "Actual";
+        readonly long: "Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.";
+    };
+    readonly retGap: {
+        readonly label: "Ret. Gap";
+        readonly long: "Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.";
+    };
+    readonly efficiency: {
+        readonly label: "Efficiency";
+        readonly long: "What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).";
+    };
+    readonly invertedRetGap: {
+        readonly label: "Inverted Retrieval Gap";
+        readonly long: "⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.";
+    };
+    readonly score: {
+        readonly label: "Score";
+        readonly long: "Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.";
+    };
+    readonly taskCompletion: {
+        readonly label: "Task Completion";
+        readonly long: "Can the LLM implement the requested feature? Graded 0–100.";
+    };
+    readonly codeCorrectness: {
+        readonly label: "Code Correctness";
+        readonly long: "Is the generated code idiomatic, correct, and following best practices? Graded 0–100.";
+    };
+    readonly docCoverage: {
+        readonly label: "Doc Coverage";
+        readonly long: "Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.";
+    };
+    readonly tests: {
+        readonly label: "Tests";
+        readonly long: "Number of test cases in this feature area.";
+    };
+    readonly overallDelta: {
+        readonly label: "Overall Δ";
+        readonly long: "Change in overall score between the two runs. Positive means the experiment scored higher.";
+    };
+    readonly actualDelta: {
+        readonly label: "Actual Δ";
+        readonly long: "Change in actual (agent-retrieved) score between runs. Positive means agents did better.";
+    };
+    readonly retGapDelta: {
+        readonly label: "Ret. Gap Δ";
+        readonly long: "Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.";
+    };
+    readonly efficiencyDelta: {
+        readonly label: "Efficiency Δ";
+        readonly long: "Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.";
+    };
+    readonly baseline: {
+        readonly label: "Baseline";
+        readonly long: "The reference run you're comparing against.";
+    };
+    readonly experiment: {
+        readonly label: "Experiment";
+        readonly long: "The new run you're evaluating.";
+    };
+    readonly delta: {
+        readonly label: "Delta";
+        readonly long: "Difference between experiment and baseline. Positive means improvement, negative means regression.";
+    };
+    readonly change: {
+        readonly label: "Change";
+        readonly long: "Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).";
+    };
+    readonly lowScoringJudgments: {
+        readonly label: "Low-Scoring Judgments";
+        readonly long: "The grading model's explanations for tests that scored below 70/100.";
+    };
+    readonly judgmentReason: {
+        readonly label: "Judgment Reason";
+        readonly long: "The grading model's natural language explanation of what went wrong.";
+    };
+    readonly healthStrong: {
+        readonly label: "Strong (80+)";
+        readonly long: "Feature areas scoring 80 or above. The docs are working well for these features — AI agents produce correct, complete implementations.";
+    };
+    readonly healthAttention: {
+        readonly label: "Needs Attention (70–79)";
+        readonly long: "Feature areas scoring 70–79. These are okay but could be improved — there may be gaps in specific dimensions like doc coverage or code correctness.";
+    };
+    readonly healthWeak: {
+        readonly label: "Weak (<70)";
+        readonly long: "Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.";
+    };
+    readonly negativeDocLiftMetric: {
+        readonly label: "Negative Doc Lift";
+        readonly long: "Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.";
+    };
+    readonly weakAreas: {
+        readonly label: "Weak Areas";
+        readonly long: "Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.";
+    };
+    readonly docsHurt: {
+        readonly label: "Docs Hurt Performance";
+        readonly long: "Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.";
+    };
+    readonly retrievalIssues: {
+        readonly label: "Retrieval Issues";
+        readonly long: "Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.";
+    };
+    readonly dimWeaknesses: {
+        readonly label: "Dimension Weaknesses";
+        readonly long: "Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).";
+    };
+    readonly efficiencyAnomalies: {
+        readonly label: "Efficiency Anomalies";
+        readonly long: "Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.";
+    };
+    readonly docLiftWins: {
+        readonly label: "Doc Lift Wins";
+        readonly long: "Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.";
+    };
+    readonly retrievalExcellence: {
+        readonly label: "Retrieval Excellence";
+        readonly long: "Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.";
+    };
+    readonly modelBreakdown: {
+        readonly label: "Model Breakdown";
+        readonly long: "Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently — useful for spotting models that struggle with specific feature areas.";
+    };
+    readonly strengths: {
+        readonly label: "Strengths";
+        readonly long: "What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.";
+    };
+    readonly recommendations: {
+        readonly label: "Recommendations";
+        readonly long: "Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.";
+    };
+    readonly totalPotentialLift: {
+        readonly label: "Total Potential Lift";
+        readonly long: "Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate — each gap targets the median of non-bottlenecked dimensions, not 100.";
+    };
+    readonly failureMode: {
+        readonly label: "Failure Mode";
+        readonly long: "The type of failure the grader emitted. Cross-cutting modes apply to any dimension (api-error, model-limitation, false-floor, unclassified). Per-dimension extensions cover documentation problems (missing-docs, incorrect-docs, outdated-docs, poor-structure), spec adherence (spec-mismatch), tool use (tool-misuse, chaotic-process, missing-recovery), and knowledge probes (factual-error, incompleteness, currency-violation, hallucination).";
+    };
+    readonly estimatedLift: {
+        readonly label: "Estimated Lift";
+        readonly long: "Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.";
+    };
+    readonly confidence: {
+        readonly label: "Confidence";
+        readonly long: "How confident we are in this diagnosis (D0049 ceiling-cross-check derivation). High = the grader's emitted failure mode agrees with the structural ceiling-decomposition signal. Medium = signals disagree (or the ceiling pattern is not informative for this score). Low = passing scores never classify; treat as absent.";
+    };
+    readonly agentBehaviorOverview: {
+        readonly label: "Agent Behavior";
+        readonly long: "How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.";
+    };
+    readonly searchQueries: {
+        readonly label: "Search Queries";
+        readonly long: "The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.";
+    };
+    readonly docSlugsVisited: {
+        readonly label: "Unique Doc Slugs";
+        readonly long: "Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.";
+    };
+    readonly externalDomains: {
+        readonly label: "External Domains";
+        readonly long: "Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.";
+    };
+    readonly avgDocPagesVisited: {
+        readonly label: "Avg Pages Visited";
+        readonly long: "Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.";
+    };
+    readonly avgSearchesPerformed: {
+        readonly label: "Avg Searches";
+        readonly long: "Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.";
+    };
+    readonly avgNetworkTimeMs: {
+        readonly label: "Avg Network Time";
+        readonly long: "Average time spent on network requests per test. Includes page fetches, search queries, and API calls.";
+    };
+    readonly totalRequests: {
+        readonly label: "Total Requests";
+        readonly long: "Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.";
+    };
+    readonly totalBytesDownloaded: {
+        readonly label: "Total Bytes Downloaded";
+        readonly long: "Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.";
+    };
+    readonly dimTaskCompletion: {
+        readonly label: "Task Completion Δ";
+        readonly long: "Change in task completion between runs. Positive means implementations are more complete.";
+    };
+    readonly dimCodeCorrectness: {
+        readonly label: "Code Correctness Δ";
+        readonly long: "Change in code correctness between runs. Positive means better code quality.";
+    };
+    readonly dimDocCoverage: {
+        readonly label: "Doc Coverage Δ";
+        readonly long: "Change in doc coverage between runs. Positive means the docs are providing more useful information.";
+    };
+    readonly areaDelta: {
+        readonly label: "Area Δ";
+        readonly long: "Score change for this area compared to the previous evaluation run.";
+    };
+    readonly sourceProduction: {
+        readonly label: "Production";
+        readonly long: "Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.";
+    };
+    readonly sourceBranch: {
+        readonly label: "Branch";
+        readonly long: "Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.";
+    };
+    readonly sourceLocal: {
+        readonly label: "Local";
+        readonly long: "Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.";
+    };
+    readonly reportScore: {
+        readonly label: "Score";
+        readonly long: "The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.";
+    };
+    readonly reportMode: {
+        readonly label: "Mode";
+        readonly long: "The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.";
+    };
+    readonly reportTrigger: {
+        readonly label: "Trigger";
+        readonly long: "What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.";
+    };
+    readonly modeBaseline: {
+        readonly label: "Baseline";
+        readonly long: "Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).";
+    };
+    readonly modeFull: {
+        readonly label: "Full";
+        readonly long: "Full mode — runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.";
+    };
+    readonly modeAgentic: {
+        readonly label: "Agentic";
+        readonly long: "Agentic mode — the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?";
+    };
+    readonly modeObserved: {
+        readonly label: "Observed";
+        readonly long: "Observed mode — records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.";
+    };
+    readonly modeDebug: {
+        readonly label: "Debug";
+        readonly long: "Debug mode — a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.";
+    };
+    readonly triggerManual: {
+        readonly label: "Manual";
+        readonly long: "Manually triggered — someone ran the evaluation pipeline by hand, either locally or via the Studio UI.";
+    };
+    readonly triggerCi: {
+        readonly label: "CI";
+        readonly long: "CI-triggered — the evaluation ran automatically as part of a pull request or merge pipeline.";
+    };
+    readonly triggerSchedule: {
+        readonly label: "Scheduled";
+        readonly long: "Scheduled — the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.";
+    };
+    readonly triggerWebhook: {
+        readonly label: "Webhook";
+        readonly long: "Webhook-triggered — a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.";
+    };
+    readonly triggerCrossRepo: {
+        readonly label: "Cross-Repo";
+        readonly long: "Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.";
+    };
+};
+export type GlossarySlug = keyof typeof GLOSSARY;

package/dist/_vendor/ailf-shared/glossary.js

@@ -0,0 +1,330 @@
+/**
+ * glossary.ts
+ *
+ * Centralized metric glossary used by Studio and dashboard alike. Each entry
+ * is keyed by a canonical slug. The value carries:
+ *
+ *   - `label`: the canonical user-facing label, e.g., "Overall Score". Used
+ *     by the dashboard's `<MetricLabel>` and may be reused as table-header
+ *     copy where space allows.
+ *   - `long`: full description for tooltips and help drawers. Copy was
+ *     ported verbatim from the Studio-era `packages/studio/src/glossary.ts`.
+ *
+ * A `short` field is a planned future addition for tight contexts where the
+ * `long` description is too verbose. Adding it later is a non-breaking
+ * schema change.
+ *
+ * @see docs/design-docs/scenario-matrix/evaluation-ceiling.md (three reference points)
+ * @see docs/architecture.md (scoring model)
+ */
+export const GLOSSARY = {
+    // -- Overview stats -------------------------------------------------------
+    overallScore: {
+        label: "Overall Score",
+        long: "A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).",
+    },
+    docLift: {
+        label: "Doc Lift",
+        long: "How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.",
+    },
+    actualScore: {
+        label: "Actual Score",
+        long: "How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.",
+    },
+    retrievalGap: {
+        label: "Retrieval Gap",
+        long: "The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.",
+    },
+    infraEfficiency: {
+        label: "Infra Efficiency",
+        long: "What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.",
+    },
+    // -- Three-layer decomposition columns ------------------------------------
+    floor: {
+        label: "Floor",
+        long: "Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.",
+    },
+    ceiling: {
+        label: "Ceiling",
+        long: "Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.",
+    },
+    actual: {
+        label: "Actual",
+        long: "Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.",
+    },
+    retGap: {
+        label: "Ret. Gap",
+        long: "Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.",
+    },
+    efficiency: {
+        label: "Efficiency",
+        long: "What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).",
+    },
+    invertedRetGap: {
+        label: "Inverted Retrieval Gap",
+        long: "⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.",
+    },
+    // -- Per-area score columns -----------------------------------------------
+    score: {
+        label: "Score",
+        long: "Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.",
+    },
+    taskCompletion: {
+        label: "Task Completion",
+        long: "Can the LLM implement the requested feature? Graded 0–100.",
+    },
+    codeCorrectness: {
+        label: "Code Correctness",
+        long: "Is the generated code idiomatic, correct, and following best practices? Graded 0–100.",
+    },
+    docCoverage: {
+        label: "Doc Coverage",
+        long: "Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.",
+    },
+    tests: {
+        label: "Tests",
+        long: "Number of test cases in this feature area.",
+    },
+    // -- Comparison deltas ----------------------------------------------------
+    overallDelta: {
+        label: "Overall Δ",
+        long: "Change in overall score between the two runs. Positive means the experiment scored higher.",
+    },
+    actualDelta: {
+        label: "Actual Δ",
+        long: "Change in actual (agent-retrieved) score between runs. Positive means agents did better.",
+    },
+    retGapDelta: {
+        label: "Ret. Gap Δ",
+        long: "Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.",
+    },
+    efficiencyDelta: {
+        label: "Efficiency Δ",
+        long: "Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.",
+    },
+    // -- Comparison table columns ---------------------------------------------
+    baseline: {
+        label: "Baseline",
+        long: "The reference run you're comparing against.",
+    },
+    experiment: {
+        label: "Experiment",
+        long: "The new run you're evaluating.",
+    },
+    delta: {
+        label: "Delta",
+        long: "Difference between experiment and baseline. Positive means improvement, negative means regression.",
+    },
+    change: {
+        label: "Change",
+        long: "Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).",
+    },
+    // -- Grader judgments ------------------------------------------------------
+    lowScoringJudgments: {
+        label: "Low-Scoring Judgments",
+        long: "The grading model's explanations for tests that scored below 70/100.",
+    },
+    judgmentReason: {
+        label: "Judgment Reason",
+        long: "The grading model's natural language explanation of what went wrong.",
+    },
+    // -- Diagnostics overview ---------------------------------------------------
+    healthStrong: {
+        label: "Strong (80+)",
+        long: "Feature areas scoring 80 or above. The docs are working well for these features — AI agents produce correct, complete implementations.",
+    },
+    healthAttention: {
+        label: "Needs Attention (70–79)",
+        long: "Feature areas scoring 70–79. These are okay but could be improved — there may be gaps in specific dimensions like doc coverage or code correctness.",
+    },
+    healthWeak: {
+        label: "Weak (<70)",
+        long: "Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.",
+    },
+    negativeDocLiftMetric: {
+        label: "Negative Doc Lift",
+        long: "Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.",
+    },
+    weakAreas: {
+        label: "Weak Areas",
+        long: "Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.",
+    },
+    docsHurt: {
+        label: "Docs Hurt Performance",
+        long: "Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.",
+    },
+    retrievalIssues: {
+        label: "Retrieval Issues",
+        long: "Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.",
+    },
+    dimWeaknesses: {
+        label: "Dimension Weaknesses",
+        long: "Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).",
+    },
+    efficiencyAnomalies: {
+        label: "Efficiency Anomalies",
+        long: "Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.",
+    },
+    docLiftWins: {
+        label: "Doc Lift Wins",
+        long: "Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.",
+    },
+    retrievalExcellence: {
+        label: "Retrieval Excellence",
+        long: "Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.",
+    },
+    // -- Model breakdown --------------------------------------------------------
+    modelBreakdown: {
+        label: "Model Breakdown",
+        long: "Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently — useful for spotting models that struggle with specific feature areas.",
+    },
+    // -- Strengths (positive diagnostics) ---------------------------------------
+    strengths: {
+        label: "Strengths",
+        long: "What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.",
+    },
+    // -- Recommendations / gap analysis ----------------------------------------
+    recommendations: {
+        label: "Recommendations",
+        long: "Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.",
+    },
+    totalPotentialLift: {
+        label: "Total Potential Lift",
+        long: "Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate — each gap targets the median of non-bottlenecked dimensions, not 100.",
+    },
+    failureMode: {
+        label: "Failure Mode",
+        long: "The type of failure the grader emitted. Cross-cutting modes apply to any dimension (api-error, model-limitation, false-floor, unclassified). Per-dimension extensions cover documentation problems (missing-docs, incorrect-docs, outdated-docs, poor-structure), spec adherence (spec-mismatch), tool use (tool-misuse, chaotic-process, missing-recovery), and knowledge probes (factual-error, incompleteness, currency-violation, hallucination).",
+    },
+    estimatedLift: {
+        label: "Estimated Lift",
+        long: "Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.",
+    },
+    confidence: {
+        label: "Confidence",
+        long: "How confident we are in this diagnosis (D0049 ceiling-cross-check derivation). High = the grader's emitted failure mode agrees with the structural ceiling-decomposition signal. Medium = signals disagree (or the ceiling pattern is not informative for this score). Low = passing scores never classify; treat as absent.",
+    },
+    // -- Agent behavior --------------------------------------------------------
+    agentBehaviorOverview: {
+        label: "Agent Behavior",
+        long: "How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.",
+    },
+    searchQueries: {
+        label: "Search Queries",
+        long: "The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.",
+    },
+    docSlugsVisited: {
+        label: "Unique Doc Slugs",
+        long: "Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.",
+    },
+    externalDomains: {
+        label: "External Domains",
+        long: "Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.",
+    },
+    avgDocPagesVisited: {
+        label: "Avg Pages Visited",
+        long: "Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.",
+    },
+    avgSearchesPerformed: {
+        label: "Avg Searches",
+        long: "Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.",
+    },
+    avgNetworkTimeMs: {
+        label: "Avg Network Time",
+        long: "Average time spent on network requests per test. Includes page fetches, search queries, and API calls.",
+    },
+    totalRequests: {
+        label: "Total Requests",
+        long: "Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.",
+    },
+    totalBytesDownloaded: {
+        label: "Total Bytes Downloaded",
+        long: "Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.",
+    },
+    // -- Dimension deltas -----------------------------------------------------
+    dimTaskCompletion: {
+        label: "Task Completion Δ",
+        long: "Change in task completion between runs. Positive means implementations are more complete.",
+    },
+    dimCodeCorrectness: {
+        label: "Code Correctness Δ",
+        long: "Change in code correctness between runs. Positive means better code quality.",
+    },
+    dimDocCoverage: {
+        label: "Doc Coverage Δ",
+        long: "Change in doc coverage between runs. Positive means the docs are providing more useful information.",
+    },
+    // -- Per-area trend delta ----------------------------------------------------
+    areaDelta: {
+        label: "Area Δ",
+        long: "Score change for this area compared to the previous evaluation run.",
+    },
+    // -- Source values -----------------------------------------------------------
+    sourceProduction: {
+        label: "Production",
+        long: "Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.",
+    },
+    sourceBranch: {
+        label: "Branch",
+        long: "Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.",
+    },
+    sourceLocal: {
+        label: "Local",
+        long: "Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.",
+    },
+    // -- Report list columns ----------------------------------------------------
+    reportScore: {
+        label: "Score",
+        long: "The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.",
+    },
+    reportMode: {
+        label: "Mode",
+        long: "The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.",
+    },
+    reportTrigger: {
+        label: "Trigger",
+        long: "What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.",
+    },
+    // -- Mode values -----------------------------------------------------------
+    modeBaseline: {
+        label: "Baseline",
+        long: "Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).",
+    },
+    modeFull: {
+        label: "Full",
+        long: "Full mode — runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.",
+    },
+    modeAgentic: {
+        label: "Agentic",
+        long: "Agentic mode — the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?",
+    },
+    modeObserved: {
+        label: "Observed",
+        long: "Observed mode — records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.",
+    },
+    modeDebug: {
+        label: "Debug",
+        long: "Debug mode — a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.",
+    },
+    // -- Trigger values --------------------------------------------------------
+    triggerManual: {
+        label: "Manual",
+        long: "Manually triggered — someone ran the evaluation pipeline by hand, either locally or via the Studio UI.",
+    },
+    triggerCi: {
+        label: "CI",
+        long: "CI-triggered — the evaluation ran automatically as part of a pull request or merge pipeline.",
+    },
+    triggerSchedule: {
+        label: "Scheduled",
+        long: "Scheduled — the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.",
+    },
+    triggerWebhook: {
+        label: "Webhook",
+        long: "Webhook-triggered — a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.",
+    },
+    triggerCrossRepo: {
+        label: "Cross-Repo",
+        long: "Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
+    },
+};

package/dist/_vendor/ailf-shared/help-content.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Re-export of the build-generated help-topic table.
+ *
+ * The underlying file `src/generated/help-content.ts` is emitted by
+ * `scripts/extract-help.ts` and is gitignored. Run `pnpm extract-help`
+ * (invoked automatically by this package's `prebuild`) to (re)generate it.
+ *
+ * @see scripts/extract-help.ts
+ */
+export { HELP_TOPICS } from "./generated/help-content.js";