@sanity/ailf 7.2.2 → 7.2.3

package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts

@@ -76,6 +76,28 @@ export interface ArtifactWriter {
      */
     writePerEntry(type: ArtifactType, runId: RunId, entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
 }
+/**
+ * Optional capability: check whether an object physically exists at a
+ * store-relative `path`. Kept **separate** from `ArtifactWriter` so writers
+ * opt in by structural typing rather than every implementor being forced to
+ * provide it — consumers feature-detect it (see the writer-chain walkers in
+ * the eval package).
+ *
+ * Used by the cache-hit restore path (D0040 / D0057) to drop artifact refs
+ * a prior report over-claimed: a degraded source run can list `rawResults`
+ * entries whose GCS objects were never written, and `remapToCacheHitRefs`
+ * would otherwise copy those phantom entries into the new run's manifest.
+ *
+ * Contract:
+ *   - resolves `true`  — the object exists.
+ *   - resolves `false` — the object is *definitively* absent.
+ *   - **throws** — existence could not be determined (auth / network /
+ *     transient error). Callers must treat a throw as "unknown" and
+ *     fail open (keep the ref) rather than dropping a real artifact.
+ */
+export interface ArtifactObjectChecker {
+    objectExists(path: string): Promise<boolean>;
+}
 /**
  * Thrown by writers that can't satisfy a method — e.g. an
  * `ApiGatewayArtifactWriter` cannot implement `appendNdjson` until the batch

package/dist/_vendor/ailf-core/ports/index.d.ts

@@ -4,7 +4,7 @@
  * Ports define the contracts between the domain kernel and the outside world.
  * Adapters (in packages/eval) implement these interfaces.
  */
-export type { ArtifactEntry, ArtifactWriter } from "./artifact-writer.js";
+export type { ArtifactEntry, ArtifactObjectChecker, ArtifactWriter, } from "./artifact-writer.js";
 export { NoOpArtifactWriter } from "./artifact-writer.js";
 export type { CacheEntryMetadata, CacheKey, CacheLookupResult, CacheRecordInput, CacheStore, } from "./cache-store.js";
 export type { ConfigSource } from "./config-source.js";

package/dist/_vendor/ailf-core/services/diagnosis/cards/top-recommendations.js

@@ -59,6 +59,20 @@ export const generateTopRecommendations = async (report, ctx) => {
     }
     // Build allow-list from the runtime report
     const allowList = buildDocSlugAllowList(report);
+    // Short-circuit BEFORE the LLM call when the allow-list is empty — the shape
+    // a degraded run produces (enrichment never wrote `documentManifest`, and
+    // per-score `documents[]` carry no slugs). With an empty allow-list the
+    // system prompt instructs the model to omit every recommendation, so it
+    // returns `suggestions: []` and trips the `.min(1)` schema, surfacing as a
+    // degraded card with a raw ZodError. Degrade gracefully to "missing"
+    // instead — same structural seam the sibling attribution cards use.
+    if (allowList.size === 0) {
+        return {
+            status: "missing",
+            cardType: "top-recommendations",
+            reason: "no doc-slug allow-list for this run (enrichment incomplete)",
+        };
+    }
     // Per-call schema: additive docSlug allow-list refine (AI-SPEC §3 Pitfall 1)
     const PerCallSchema = z.object({
         summary: z.string().min(1).max(500),

package/dist/_vendor/ailf-shared/generated/help-content.js

@@ -142,7 +142,7 @@ export const HELP_TOPICS = [
     {
         "id": "glossary",
         "title": "Glossary",
-        "body": "**Overall Score**\n: A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).\n\n**Doc Lift**\n: How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.\n\n**Actual Score**\n: How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.\n\n**Retrieval Gap**\n: The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.\n\n**Infra Efficiency**\n: What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.\n\n**Floor**\n: Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.\n\n**Ceiling**\n: Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.\n\n**Actual**\n: Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.\n\n**Ret. Gap**\n: Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.\n\n**Efficiency**\n: What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).\n\n**Inverted Retrieval Gap**\n: ⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.\n\n**Score**\n: Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.\n\n**Task Completion**\n: Can the LLM implement the requested feature? Graded 0–100.\n\n**Code Correctness**\n: Is the generated code idiomatic, correct, and following best practices? Graded 0–100.\n\n**Doc Coverage**\n: Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.\n\n**Tests**\n: Number of test cases in this feature area.\n\n**Overall Δ**\n: Change in overall score between the two runs. Positive means the experiment scored higher.\n\n**Actual Δ**\n: Change in actual (agent-retrieved) score between runs. Positive means agents did better.\n\n**Ret. Gap Δ**\n: Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.\n\n**Efficiency Δ**\n: Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.\n\n**Baseline**\n: The reference run you're comparing against.\n\n**Experiment**\n: The new run you're evaluating.\n\n**Delta**\n: Difference between experiment and baseline. Positive means improvement, negative means regression.\n\n**Change**\n: Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).\n\n**Low-Scoring Judgments**\n: The grading model's explanations for tests that scored below 70/100.\n\n**Judgment Reason**\n: The grading model's natural language explanation of what went wrong.\n\n**Strong (80+)**\n: Feature areas scoring 80 or above. The docs are working well for these features — AI agents produce correct, complete implementations.\n\n**Needs Attention (70–79)**\n: Feature areas scoring 70–79. These are okay but could be improved — there may be gaps in specific dimensions like doc coverage or code correctness.\n\n**Weak (<70)**\n: Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.\n\n**Negative Doc Lift**\n: Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.\n\n**Weak Areas**\n: Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.\n\n**Docs Hurt Performance**\n: Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.\n\n**Retrieval Issues**\n: Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.\n\n**Dimension Weaknesses**\n: Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).\n\n**Efficiency Anomalies**\n: Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.\n\n**Doc Lift Wins**\n: Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.\n\n**Retrieval Excellence**\n: Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.\n\n**Model Breakdown**\n: Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently — useful for spotting models that struggle with specific feature areas.\n\n**Strengths**\n: What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.\n\n**Recommendations**\n: Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.\n\n**Total Potential Lift**\n: Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate — each gap targets the median of non-bottlenecked dimensions, not 100.\n\n**Failure Mode**\n: The type of failure the grader emitted. Cross-cutting modes apply to any dimension (api-error, model-limitation, false-floor, unclassified). Per-dimension extensions cover documentation problems (missing-docs, incorrect-docs, outdated-docs, poor-structure), spec adherence (spec-mismatch), tool use (tool-misuse, chaotic-process, missing-recovery), and knowledge probes (factual-error, incompleteness, currency-violation, hallucination).\n\n**Estimated Lift**\n: Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.\n\n**Confidence**\n: How confident we are in this diagnosis (D0049 ceiling-cross-check derivation). High = the grader's emitted failure mode agrees with the structural ceiling-decomposition signal. Medium = signals disagree (or the ceiling pattern is not informative for this score). Low = passing scores never classify; treat as absent.\n\n**Agent Behavior**\n: How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.\n\n**Search Queries**\n: The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.\n\n**Unique Doc Slugs**\n: Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.\n\n**External Domains**\n: Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.\n\n**Avg Pages Visited**\n: Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.\n\n**Avg Searches**\n: Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.\n\n**Avg Network Time**\n: Average time spent on network requests per test. Includes page fetches, search queries, and API calls.\n\n**Total Requests**\n: Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.\n\n**Total Bytes Downloaded**\n: Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.\n\n**Task Completion Δ**\n: Change in task completion between runs. Positive means implementations are more complete.\n\n**Code Correctness Δ**\n: Change in code correctness between runs. Positive means better code quality.\n\n**Doc Coverage Δ**\n: Change in doc coverage between runs. Positive means the docs are providing more useful information.\n\n**Area Δ**\n: Score change for this area compared to the previous evaluation run.\n\n**Production**\n: Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.\n\n**Branch**\n: Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.\n\n**Local**\n: Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.\n\n**Score**\n: The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.\n\n**Mode**\n: The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.\n\n**Trigger**\n: What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.\n\n**Baseline**\n: Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).\n\n**Full**\n: Full mode — runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.\n\n**Agentic**\n: Agentic mode — the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?\n\n**Observed**\n: Observed mode — records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.\n\n**Debug**\n: Debug mode — a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.\n\n**Manual**\n: Manually triggered — someone ran the evaluation pipeline by hand, either locally or via the Studio UI.\n\n**CI**\n: CI-triggered — the evaluation ran automatically as part of a pull request or merge pipeline.\n\n**Scheduled**\n: Scheduled — the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.\n\n**Webhook**\n: Webhook-triggered — a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.\n\n**Cross-Repo**\n: Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
+        "body": "**Overall Score**\n: A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).\n\n**Doc Lift**\n: How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.\n\n**Actual Score**\n: How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.\n\n**Retrieval Gap**\n: The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.\n\n**Infra Efficiency**\n: What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.\n\n**Floor**\n: Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.\n\n**Ceiling**\n: Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.\n\n**Actual**\n: Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.\n\n**Ret. Gap**\n: Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.\n\n**Efficiency**\n: What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).\n\n**Inverted Retrieval Gap**\n: ⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.\n\n**Score**\n: Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.\n\n**Task Completion**\n: Can the LLM implement the requested feature? Graded 0–100.\n\n**Code Correctness**\n: Is the generated code idiomatic, correct, and following best practices? Graded 0–100.\n\n**Doc Coverage**\n: Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.\n\n**Tests**\n: Number of test cases in this feature area.\n\n**Overall Δ**\n: Change in overall score between the two runs. Positive means the experiment scored higher.\n\n**Actual Δ**\n: Change in actual (agent-retrieved) score between runs. Positive means agents did better.\n\n**Ret. Gap Δ**\n: Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.\n\n**Efficiency Δ**\n: Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.\n\n**Baseline**\n: The reference run you're comparing against.\n\n**Experiment**\n: The new run you're evaluating.\n\n**Delta**\n: Difference between experiment and baseline. Positive means improvement, negative means regression.\n\n**Change**\n: Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).\n\n**Low-Scoring Judgments**\n: The grading model's explanations for tests that scored below 70/100.\n\n**Judgment Reason**\n: The grading model's natural language explanation of what went wrong.\n\n**Strong (80+)**\n: Feature areas scoring 80 or above. The docs are working well for these features — AI agents produce correct, complete implementations.\n\n**Needs Attention (70–79)**\n: Feature areas scoring 70–79. These are okay but could be improved — there may be gaps in specific dimensions like doc coverage or code correctness.\n\n**Weak (<70)**\n: Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.\n\n**Negative Doc Lift**\n: Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.\n\n**Weak Areas**\n: Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.\n\n**Docs Hurt Performance**\n: Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.\n\n**Retrieval Issues**\n: Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.\n\n**Dimension Weaknesses**\n: Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).\n\n**Efficiency Anomalies**\n: Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.\n\n**Doc Lift Wins**\n: Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.\n\n**Retrieval Excellence**\n: Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.\n\n**Model Breakdown**\n: Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently — useful for spotting models that struggle with specific feature areas.\n\n**Strengths**\n: What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.\n\n**Recommendations**\n: Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.\n\n**Total Potential Lift**\n: Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate — each gap targets the median of non-bottlenecked dimensions, not 100.\n\n**Failure Mode**\n: The type of failure the grader emitted. Cross-cutting modes apply to any dimension (api-error, model-limitation, false-floor, unclassified). Per-dimension extensions cover documentation problems (missing-docs, incorrect-docs, outdated-docs, poor-structure), spec adherence (spec-mismatch), tool use (tool-misuse, chaotic-process, missing-recovery), and knowledge probes (factual-error, incompleteness, currency-violation, hallucination).\n\n**Estimated Lift**\n: Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.\n\n**Confidence**\n: How confident we are in this diagnosis (D0049 ceiling-cross-check derivation). High = the grader's emitted failure mode agrees with the structural ceiling-decomposition signal. Medium = signals disagree (or the ceiling pattern is not informative for this score). Low = passing scores never classify; treat as absent.\n\n**Agent Behavior**\n: How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.\n\n**Search Queries**\n: The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.\n\n**Unique Doc Slugs**\n: Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.\n\n**External Domains**\n: Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.\n\n**Avg Pages Visited**\n: Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.\n\n**Avg Searches**\n: Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.\n\n**Avg Network Time**\n: Average time spent on network requests per test. Includes page fetches, search queries, and API calls.\n\n**Total Requests**\n: Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.\n\n**Total Bytes Downloaded**\n: Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.\n\n**Task Completion Δ**\n: Change in task completion between runs. Positive means implementations are more complete.\n\n**Code Correctness Δ**\n: Change in code correctness between runs. Positive means better code quality.\n\n**Doc Coverage Δ**\n: Change in doc coverage between runs. Positive means the docs are providing more useful information.\n\n**Area Δ**\n: Score change for this area compared to the previous evaluation run.\n\n**Production**\n: Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.\n\n**Branch**\n: Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.\n\n**Local**\n: Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.\n\n**Score**\n: The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.\n\n**Mode**\n: The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.\n\n**Trigger**\n: What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.\n\n**Baseline**\n: Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).\n\n**Full**\n: Full mode — runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.\n\n**Agentic**\n: Agentic mode — the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?\n\n**Observed**\n: Observed mode — records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.\n\n**Debug**\n: Debug mode — a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.\n\n**Manual**\n: Manually triggered — someone ran the evaluation pipeline by hand, either locally or via the Studio UI.\n\n**CI**\n: CI-triggered — the evaluation ran automatically as part of a pull request or merge pipeline.\n\n**Scheduled**\n: Scheduled — the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.\n\n**Webhook**\n: Webhook-triggered — a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.\n\n**Cross-Repo**\n: Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.\n\n**Gold**\n: Gold variant — the relevant docs were injected into the prompt as context. This is the ceiling condition: the best the documentation can do.\n\n**Baseline**\n: Baseline variant — no documentation in the prompt. This is the floor condition: what the model already knows from its training data, used as the control for doc lift.\n\n**Naive**\n: Naive — the in-house agentic harness with naive prompting. The model runs in an agent loop with no doc-targeting strategy.\n\n**Optimized**\n: Optimized — the in-house agentic harness with optimized prompting. The model runs in an agent loop tuned for documentation retrieval.\n\n**Normal**\n: Normal — a direct vendor-API call. The model produces a single-shot completion with no agent loop.\n\n**Fail**\n: Fail — the model produced no usable output (empty response, API error, or token exhaustion). Distinct from a low score on output that was produced.\n\n**Low dim**\n: Low dim — the run produced output but at least one grading dimension scored below 60.\n\n**OK**\n: OK — the run produced output and every grading dimension scored 60 or above.",
         "source": "packages/shared/src/glossary.ts",
         "tags": [
             "reference",

package/dist/_vendor/ailf-shared/glossary.d.ts

@@ -314,5 +314,37 @@ export declare const GLOSSARY: {
         readonly label: "Cross-Repo";
         readonly long: "Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.";
     };
+    readonly variantGold: {
+        readonly label: "Gold";
+        readonly long: "Gold variant — the relevant docs were injected into the prompt as context. This is the ceiling condition: the best the documentation can do.";
+    };
+    readonly variantBaseline: {
+        readonly label: "Baseline";
+        readonly long: "Baseline variant — no documentation in the prompt. This is the floor condition: what the model already knows from its training data, used as the control for doc lift.";
+    };
+    readonly engineNaive: {
+        readonly label: "Naive";
+        readonly long: "Naive — the in-house agentic harness with naive prompting. The model runs in an agent loop with no doc-targeting strategy.";
+    };
+    readonly engineOptimized: {
+        readonly label: "Optimized";
+        readonly long: "Optimized — the in-house agentic harness with optimized prompting. The model runs in an agent loop tuned for documentation retrieval.";
+    };
+    readonly engineNormal: {
+        readonly label: "Normal";
+        readonly long: "Normal — a direct vendor-API call. The model produces a single-shot completion with no agent loop.";
+    };
+    readonly statusFail: {
+        readonly label: "Fail";
+        readonly long: "Fail — the model produced no usable output (empty response, API error, or token exhaustion). Distinct from a low score on output that was produced.";
+    };
+    readonly statusLowDim: {
+        readonly label: "Low dim";
+        readonly long: "Low dim — the run produced output but at least one grading dimension scored below 60.";
+    };
+    readonly statusOk: {
+        readonly label: "OK";
+        readonly long: "OK — the run produced output and every grading dimension scored 60 or above.";
+    };
 };
 export type GlossarySlug = keyof typeof GLOSSARY;

package/dist/_vendor/ailf-shared/glossary.js

@@ -327,4 +327,39 @@ export const GLOSSARY = {
         label: "Cross-Repo",
         long: "Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
     },
+    // -- Variant values (per-test docs condition) ------------------------------
+    variantGold: {
+        label: "Gold",
+        long: "Gold variant — the relevant docs were injected into the prompt as context. This is the ceiling condition: the best the documentation can do.",
+    },
+    variantBaseline: {
+        label: "Baseline",
+        long: "Baseline variant — no documentation in the prompt. This is the floor condition: what the model already knows from its training data, used as the control for doc lift.",
+    },
+    // -- Execution mode values (how the model was driven) ----------------------
+    engineNaive: {
+        label: "Naive",
+        long: "Naive — the in-house agentic harness with naive prompting. The model runs in an agent loop with no doc-targeting strategy.",
+    },
+    engineOptimized: {
+        label: "Optimized",
+        long: "Optimized — the in-house agentic harness with optimized prompting. The model runs in an agent loop tuned for documentation retrieval.",
+    },
+    engineNormal: {
+        label: "Normal",
+        long: "Normal — a direct vendor-API call. The model produces a single-shot completion with no agent loop.",
+    },
+    // -- Status values (per-test outcome) --------------------------------------
+    statusFail: {
+        label: "Fail",
+        long: "Fail — the model produced no usable output (empty response, API error, or token exhaustion). Distinct from a low score on output that was produced.",
+    },
+    statusLowDim: {
+        label: "Low dim",
+        long: "Low dim — the run produced output but at least one grading dimension scored below 60.",
+    },
+    statusOk: {
+        label: "OK",
+        long: "OK — the run produced output and every grading dimension scored 60 or above.",
+    },
 };

package/dist/artifact-capture/fanout-artifact-writer.d.ts

@@ -38,6 +38,14 @@ export declare class FanoutArtifactWriter implements ArtifactWriter {
     private readonly writers;
     private readonly progress?;
     constructor(writers: readonly ArtifactWriter[], options?: FanoutArtifactWriterOptions);
+    /**
+     * The delegate writers in declaration order. Exposed read-only so callers
+     * walking the writer chain (e.g. to feature-detect an
+     * `ArtifactObjectChecker`) can descend into the fanout without it having to
+     * re-implement every optional capability. Mirrors the decorators' readonly
+     * `inner` accessor.
+     */
+    get delegates(): readonly ArtifactWriter[];
     private reportProgress;
     emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
     appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;

package/dist/artifact-capture/fanout-artifact-writer.js

@@ -33,6 +33,16 @@ export class FanoutArtifactWriter {
         this.writers = writers;
         this.progress = options.progress;
     }
+    /**
+     * The delegate writers in declaration order. Exposed read-only so callers
+     * walking the writer chain (e.g. to feature-detect an
+     * `ArtifactObjectChecker`) can descend into the fanout without it having to
+     * re-implement every optional capability. Mirrors the decorators' readonly
+     * `inner` accessor.
+     */
+    get delegates() {
+        return this.writers;
+    }
     reportProgress(ref) {
         if (!this.progress)
             return;

package/dist/artifact-capture/gcs-artifact-writer.d.ts

@@ -28,7 +28,7 @@
  * @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
  */
 import { Storage } from "@google-cloud/storage";
-import { type ArtifactEntry, type ArtifactRef, type ArtifactType, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssociationValues, type RunId, type RunManifest, type WriteSource } from "../_vendor/ailf-core/index.d.ts";
+import { type ArtifactEntry, type ArtifactObjectChecker, type ArtifactRef, type ArtifactType, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssociationValues, type RunId, type RunManifest, type WriteSource } from "../_vendor/ailf-core/index.d.ts";
 import { type UploadMetricsSink } from "./upload-metrics.js";
 export interface GcsArtifactWriterOptions {
     /** GCS bucket name (e.g., "ailf-artifacts") */
@@ -61,7 +61,7 @@ export interface GcsArtifactWriterOptions {
      */
     writerSource?: WriteSource;
 }
-export declare class GcsArtifactWriter implements ArtifactWriter {
+export declare class GcsArtifactWriter implements ArtifactWriter, ArtifactObjectChecker {
     private client;
     private readonly options;
     private readonly ndjsonStreams;
@@ -83,6 +83,16 @@ export declare class GcsArtifactWriter implements ArtifactWriter {
     emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
     appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;
     writeManifest(runId: RunId, manifest: RunManifest): Promise<ArtifactRef | null>;
+    /**
+     * Existence probe used by the cache-hit restore prune (D0057). Unlike the
+     * write methods (P5 non-blocking — swallow errors, return null), this
+     * resolves `false` ONLY for a definitively-absent object and **throws** on
+     * any other failure (auth / network / quota) so the caller can fail open
+     * and keep the ref rather than dropping a real artifact on a transient
+     * blip. `file.exists()` rejects only on real errors; a missing object
+     * resolves `[false]`.
+     */
+    objectExists(path: string): Promise<boolean>;
     /** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
     writeBulk(type: ArtifactType, runId: RunId, data: unknown): Promise<ArtifactRef | null>;
     /** @deprecated Use `emit()` per entry instead. */

package/dist/artifact-capture/gcs-artifact-writer.js

@@ -223,6 +223,24 @@ export class GcsArtifactWriter {
             this.reportProgress(ref);
         return ref;
     }
+    // ---- ArtifactObjectChecker (D0057) --------------------------------------
+    /**
+     * Existence probe used by the cache-hit restore prune (D0057). Unlike the
+     * write methods (P5 non-blocking — swallow errors, return null), this
+     * resolves `false` ONLY for a definitively-absent object and **throws** on
+     * any other failure (auth / network / quota) so the caller can fail open
+     * and keep the ref rather than dropping a real artifact on a transient
+     * blip. `file.exists()` rejects only on real errors; a missing object
+     * resolves `[false]`.
+     */
+    async objectExists(path) {
+        const storage = this.getClient();
+        const [exists] = await storage
+            .bucket(this.options.bucket)
+            .file(path)
+            .exists();
+        return exists;
+    }
     // ---- Deprecated legacy surface (W0052) ----------------------------------
     /** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
     async writeBulk(type, runId, data) {

package/dist/orchestration/steps/run-eval-step.js

@@ -11,9 +11,11 @@ import { emitPerEntryEvalResults } from "../../pipeline/emit-eval-results.js";
 import { emitSymbolPreflight } from "../../pipeline/preflight/emit-symbol-preflight.js";
 import { loadPackageSurface } from "../../pipeline/preflight/load-package-surface.js";
 import { AccumulatingArtifactWriter } from "../../artifact-capture/accumulating-artifact-writer.js";
+import { FanoutArtifactWriter } from "../../artifact-capture/fanout-artifact-writer.js";
+import { InstrumentedArtifactWriter } from "../../artifact-capture/instrumented-artifact-writer.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { buildCacheContext } from "../cache-context.js";
-import { remapToCacheHitRefs } from "../../pipeline/cache-hit-restore.js";
+import { pruneToResolvableRefs, remapToCacheHitRefs, } from "../../pipeline/cache-hit-restore.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
 import { computeEvalFingerprint } from "../../pipeline/eval-fingerprint.js";
 import { loadGraderModel } from "../../pipeline/grader-api.js";
@@ -147,11 +149,29 @@ export class RunEvalStep {
                     remoteCacheResult.sourceRunId &&
                     ctx.artifactWriter instanceof AccumulatingArtifactWriter) {
                     const restored = remapToCacheHitRefs(remoteCacheResult.artifactManifest, { sourceRunId: remoteCacheResult.sourceRunId });
-                    ctx.artifactWriter.injectAccumulated(restored);
-                    const count = Object.keys(restored).length;
+                    // W0350 / D0057 — a degraded source run can advertise per-entry
+                    // artifacts (e.g. rawResults) whose objects were never written under
+                    // its prefix. Drop those over-claims here, at the restore boundary,
+                    // so the new run's manifest advertises only artifacts that resolve —
+                    // rather than pushing per-object HEAD checks onto the read side's hot
+                    // signing path (AC 3). When no object checker is reachable in the
+                    // writer chain (local-only / NoOp / gateway backends), skip the prune
+                    // and restore verbatim, preserving prior behavior.
+                    const checker = findObjectChecker(ctx.artifactWriter);
+                    const { manifest: resolvable, droppedEntries, droppedRefs, } = checker
+                        ? await pruneToResolvableRefs(restored, checker)
+                        : { manifest: restored, droppedEntries: 0, droppedRefs: 0 };
+                    ctx.artifactWriter.injectAccumulated(resolvable);
+                    const count = Object.keys(resolvable).length;
                     if (count > 0) {
                         console.log(`  ↪ Restored ${count} artifact ref${count === 1 ? "" : "s"} from run ${remoteCacheResult.sourceRunId}`);
                     }
+                    if (droppedEntries > 0 || droppedRefs > 0) {
+                        const refsNote = droppedRefs > 0
+                            ? ` and ${droppedRefs} ref${droppedRefs === 1 ? "" : "s"}`
+                            : "";
+                        console.log(`  ⚠️  Dropped ${droppedEntries} unresolvable artifact entr${droppedEntries === 1 ? "y" : "ies"}${refsNote} over-claimed by cache parent ${remoteCacheResult.sourceRunId}`);
+                    }
                 }
                 return {
                     durationMs: Date.now() - start,
@@ -275,6 +295,39 @@ export class RunEvalStep {
     }
 }
 // ---------------------------------------------------------------------------
+// Object-checker discovery (D0057 / W0350)
+// ---------------------------------------------------------------------------
+const FIND_CHECKER_MAX_STEPS = 16;
+/**
+ * Walk a writer's decorator/fanout chain to feature-detect an
+ * `ArtifactObjectChecker`. The composition root wraps the backend in
+ * `AccumulatingArtifactWriter` (and optionally `InstrumentedArtifactWriter`)
+ * and layers GCS over local via `FanoutArtifactWriter`. Only
+ * `GcsArtifactWriter` implements `objectExists`, so on local-only / NoOp /
+ * gateway chains this returns null and the cache-hit restore skips pruning.
+ * `MAX_STEPS` is a cycle guard against a future decorator self-reference.
+ */
+function findObjectChecker(writer) {
+    const stack = [writer];
+    for (let steps = 0; stack.length > 0 && steps < FIND_CHECKER_MAX_STEPS; steps++) {
+        const cursor = stack.pop();
+        if (!cursor)
+            continue;
+        if (hasObjectExists(cursor))
+            return cursor;
+        if (cursor instanceof AccumulatingArtifactWriter)
+            stack.push(cursor.inner);
+        else if (cursor instanceof InstrumentedArtifactWriter)
+            stack.push(cursor.inner);
+        else if (cursor instanceof FanoutArtifactWriter)
+            stack.push(...cursor.delegates);
+    }
+    return null;
+}
+function hasObjectExists(w) {
+    return (typeof w.objectExists === "function");
+}
+// ---------------------------------------------------------------------------
 // Remote cache helpers
 // ---------------------------------------------------------------------------
 async function checkRemoteCache(fingerprint, reportStore, rootDir) {

package/dist/pipeline/cache-hit-restore.d.ts

@@ -8,7 +8,7 @@
  * @see docs/decisions/D0040-artifact-ref-source-run-id.md
  * @see docs/design-docs/cache-hit-artifact-restoration.md
  */
-import { type ArtifactManifest, type RunId } from "../_vendor/ailf-core/index.d.ts";
+import { type ArtifactManifest, type ArtifactObjectChecker, type RunId } from "../_vendor/ailf-core/index.d.ts";
 /**
  * Copy an artifact manifest verbatim and stamp `sourceRunId` on every ref
  * that doesn't already carry one.
@@ -47,3 +47,39 @@ import { type ArtifactManifest, type RunId } from "../_vendor/ailf-core/index.d.
 export declare function remapToCacheHitRefs(source: ArtifactManifest, opts: {
     sourceRunId: RunId;
 }): ArtifactManifest;
+/** Outcome of `pruneToResolvableRefs`. */
+export interface PruneResult {
+    /** Manifest with over-claimed entries/refs removed. */
+    readonly manifest: ArtifactManifest;
+    /** Per-entry entries dropped because their object did not exist. */
+    readonly droppedEntries: number;
+    /** Refs dropped entirely (bulk missing, or per-entry left with no entries). */
+    readonly droppedRefs: number;
+}
+/**
+ * Drop artifact refs (and per-entry entries) a cached report over-claimed —
+ * entries whose backing object was never written under the source run's
+ * storage prefix (D0040 / D0057, W0350).
+ *
+ * A degraded source run can publish a manifest that lists `rawResults`
+ * entries with no GCS object behind them; `remapToCacheHitRefs` copies those
+ * phantom entries forward into the new run's manifest, and the read side then
+ * signs URLs that 404 ("the specified key does not exist"). Pruning here, at
+ * the cache-hit restore boundary, removes the over-claim at the source so the
+ * written manifest's `entryCount` / `entries[]` reflect only artifacts that
+ * actually resolve — instead of pushing per-object HEAD checks onto the hot
+ * signing path (W0350 AC 3).
+ *
+ * Resolution mirrors the gateway: a per-entry object lives at
+ * `descriptor.objectPath(sourceRunId, entry.key)`, where `sourceRunId` is the
+ * runId encoded in `ref.path` (preferred — structurally tied to where bytes
+ * physically live) falling back to `ref.sourceRunId` (the lineage hint).
+ *
+ * **Fail open.** `checker.objectExists` throws when existence can't be
+ * determined (auth / network / quota). A throw KEEPS the ref/entry — we never
+ * drop a real artifact on a transient blip; the read side already tolerates a
+ * rare residual 404 (W0349). Only a definitive `false` drops an entry.
+ *
+ * Pure w.r.t. its inputs: returns a fresh manifest, never mutates `source`.
+ */
+export declare function pruneToResolvableRefs(source: ArtifactManifest, checker: ArtifactObjectChecker): Promise<PruneResult>;

package/dist/pipeline/cache-hit-restore.js

@@ -8,7 +8,7 @@
  * @see docs/decisions/D0040-artifact-ref-source-run-id.md
  * @see docs/design-docs/cache-hit-artifact-restoration.md
  */
-import { ARTIFACT_REGISTRY, } from "../_vendor/ailf-core/index.js";
+import { ARTIFACT_REGISTRY, runId as parseRunId, } from "../_vendor/ailf-core/index.js";
 /**
  * Copy an artifact manifest verbatim and stamp `sourceRunId` on every ref
  * that doesn't already carry one.
@@ -60,3 +60,110 @@ export function remapToCacheHitRefs(source, opts) {
     }
     return out;
 }
+/**
+ * Drop artifact refs (and per-entry entries) a cached report over-claimed —
+ * entries whose backing object was never written under the source run's
+ * storage prefix (D0040 / D0057, W0350).
+ *
+ * A degraded source run can publish a manifest that lists `rawResults`
+ * entries with no GCS object behind them; `remapToCacheHitRefs` copies those
+ * phantom entries forward into the new run's manifest, and the read side then
+ * signs URLs that 404 ("the specified key does not exist"). Pruning here, at
+ * the cache-hit restore boundary, removes the over-claim at the source so the
+ * written manifest's `entryCount` / `entries[]` reflect only artifacts that
+ * actually resolve — instead of pushing per-object HEAD checks onto the hot
+ * signing path (W0350 AC 3).
+ *
+ * Resolution mirrors the gateway: a per-entry object lives at
+ * `descriptor.objectPath(sourceRunId, entry.key)`, where `sourceRunId` is the
+ * runId encoded in `ref.path` (preferred — structurally tied to where bytes
+ * physically live) falling back to `ref.sourceRunId` (the lineage hint).
+ *
+ * **Fail open.** `checker.objectExists` throws when existence can't be
+ * determined (auth / network / quota). A throw KEEPS the ref/entry — we never
+ * drop a real artifact on a transient blip; the read side already tolerates a
+ * rare residual 404 (W0349). Only a definitive `false` drops an entry.
+ *
+ * Pure w.r.t. its inputs: returns a fresh manifest, never mutates `source`.
+ */
+export async function pruneToResolvableRefs(source, checker) {
+    const out = {};
+    let droppedEntries = 0;
+    let droppedRefs = 0;
+    for (const [type, ref] of Object.entries(source)) {
+        if (!ref)
+            continue;
+        const artifactType = type;
+        const descriptor = ARTIFACT_REGISTRY[artifactType];
+        // Bulk: a single object at ref.path.
+        if (ref.layout === "bulk") {
+            if (await existsOrKeep(checker, ref.path))
+                out[artifactType] = ref;
+            else
+                droppedRefs++;
+            continue;
+        }
+        // Per-entry: each entry is its own object under the source run prefix.
+        const sourceRunId = resolveSourceRunId(ref);
+        const entries = ref.entries ?? [];
+        if (!descriptor || sourceRunId === undefined || entries.length === 0) {
+            // Can't resolve per-entry object paths — fail open, keep verbatim.
+            out[artifactType] = ref;
+            continue;
+        }
+        const keptFlags = await Promise.all(entries.map(async (entry) => {
+            let objectPath;
+            try {
+                objectPath = descriptor.objectPath(sourceRunId, entry.key);
+            }
+            catch {
+                return true; // malformed key — fail open rather than drop
+            }
+            return existsOrKeep(checker, objectPath);
+        }));
+        const kept = entries.filter((_, i) => keptFlags[i]);
+        if (kept.length === entries.length) {
+            out[artifactType] = ref;
+            continue;
+        }
+        droppedEntries += entries.length - kept.length;
+        if (kept.length === 0) {
+            droppedRefs++; // nothing resolvable — drop the whole over-claimed ref
+            continue;
+        }
+        out[artifactType] = {
+            ...ref,
+            entries: kept,
+            entryCount: kept.length,
+            bytes: kept.reduce((sum, e) => sum + (e.bytes ?? 0), 0),
+        };
+    }
+    return { manifest: out, droppedEntries, droppedRefs };
+}
+/**
+ * Resolve where a ref's bytes physically live. Prefers the runId encoded in
+ * `ref.path` (validated through the canonical parser so a malformed manifest
+ * path can't propagate into a synthesized object name) over the
+ * `ref.sourceRunId` lineage hint — matching the gateway's resolution order.
+ */
+function resolveSourceRunId(ref) {
+    const fromPath = /^runs\/([^/]+)/.exec(ref.path)?.[1];
+    if (fromPath) {
+        const parsed = parseRunId(fromPath);
+        if (parsed.ok)
+            return parsed.value;
+    }
+    return ref.sourceRunId;
+}
+/**
+ * `checker.objectExists` wrapper that returns `true` (keep) on a thrown,
+ * indeterminate result — only a definitive `false` drops the artifact.
+ */
+async function existsOrKeep(checker, path) {
+    try {
+        return await checker.objectExists(path);
+    }
+    catch {
+        return true;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "7.2.2",
+  "version": "7.2.3",
   "private": false,
   "publishConfig": {
     "access": "public"