@sanity/ailf 7.2.1 → 7.2.3

package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts

@@ -76,6 +76,28 @@ export interface ArtifactWriter {
      */
     writePerEntry(type: ArtifactType, runId: RunId, entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
 }
+/**
+ * Optional capability: check whether an object physically exists at a
+ * store-relative `path`. Kept **separate** from `ArtifactWriter` so writers
+ * opt in by structural typing rather than every implementor being forced to
+ * provide it — consumers feature-detect it (see the writer-chain walkers in
+ * the eval package).
+ *
+ * Used by the cache-hit restore path (D0040 / D0057) to drop artifact refs
+ * a prior report over-claimed: a degraded source run can list `rawResults`
+ * entries whose GCS objects were never written, and `remapToCacheHitRefs`
+ * would otherwise copy those phantom entries into the new run's manifest.
+ *
+ * Contract:
+ *   - resolves `true`  — the object exists.
+ *   - resolves `false` — the object is *definitively* absent.
+ *   - **throws** — existence could not be determined (auth / network /
+ *     transient error). Callers must treat a throw as "unknown" and
+ *     fail open (keep the ref) rather than dropping a real artifact.
+ */
+export interface ArtifactObjectChecker {
+    objectExists(path: string): Promise<boolean>;
+}
 /**
  * Thrown by writers that can't satisfy a method — e.g. an
  * `ApiGatewayArtifactWriter` cannot implement `appendNdjson` until the batch

package/dist/_vendor/ailf-core/ports/index.d.ts

@@ -4,7 +4,7 @@
  * Ports define the contracts between the domain kernel and the outside world.
  * Adapters (in packages/eval) implement these interfaces.
  */
-export type { ArtifactEntry, ArtifactWriter } from "./artifact-writer.js";
+export type { ArtifactEntry, ArtifactObjectChecker, ArtifactWriter, } from "./artifact-writer.js";
 export { NoOpArtifactWriter } from "./artifact-writer.js";
 export type { CacheEntryMetadata, CacheKey, CacheLookupResult, CacheRecordInput, CacheStore, } from "./cache-store.js";
 export type { ConfigSource } from "./config-source.js";

package/dist/_vendor/ailf-core/services/diagnosis/cards/top-recommendations.js

@@ -59,6 +59,20 @@ export const generateTopRecommendations = async (report, ctx) => {
     }
     // Build allow-list from the runtime report
     const allowList = buildDocSlugAllowList(report);
+    // Short-circuit BEFORE the LLM call when the allow-list is empty — the shape
+    // a degraded run produces (enrichment never wrote `documentManifest`, and
+    // per-score `documents[]` carry no slugs). With an empty allow-list the
+    // system prompt instructs the model to omit every recommendation, so it
+    // returns `suggestions: []` and trips the `.min(1)` schema, surfacing as a
+    // degraded card with a raw ZodError. Degrade gracefully to "missing"
+    // instead — same structural seam the sibling attribution cards use.
+    if (allowList.size === 0) {
+        return {
+            status: "missing",
+            cardType: "top-recommendations",
+            reason: "no doc-slug allow-list for this run (enrichment incomplete)",
+        };
+    }
     // Per-call schema: additive docSlug allow-list refine (AI-SPEC §3 Pitfall 1)
     const PerCallSchema = z.object({
         summary: z.string().min(1).max(500),

package/dist/_vendor/ailf-shared/generated/help-content.js

@@ -142,7 +142,7 @@ export const HELP_TOPICS = [
     {
         "id": "glossary",
         "title": "Glossary",
-        "body": "**Overall Score**\n: A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).\n\n**Doc Lift**\n: How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.\n\n**Actual Score**\n: How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.\n\n**Retrieval Gap**\n: The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.\n\n**Infra Efficiency**\n: What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.\n\n**Floor**\n: Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.\n\n**Ceiling**\n: Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.\n\n**Actual**\n: Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.\n\n**Ret. Gap**\n: Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.\n\n**Efficiency**\n: What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).\n\n**Inverted Retrieval Gap**\n: ⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.\n\n**Score**\n: Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.\n\n**Task Completion**\n: Can the LLM implement the requested feature? Graded 0–100.\n\n**Code Correctness**\n: Is the generated code idiomatic, correct, and following best practices? Graded 0–100.\n\n**Doc Coverage**\n: Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.\n\n**Tests**\n: Number of test cases in this feature area.\n\n**Overall Δ**\n: Change in overall score between the two runs. Positive means the experiment scored higher.\n\n**Actual Δ**\n: Change in actual (agent-retrieved) score between runs. Positive means agents did better.\n\n**Ret. Gap Δ**\n: Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.\n\n**Efficiency Δ**\n: Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.\n\n**Baseline**\n: The reference run you're comparing against.\n\n**Experiment**\n: The new run you're evaluating.\n\n**Delta**\n: Difference between experiment and baseline. Positive means improvement, negative means regression.\n\n**Change**\n: Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).\n\n**Low-Scoring Judgments**\n: The grading model's explanations for tests that scored below 70/100.\n\n**Judgment Reason**\n: The grading model's natural language explanation of what went wrong.\n\n**Strong (80+)**\n: Feature areas scoring 80 or above. The docs are working well for these features — AI agents produce correct, complete implementations.\n\n**Needs Attention (70–79)**\n: Feature areas scoring 70–79. These are okay but could be improved — there may be gaps in specific dimensions like doc coverage or code correctness.\n\n**Weak (<70)**\n: Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.\n\n**Negative Doc Lift**\n: Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.\n\n**Weak Areas**\n: Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.\n\n**Docs Hurt Performance**\n: Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.\n\n**Retrieval Issues**\n: Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.\n\n**Dimension Weaknesses**\n: Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).\n\n**Efficiency Anomalies**\n: Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.\n\n**Doc Lift Wins**\n: Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.\n\n**Retrieval Excellence**\n: Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.\n\n**Model Breakdown**\n: Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently — useful for spotting models that struggle with specific feature areas.\n\n**Strengths**\n: What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.\n\n**Recommendations**\n: Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.\n\n**Total Potential Lift**\n: Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate — each gap targets the median of non-bottlenecked dimensions, not 100.\n\n**Failure Mode**\n: The type of failure the grader emitted. Cross-cutting modes apply to any dimension (api-error, model-limitation, false-floor, unclassified). Per-dimension extensions cover documentation problems (missing-docs, incorrect-docs, outdated-docs, poor-structure), spec adherence (spec-mismatch), tool use (tool-misuse, chaotic-process, missing-recovery), and knowledge probes (factual-error, incompleteness, currency-violation, hallucination).\n\n**Estimated Lift**\n: Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.\n\n**Confidence**\n: How confident we are in this diagnosis (D0049 ceiling-cross-check derivation). High = the grader's emitted failure mode agrees with the structural ceiling-decomposition signal. Medium = signals disagree (or the ceiling pattern is not informative for this score). Low = passing scores never classify; treat as absent.\n\n**Agent Behavior**\n: How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.\n\n**Search Queries**\n: The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.\n\n**Unique Doc Slugs**\n: Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.\n\n**External Domains**\n: Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.\n\n**Avg Pages Visited**\n: Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.\n\n**Avg Searches**\n: Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.\n\n**Avg Network Time**\n: Average time spent on network requests per test. Includes page fetches, search queries, and API calls.\n\n**Total Requests**\n: Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.\n\n**Total Bytes Downloaded**\n: Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.\n\n**Task Completion Δ**\n: Change in task completion between runs. Positive means implementations are more complete.\n\n**Code Correctness Δ**\n: Change in code correctness between runs. Positive means better code quality.\n\n**Doc Coverage Δ**\n: Change in doc coverage between runs. Positive means the docs are providing more useful information.\n\n**Area Δ**\n: Score change for this area compared to the previous evaluation run.\n\n**Production**\n: Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.\n\n**Branch**\n: Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.\n\n**Local**\n: Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.\n\n**Score**\n: The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.\n\n**Mode**\n: The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.\n\n**Trigger**\n: What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.\n\n**Baseline**\n: Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).\n\n**Full**\n: Full mode — runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.\n\n**Agentic**\n: Agentic mode — the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?\n\n**Observed**\n: Observed mode — records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.\n\n**Debug**\n: Debug mode — a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.\n\n**Manual**\n: Manually triggered — someone ran the evaluation pipeline by hand, either locally or via the Studio UI.\n\n**CI**\n: CI-triggered — the evaluation ran automatically as part of a pull request or merge pipeline.\n\n**Scheduled**\n: Scheduled — the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.\n\n**Webhook**\n: Webhook-triggered — a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.\n\n**Cross-Repo**\n: Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
+        "body": "**Overall Score**\n: A weighted average across all feature areas, using the gold scoring profile: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%).\n\n**Doc Lift**\n: How much the docs help, compared to the model's training data alone. Calculated as ceiling minus floor, where ceiling includes Doc Coverage and floor does not. Higher is better.\n\n**Actual Score**\n: How well an AI agent scores when it has to find docs on its own through web search and page fetching. This is the real-world scenario. Only available in full mode.\n\n**Retrieval Gap**\n: The score lost because agents can't find or use all the relevant docs. Calculated as ceiling minus actual. Lower is better; zero means agents find everything.\n\n**Infra Efficiency**\n: What percentage of the docs' potential quality actually reaches agents (actual ÷ ceiling). 100% means agents find and use all relevant docs perfectly.\n\n**Floor**\n: Output-quality composite without documentation — Task Completion (60%) and Code Correctness (40%) only. Doc Coverage is excluded because it's undefined when no docs are provided. This tells you what the model already knows from its training data.\n\n**Ceiling**\n: Score with gold-standard docs injected directly into the prompt. This is the best the documentation can do.\n\n**Actual**\n: Score when an AI agent finds docs on its own through web search and page fetching. This is the real-world experience.\n\n**Ret. Gap**\n: Quality lost to discoverability (ceiling minus actual). The gap between what the docs could deliver and what agents actually get.\n\n**Efficiency**\n: What fraction of the docs' quality reaches agents in practice (actual ÷ ceiling, shown as a percentage).\n\n**Inverted Retrieval Gap**\n: ⚠️ Inverted retrieval gap: agents that can't find the docs actually score higher, because the docs hurt performance. This usually means there's a doc quality problem.\n\n**Score**\n: Ceiling composite for this feature area: Task Completion × 50% + Code Correctness × 25% + Doc Coverage × 25%.\n\n**Task Completion**\n: Can the LLM implement the requested feature? Graded 0–100.\n\n**Code Correctness**\n: Is the generated code idiomatic, correct, and following best practices? Graded 0–100.\n\n**Doc Coverage**\n: Did the docs provide the information needed to implement the feature? Graded 0–100. This dimension only contributes to the ceiling composite (with docs) — it's excluded from the floor composite because it's undefined without documentation.\n\n**Tests**\n: Number of test cases in this feature area.\n\n**Overall Δ**\n: Change in overall score between the two runs. Positive means the experiment scored higher.\n\n**Actual Δ**\n: Change in actual (agent-retrieved) score between runs. Positive means agents did better.\n\n**Ret. Gap Δ**\n: Change in retrieval gap between runs. Negative is good here: it means the gap shrank and agents found more relevant docs.\n\n**Efficiency Δ**\n: Change in infrastructure efficiency between runs. Positive means agents are capturing more of the docs' potential.\n\n**Baseline**\n: The reference run you're comparing against.\n\n**Experiment**\n: The new run you're evaluating.\n\n**Delta**\n: Difference between experiment and baseline. Positive means improvement, negative means regression.\n\n**Change**\n: Whether the change is meaningful: improved, regressed, or unchanged (within the noise threshold).\n\n**Low-Scoring Judgments**\n: The grading model's explanations for tests that scored below 70/100.\n\n**Judgment Reason**\n: The grading model's natural language explanation of what went wrong.\n\n**Strong (80+)**\n: Feature areas scoring 80 or above. The docs are working well for these features — AI agents produce correct, complete implementations.\n\n**Needs Attention (70–79)**\n: Feature areas scoring 70–79. These are okay but could be improved — there may be gaps in specific dimensions like doc coverage or code correctness.\n\n**Weak (<70)**\n: Feature areas scoring below 70. The docs are not providing enough support for AI agents to implement these features correctly.\n\n**Negative Doc Lift**\n: Number of areas where the documentation actually hurts AI performance — the model scores higher without docs than with them. This usually means the docs contain outdated patterns or incorrect examples.\n\n**Weak Areas**\n: Feature areas where the overall score is below 70. These need the most attention — low scores mean AI agents consistently struggle to implement these features.\n\n**Docs Hurt Performance**\n: Areas where the floor score (no docs) is higher than the ceiling score (with docs). The documentation may be actively misleading the model. These docs should be reviewed.\n\n**Retrieval Issues**\n: Areas where AI agents can find less than 70% of the available doc quality. The docs exist and are good, but agents can't discover them through search. Consider improving page titles, metadata, or search engine indexing.\n\n**Dimension Weaknesses**\n: Individual grading dimensions scoring below 50 within an area. These are the specific skills where AI agents fail most — task completion (can it build the feature?), code correctness (is the code right?), or doc coverage (did it use the docs?).\n\n**Efficiency Anomalies**\n: Areas where agent efficiency exceeds 100% — meaning agents perform better with self-found docs than with gold-standard docs injected directly. This can indicate doc quality issues (injected docs confuse the model) or agent memorization.\n\n**Doc Lift Wins**\n: Areas where documentation boosts AI performance by 5 or more points. Higher doc lift means the docs are providing crucial information that the model doesn't already know.\n\n**Retrieval Excellence**\n: Areas where AI agents successfully find and use at least 85% of the available doc quality through web search. Good retrieval means your docs are well-indexed and easy for agents to discover.\n\n**Model Breakdown**\n: Break down scores by individual LLM model. The default 'All Models' view shows the cross-model average. Select a specific model to see how it performed independently — useful for spotting models that struggle with specific feature areas.\n\n**Strengths**\n: What's working well: high-scoring areas, dimensions where the docs are strong, and areas where AI agents successfully find and use the documentation.\n\n**Recommendations**\n: Prioritized remediation plan from gap analysis. Each recommendation identifies a documentation problem, the affected feature area, and the estimated score lift from fixing it.\n\n**Total Potential Lift**\n: Aggregate potential score lift if all identified gaps were fixed. This is a conservative estimate — each gap targets the median of non-bottlenecked dimensions, not 100.\n\n**Failure Mode**\n: The type of failure the grader emitted. Cross-cutting modes apply to any dimension (api-error, model-limitation, false-floor, unclassified). Per-dimension extensions cover documentation problems (missing-docs, incorrect-docs, outdated-docs, poor-structure), spec adherence (spec-mismatch), tool use (tool-misuse, chaotic-process, missing-recovery), and knowledge probes (factual-error, incompleteness, currency-violation, hallucination).\n\n**Estimated Lift**\n: Estimated composite score improvement if this gap is fully fixed. Based on raising bottleneck dimensions to the median of non-bottlenecked dimensions.\n\n**Confidence**\n: How confident we are in this diagnosis (D0049 ceiling-cross-check derivation). High = the grader's emitted failure mode agrees with the structural ceiling-decomposition signal. Medium = signals disagree (or the ceiling pattern is not informative for this score). Low = passing scores never classify; treat as absent.\n\n**Agent Behavior**\n: How AI agents interacted with your documentation during evaluation: what they searched for, which pages they visited, and how much time they spent on network requests.\n\n**Search Queries**\n: The exact search queries agents used to find documentation. Helps you understand how agents discover your content and whether your docs appear for relevant queries.\n\n**Unique Doc Slugs**\n: Documentation page slugs that agents actually visited during evaluation. Compare against canonical docs to see if agents found the right pages.\n\n**External Domains**\n: Non-Sanity domains that agents contacted during evaluation. High external domain counts may indicate agents couldn't find what they needed in your docs.\n\n**Avg Pages Visited**\n: Average number of documentation pages visited per test. Higher counts can mean agents need to consult many pages (complex task) or can't find the right one quickly.\n\n**Avg Searches**\n: Average number of web searches performed per test. High search counts can indicate docs are hard to discover through search engines.\n\n**Avg Network Time**\n: Average time spent on network requests per test. Includes page fetches, search queries, and API calls.\n\n**Total Requests**\n: Total number of HTTP requests the agent made during the test, including searches, page visits, and API calls.\n\n**Total Bytes Downloaded**\n: Total bytes downloaded by the agent. Large downloads may indicate the agent is fetching many pages or very large documents.\n\n**Task Completion Δ**\n: Change in task completion between runs. Positive means implementations are more complete.\n\n**Code Correctness Δ**\n: Change in code correctness between runs. Positive means better code quality.\n\n**Doc Coverage Δ**\n: Change in doc coverage between runs. Positive means the docs are providing more useful information.\n\n**Area Δ**\n: Score change for this area compared to the previous evaluation run.\n\n**Production**\n: Production source — docs fetched from the live production dataset. Scores reflect what real users and AI agents experience today.\n\n**Branch**\n: Branch source — docs fetched from a branch or draft dataset. Use this to preview how content changes affect scores before publishing.\n\n**Local**\n: Local source — docs fetched from local files or a local dev server. Useful for testing doc changes before pushing.\n\n**Score**\n: The overall ceiling composite for this evaluation run: Task Completion (50%), Code Correctness (25%), and Doc Coverage (25%), averaged across all feature areas.\n\n**Mode**\n: The evaluation mode determines which reference points are measured. Different modes test different aspects of how AI agents interact with documentation.\n\n**Trigger**\n: What initiated this evaluation run. Knowing the trigger helps you understand whether a score change was from a content edit, a code deploy, or a scheduled check.\n\n**Baseline**\n: Baseline mode — tests the model with gold-standard docs injected directly. Measures ceiling performance (best the docs can do).\n\n**Full**\n: Full mode — runs baseline + agentic. Compares ceiling (injected docs) against actual (agent-retrieved docs) to measure retrieval gap and infrastructure efficiency.\n\n**Agentic**\n: Agentic mode — the AI agent finds docs on its own via web search. Measures real-world performance: can agents actually discover and use your documentation?\n\n**Observed**\n: Observed mode — records how agents interact with docs without scoring. Captures search queries, pages visited, and browsing patterns for analysis.\n\n**Debug**\n: Debug mode — a diagnostic run for pipeline development. May use non-standard configurations or limited task sets.\n\n**Manual**\n: Manually triggered — someone ran the evaluation pipeline by hand, either locally or via the Studio UI.\n\n**CI**\n: CI-triggered — the evaluation ran automatically as part of a pull request or merge pipeline.\n\n**Scheduled**\n: Scheduled — the evaluation ran on a recurring schedule (e.g. nightly or weekly) to track score trends over time.\n\n**Webhook**\n: Webhook-triggered — a content change in Sanity triggered the evaluation automatically. Helps catch doc regressions early.\n\n**Cross-Repo**\n: Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.\n\n**Gold**\n: Gold variant — the relevant docs were injected into the prompt as context. This is the ceiling condition: the best the documentation can do.\n\n**Baseline**\n: Baseline variant — no documentation in the prompt. This is the floor condition: what the model already knows from its training data, used as the control for doc lift.\n\n**Naive**\n: Naive — the in-house agentic harness with naive prompting. The model runs in an agent loop with no doc-targeting strategy.\n\n**Optimized**\n: Optimized — the in-house agentic harness with optimized prompting. The model runs in an agent loop tuned for documentation retrieval.\n\n**Normal**\n: Normal — a direct vendor-API call. The model produces a single-shot completion with no agent loop.\n\n**Fail**\n: Fail — the model produced no usable output (empty response, API error, or token exhaustion). Distinct from a low score on output that was produced.\n\n**Low dim**\n: Low dim — the run produced output but at least one grading dimension scored below 60.\n\n**OK**\n: OK — the run produced output and every grading dimension scored 60 or above.",
         "source": "packages/shared/src/glossary.ts",
         "tags": [
             "reference",

package/dist/_vendor/ailf-shared/glossary.d.ts

@@ -314,5 +314,37 @@ export declare const GLOSSARY: {
         readonly label: "Cross-Repo";
         readonly long: "Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.";
     };
+    readonly variantGold: {
+        readonly label: "Gold";
+        readonly long: "Gold variant — the relevant docs were injected into the prompt as context. This is the ceiling condition: the best the documentation can do.";
+    };
+    readonly variantBaseline: {
+        readonly label: "Baseline";
+        readonly long: "Baseline variant — no documentation in the prompt. This is the floor condition: what the model already knows from its training data, used as the control for doc lift.";
+    };
+    readonly engineNaive: {
+        readonly label: "Naive";
+        readonly long: "Naive — the in-house agentic harness with naive prompting. The model runs in an agent loop with no doc-targeting strategy.";
+    };
+    readonly engineOptimized: {
+        readonly label: "Optimized";
+        readonly long: "Optimized — the in-house agentic harness with optimized prompting. The model runs in an agent loop tuned for documentation retrieval.";
+    };
+    readonly engineNormal: {
+        readonly label: "Normal";
+        readonly long: "Normal — a direct vendor-API call. The model produces a single-shot completion with no agent loop.";
+    };
+    readonly statusFail: {
+        readonly label: "Fail";
+        readonly long: "Fail — the model produced no usable output (empty response, API error, or token exhaustion). Distinct from a low score on output that was produced.";
+    };
+    readonly statusLowDim: {
+        readonly label: "Low dim";
+        readonly long: "Low dim — the run produced output but at least one grading dimension scored below 60.";
+    };
+    readonly statusOk: {
+        readonly label: "OK";
+        readonly long: "OK — the run produced output and every grading dimension scored 60 or above.";
+    };
 };
 export type GlossarySlug = keyof typeof GLOSSARY;

package/dist/_vendor/ailf-shared/glossary.js

@@ -327,4 +327,39 @@ export const GLOSSARY = {
         label: "Cross-Repo",
         long: "Cross-repo — triggered from another repository via the dispatch API. Used when external repos want to validate their docs against AILF tasks.",
     },
+    // -- Variant values (per-test docs condition) ------------------------------
+    variantGold: {
+        label: "Gold",
+        long: "Gold variant — the relevant docs were injected into the prompt as context. This is the ceiling condition: the best the documentation can do.",
+    },
+    variantBaseline: {
+        label: "Baseline",
+        long: "Baseline variant — no documentation in the prompt. This is the floor condition: what the model already knows from its training data, used as the control for doc lift.",
+    },
+    // -- Execution mode values (how the model was driven) ----------------------
+    engineNaive: {
+        label: "Naive",
+        long: "Naive — the in-house agentic harness with naive prompting. The model runs in an agent loop with no doc-targeting strategy.",
+    },
+    engineOptimized: {
+        label: "Optimized",
+        long: "Optimized — the in-house agentic harness with optimized prompting. The model runs in an agent loop tuned for documentation retrieval.",
+    },
+    engineNormal: {
+        label: "Normal",
+        long: "Normal — a direct vendor-API call. The model produces a single-shot completion with no agent loop.",
+    },
+    // -- Status values (per-test outcome) --------------------------------------
+    statusFail: {
+        label: "Fail",
+        long: "Fail — the model produced no usable output (empty response, API error, or token exhaustion). Distinct from a low score on output that was produced.",
+    },
+    statusLowDim: {
+        label: "Low dim",
+        long: "Low dim — the run produced output but at least one grading dimension scored below 60.",
+    },
+    statusOk: {
+        label: "OK",
+        long: "OK — the run produced output and every grading dimension scored 60 or above.",
+    },
 };

package/dist/artifact-capture/fanout-artifact-writer.d.ts

@@ -38,6 +38,14 @@ export declare class FanoutArtifactWriter implements ArtifactWriter {
     private readonly writers;
     private readonly progress?;
     constructor(writers: readonly ArtifactWriter[], options?: FanoutArtifactWriterOptions);
+    /**
+     * The delegate writers in declaration order. Exposed read-only so callers
+     * walking the writer chain (e.g. to feature-detect an
+     * `ArtifactObjectChecker`) can descend into the fanout without it having to
+     * re-implement every optional capability. Mirrors the decorators' readonly
+     * `inner` accessor.
+     */
+    get delegates(): readonly ArtifactWriter[];
     private reportProgress;
     emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
     appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;

package/dist/artifact-capture/fanout-artifact-writer.js

@@ -33,6 +33,16 @@ export class FanoutArtifactWriter {
         this.writers = writers;
         this.progress = options.progress;
     }
+    /**
+     * The delegate writers in declaration order. Exposed read-only so callers
+     * walking the writer chain (e.g. to feature-detect an
+     * `ArtifactObjectChecker`) can descend into the fanout without it having to
+     * re-implement every optional capability. Mirrors the decorators' readonly
+     * `inner` accessor.
+     */
+    get delegates() {
+        return this.writers;
+    }
     reportProgress(ref) {
         if (!this.progress)
             return;

package/dist/artifact-capture/gcs-artifact-writer.d.ts

@@ -28,7 +28,7 @@
  * @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
  */
 import { Storage } from "@google-cloud/storage";
-import { type ArtifactEntry, type ArtifactRef, type ArtifactType, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssociationValues, type RunId, type RunManifest, type WriteSource } from "../_vendor/ailf-core/index.d.ts";
+import { type ArtifactEntry, type ArtifactObjectChecker, type ArtifactRef, type ArtifactType, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssociationValues, type RunId, type RunManifest, type WriteSource } from "../_vendor/ailf-core/index.d.ts";
 import { type UploadMetricsSink } from "./upload-metrics.js";
 export interface GcsArtifactWriterOptions {
     /** GCS bucket name (e.g., "ailf-artifacts") */
@@ -61,7 +61,7 @@ export interface GcsArtifactWriterOptions {
      */
     writerSource?: WriteSource;
 }
-export declare class GcsArtifactWriter implements ArtifactWriter {
+export declare class GcsArtifactWriter implements ArtifactWriter, ArtifactObjectChecker {
     private client;
     private readonly options;
     private readonly ndjsonStreams;
@@ -83,6 +83,16 @@ export declare class GcsArtifactWriter implements ArtifactWriter {
     emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
     appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;
     writeManifest(runId: RunId, manifest: RunManifest): Promise<ArtifactRef | null>;
+    /**
+     * Existence probe used by the cache-hit restore prune (D0057). Unlike the
+     * write methods (P5 non-blocking — swallow errors, return null), this
+     * resolves `false` ONLY for a definitively-absent object and **throws** on
+     * any other failure (auth / network / quota) so the caller can fail open
+     * and keep the ref rather than dropping a real artifact on a transient
+     * blip. `file.exists()` rejects only on real errors; a missing object
+     * resolves `[false]`.
+     */
+    objectExists(path: string): Promise<boolean>;
     /** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
     writeBulk(type: ArtifactType, runId: RunId, data: unknown): Promise<ArtifactRef | null>;
     /** @deprecated Use `emit()` per entry instead. */

package/dist/artifact-capture/gcs-artifact-writer.js

@@ -223,6 +223,24 @@ export class GcsArtifactWriter {
             this.reportProgress(ref);
         return ref;
     }
+    // ---- ArtifactObjectChecker (D0057) --------------------------------------
+    /**
+     * Existence probe used by the cache-hit restore prune (D0057). Unlike the
+     * write methods (P5 non-blocking — swallow errors, return null), this
+     * resolves `false` ONLY for a definitively-absent object and **throws** on
+     * any other failure (auth / network / quota) so the caller can fail open
+     * and keep the ref rather than dropping a real artifact on a transient
+     * blip. `file.exists()` rejects only on real errors; a missing object
+     * resolves `[false]`.
+     */
+    async objectExists(path) {
+        const storage = this.getClient();
+        const [exists] = await storage
+            .bucket(this.options.bucket)
+            .file(path)
+            .exists();
+        return exists;
+    }
     // ---- Deprecated legacy surface (W0052) ----------------------------------
     /** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
     async writeBulk(type, runId, data) {

package/dist/orchestration/required-eval-runs.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Which eval sub-runs a pipeline configuration requires, and whether every one
+ * of them was satisfied by a remote-cache hit.
+ *
+ * The post-scoring enrichment steps (gap-analysis, compute-attribution) may
+ * skip benignly when grader judgments are absent because ALL required runs came
+ * from the remote cache — a cache hit restores `score-summary.json` but never
+ * writes `grader-judgments.json`. They must NOT skip when at least one required
+ * run was evaluated fresh this pipeline: a fresh run that scored tests yet
+ * persisted no judgments is a degraded outcome that has to fail loud.
+ *
+ * Mirrors the required-run derivation in `calculate-scores-step` so the
+ * "all required runs cached" judgement is defined in exactly one place.
+ */
+interface EvalRunSelector {
+    mode: string;
+    variant?: string | null;
+}
+interface RemoteCacheState {
+    remoteCacheHits?: ReadonlySet<string>;
+}
+/**
+ * The eval sub-runs a configuration requires, keyed by the same strings
+ * `RunEvalStep` records in `state.remoteCacheHits` (`"baseline"`, `"agentic"`,
+ * or the bare mode name for non-literacy modes).
+ */
+export declare function requiredEvalRuns(config: EvalRunSelector): string[];
+/**
+ * True only when every eval sub-run the configuration requires was satisfied by
+ * a remote-cache hit. A cache hit on a subset of required runs (e.g. agentic
+ * cached, baseline fresh) returns false — the fresh run's outputs are still the
+ * pipeline's responsibility.
+ */
+export declare function allRequiredEvalRunsCached(config: EvalRunSelector, state: RemoteCacheState | undefined): boolean;
+export {};

package/dist/orchestration/required-eval-runs.js

@@ -0,0 +1,41 @@
+/**
+ * Which eval sub-runs a pipeline configuration requires, and whether every one
+ * of them was satisfied by a remote-cache hit.
+ *
+ * The post-scoring enrichment steps (gap-analysis, compute-attribution) may
+ * skip benignly when grader judgments are absent because ALL required runs came
+ * from the remote cache — a cache hit restores `score-summary.json` but never
+ * writes `grader-judgments.json`. They must NOT skip when at least one required
+ * run was evaluated fresh this pipeline: a fresh run that scored tests yet
+ * persisted no judgments is a degraded outcome that has to fail loud.
+ *
+ * Mirrors the required-run derivation in `calculate-scores-step` so the
+ * "all required runs cached" judgement is defined in exactly one place.
+ */
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
+/**
+ * The eval sub-runs a configuration requires, keyed by the same strings
+ * `RunEvalStep` records in `state.remoteCacheHits` (`"baseline"`, `"agentic"`,
+ * or the bare mode name for non-literacy modes).
+ */
+export function requiredEvalRuns(config) {
+    if (config.mode === "literacy") {
+        const variant = config.variant ?? LiteracyVariant.STANDARD;
+        return variant === LiteracyVariant.FULL
+            ? [LiteracyVariant.STANDARD, LiteracyVariant.AGENTIC]
+            : [variant];
+    }
+    return [config.mode];
+}
+/**
+ * True only when every eval sub-run the configuration requires was satisfied by
+ * a remote-cache hit. A cache hit on a subset of required runs (e.g. agentic
+ * cached, baseline fresh) returns false — the fresh run's outputs are still the
+ * pipeline's responsibility.
+ */
+export function allRequiredEvalRunsCached(config, state) {
+    const hits = state?.remoteCacheHits;
+    if (!hits || hits.size === 0)
+        return false;
+    return requiredEvalRuns(config).every((run) => hits.has(run));
+}

package/dist/orchestration/steps/calculate-scores-step.js

@@ -11,6 +11,7 @@ import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { buildCacheContext } from "../cache-context.js";
+import { allRequiredEvalRunsCached } from "../required-eval-runs.js";
 import { calculateAndWriteScores } from "../../pipeline/calculate-scores.js";
 import { checkResultsExist, checkScoreSummaryValid, } from "../../pipeline/checks.js";
 import { resultsFileForMode } from "../../pipeline/eval-constants.js";
@@ -27,30 +28,22 @@ export class CalculateScoresStep {
     }
     async execute(ctx, state) {
         const start = Date.now();
-        // When all required eval modes were satisfied by remote cache hits,
+        // When all required eval runs were satisfied by remote cache hits,
         // score-summary.json was already restored from the cached report.
-        // Skip re-calculation — the raw eval-results files don't exist.
-        if (state.remoteCacheHits?.size) {
-            // For literacy mode, determine required eval runs from variant
-            const variant = ctx.config.variant ?? LiteracyVariant.STANDARD;
-            const requiredRuns = ctx.config.mode === "literacy" && variant === LiteracyVariant.FULL
-                ? [LiteracyVariant.STANDARD, LiteracyVariant.AGENTIC]
-                : ctx.config.mode === "literacy"
-                    ? [variant]
-                    : [ctx.config.mode];
-            const allCached = requiredRuns.every((m) => state.remoteCacheHits.has(m));
-            if (allCached) {
-                // Verify the restored score-summary.json is valid
-                const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
-                const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
-                if (summaryErrors.length === 0) {
-                    return {
-                        reason: "Remote cache hit — score-summary.json restored from cached report",
-                        status: "skipped",
-                    };
-                }
-                // If the summary is invalid, fall through to normal calculation
+        // Skip re-calculation — the raw eval-results files don't exist. A partial
+        // cache hit (only some required runs cached) falls through to normal
+        // calculation: the freshly-run sub-evals produced raw results to score.
+        if (allRequiredEvalRunsCached(ctx.config, state)) {
+            // Verify the restored score-summary.json is valid
+            const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
+            const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+            if (summaryErrors.length === 0) {
+                return {
+                    reason: "Remote cache hit — score-summary.json restored from cached report",
+                    status: "skipped",
+                };
             }
+            // If the summary is invalid, fall through to normal calculation
         }
         // Primary results file to score.
         // For literacy: "full" variant uses baseline as primary; others use variant directly.

package/dist/orchestration/steps/compute-attribution-step.js

@@ -41,6 +41,7 @@ import { isSlugRef } from "../../_vendor/ailf-core/index.js";
 import { calibrationSetVersion, embeddingModel, ensembleVersion, } from "../../pipeline/attribution.js";
 import { V0_WEIGHTS, computeJudgmentAttribution, } from "../../pipeline/compute-attribution.js";
 import { classifyEnrichmentInputs, degradedEnrichmentError, } from "../../pipeline/enrichment-preconditions.js";
+import { allRequiredEvalRunsCached } from "../required-eval-runs.js";
 // ---------------------------------------------------------------------------
 // Step implementation
 // ---------------------------------------------------------------------------
@@ -89,10 +90,12 @@ export class ComputeAttributionStep {
         // grader judgments is a degraded run, not a benign skip. Fail loud so the
         // outcome surfaces in pipeline-result and on the job document. A remote
         // cache hit restores score-summary.json without grader-judgments.json, so
-        // its missing judgments are legitimate — never fail loud on a cache hit.
-        const fromRemoteCache = (state?.remoteCacheHits?.size ?? 0) > 0;
+        // missing judgments are legitimate ONLY when every required sub-eval came
+        // from the cache — a hybrid full run with a freshly-evaluated sub-eval that
+        // persisted no judgments is still degraded.
+        const allCached = allRequiredEvalRunsCached(ctx.config, state);
         const inputs = classifyEnrichmentInputs(root);
-        if (inputs.kind === "judgments-missing-after-eval" && !fromRemoteCache) {
+        if (inputs.kind === "judgments-missing-after-eval" && !allCached) {
             return {
                 durationMs: Date.now() - start,
                 status: "failed",

package/dist/orchestration/steps/gap-analysis-step.js

@@ -19,6 +19,7 @@ import { join, resolve } from "path";
 import { assoc, isSlugRef } from "../../_vendor/ailf-core/index.js";
 import { emitFileContents } from "../../artifact-capture/emit-file.js";
 import { classifyEnrichmentInputs, degradedEnrichmentError, } from "../../pipeline/enrichment-preconditions.js";
+import { allRequiredEvalRunsCached } from "../required-eval-runs.js";
 export class GapAnalysisStep {
     name = "gap-analysis";
     optional = true;
@@ -46,11 +47,14 @@ export class GapAnalysisStep {
         // reports publish with a score but no test details.
         //
         // A remote cache hit restores score-summary.json (with testCount) from a
-        // prior report but never writes grader-judgments.json, so judgments are
-        // legitimately absent — that is a benign skip, not a degraded full eval.
-        const fromRemoteCache = (state?.remoteCacheHits?.size ?? 0) > 0;
+        // prior report but never writes grader-judgments.json, so absent judgments
+        // are legitimate ONLY when every required sub-eval came from the cache. In a
+        // hybrid full run (e.g. agentic cached, baseline evaluated fresh) the fresh
+        // run's missing judgments are still degraded — gate the skip on ALL required
+        // runs being cached, not merely any.
+        const allCached = allRequiredEvalRunsCached(ctx.config, state);
         const inputs = classifyEnrichmentInputs(root);
-        if (inputs.kind === "judgments-missing-after-eval" && !fromRemoteCache) {
+        if (inputs.kind === "judgments-missing-after-eval" && !allCached) {
             return {
                 durationMs: Date.now() - start,
                 status: "failed",

package/dist/orchestration/steps/run-eval-step.js

@@ -11,9 +11,11 @@ import { emitPerEntryEvalResults } from "../../pipeline/emit-eval-results.js";
 import { emitSymbolPreflight } from "../../pipeline/preflight/emit-symbol-preflight.js";
 import { loadPackageSurface } from "../../pipeline/preflight/load-package-surface.js";
 import { AccumulatingArtifactWriter } from "../../artifact-capture/accumulating-artifact-writer.js";
+import { FanoutArtifactWriter } from "../../artifact-capture/fanout-artifact-writer.js";
+import { InstrumentedArtifactWriter } from "../../artifact-capture/instrumented-artifact-writer.js";
 import { getStepInputPaths } from "../../pipeline/cache.js";
 import { buildCacheContext } from "../cache-context.js";
-import { remapToCacheHitRefs } from "../../pipeline/cache-hit-restore.js";
+import { pruneToResolvableRefs, remapToCacheHitRefs, } from "../../pipeline/cache-hit-restore.js";
 import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
 import { computeEvalFingerprint } from "../../pipeline/eval-fingerprint.js";
 import { loadGraderModel } from "../../pipeline/grader-api.js";
@@ -147,11 +149,29 @@ export class RunEvalStep {
                     remoteCacheResult.sourceRunId &&
                     ctx.artifactWriter instanceof AccumulatingArtifactWriter) {
                     const restored = remapToCacheHitRefs(remoteCacheResult.artifactManifest, { sourceRunId: remoteCacheResult.sourceRunId });
-                    ctx.artifactWriter.injectAccumulated(restored);
-                    const count = Object.keys(restored).length;
+                    // W0350 / D0057 — a degraded source run can advertise per-entry
+                    // artifacts (e.g. rawResults) whose objects were never written under
+                    // its prefix. Drop those over-claims here, at the restore boundary,
+                    // so the new run's manifest advertises only artifacts that resolve —
+                    // rather than pushing per-object HEAD checks onto the read side's hot
+                    // signing path (AC 3). When no object checker is reachable in the
+                    // writer chain (local-only / NoOp / gateway backends), skip the prune
+                    // and restore verbatim, preserving prior behavior.
+                    const checker = findObjectChecker(ctx.artifactWriter);
+                    const { manifest: resolvable, droppedEntries, droppedRefs, } = checker
+                        ? await pruneToResolvableRefs(restored, checker)
+                        : { manifest: restored, droppedEntries: 0, droppedRefs: 0 };
+                    ctx.artifactWriter.injectAccumulated(resolvable);
+                    const count = Object.keys(resolvable).length;
                     if (count > 0) {
                         console.log(`  ↪ Restored ${count} artifact ref${count === 1 ? "" : "s"} from run ${remoteCacheResult.sourceRunId}`);
                     }
+                    if (droppedEntries > 0 || droppedRefs > 0) {
+                        const refsNote = droppedRefs > 0
+                            ? ` and ${droppedRefs} ref${droppedRefs === 1 ? "" : "s"}`
+                            : "";
+                        console.log(`  ⚠️  Dropped ${droppedEntries} unresolvable artifact entr${droppedEntries === 1 ? "y" : "ies"}${refsNote} over-claimed by cache parent ${remoteCacheResult.sourceRunId}`);
+                    }
                 }
                 return {
                     durationMs: Date.now() - start,
@@ -275,6 +295,39 @@ export class RunEvalStep {
     }
 }
 // ---------------------------------------------------------------------------
+// Object-checker discovery (D0057 / W0350)
+// ---------------------------------------------------------------------------
+const FIND_CHECKER_MAX_STEPS = 16;
+/**
+ * Walk a writer's decorator/fanout chain to feature-detect an
+ * `ArtifactObjectChecker`. The composition root wraps the backend in
+ * `AccumulatingArtifactWriter` (and optionally `InstrumentedArtifactWriter`)
+ * and layers GCS over local via `FanoutArtifactWriter`. Only
+ * `GcsArtifactWriter` implements `objectExists`, so on local-only / NoOp /
+ * gateway chains this returns null and the cache-hit restore skips pruning.
+ * `MAX_STEPS` is a cycle guard against a future decorator self-reference.
+ */
+function findObjectChecker(writer) {
+    const stack = [writer];
+    for (let steps = 0; stack.length > 0 && steps < FIND_CHECKER_MAX_STEPS; steps++) {
+        const cursor = stack.pop();
+        if (!cursor)
+            continue;
+        if (hasObjectExists(cursor))
+            return cursor;
+        if (cursor instanceof AccumulatingArtifactWriter)
+            stack.push(cursor.inner);
+        else if (cursor instanceof InstrumentedArtifactWriter)
+            stack.push(cursor.inner);
+        else if (cursor instanceof FanoutArtifactWriter)
+            stack.push(...cursor.delegates);
+    }
+    return null;
+}
+function hasObjectExists(w) {
+    return (typeof w.objectExists === "function");
+}
+// ---------------------------------------------------------------------------
 // Remote cache helpers
 // ---------------------------------------------------------------------------
 async function checkRemoteCache(fingerprint, reportStore, rootDir) {

package/dist/pipeline/assert-grader-judgments-persisted.d.ts

@@ -0,0 +1,35 @@
+/**
+ * pipeline/assert-grader-judgments-persisted.ts
+ *
+ * Post-persist guard for the grader-judgments write junction in
+ * `calculateAndWriteScores`.
+ *
+ * `extractGraderJudgmentsResilient` returns N judgments in memory, after which
+ * `runBorderlinePass` may mutate the array in place and a `judgments.length > 0`
+ * guard decides whether `grader-judgments.json` is written. A transient read
+ * anomaly or an unexpected in-place emptying can leave the file absent or empty
+ * even though extraction yielded judgments. Silently skipping the write strands
+ * gap-analysis and ships a scored report with no test details.
+ *
+ * This guard re-reads the file from disk — the same read gap-analysis performs
+ * — and fails loud when a non-empty extraction did not round-trip. The check is
+ * deliberately narrow: it fires only on the catastrophic "extracted N>0,
+ * persisted 0" divergence, never on a genuinely judgment-free run.
+ */
+/** Injectable seam — counts the grader judgments actually on disk. */
+export interface PersistVerificationDeps {
+    countPersisted: (path: string) => number;
+}
+/**
+ * Fail loud when a non-empty grader-judgment extraction did not round-trip to
+ * disk. No-ops when nothing was extracted — a judgment-free run (all api-errors
+ * / no llm-rubric) is valid and persists nothing by design.
+ *
+ * @param extractedCount  Judgments returned by extraction, captured BEFORE any
+ *   in-place mutation (e.g. the borderline-consensus pass) so the count
+ *   reflects what extraction actually produced.
+ * @param judgmentsPath  Absolute path to `grader-judgments.json`.
+ * @param deps  Injectable disk reader; defaults to the real filesystem.
+ * @throws {Error} when `extractedCount > 0` but the persisted file holds 0.
+ */
+export declare function assertGraderJudgmentsPersisted(extractedCount: number, judgmentsPath: string, deps?: PersistVerificationDeps): void;

package/dist/pipeline/assert-grader-judgments-persisted.js

@@ -0,0 +1,58 @@
+/**
+ * pipeline/assert-grader-judgments-persisted.ts
+ *
+ * Post-persist guard for the grader-judgments write junction in
+ * `calculateAndWriteScores`.
+ *
+ * `extractGraderJudgmentsResilient` returns N judgments in memory, after which
+ * `runBorderlinePass` may mutate the array in place and a `judgments.length > 0`
+ * guard decides whether `grader-judgments.json` is written. A transient read
+ * anomaly or an unexpected in-place emptying can leave the file absent or empty
+ * even though extraction yielded judgments. Silently skipping the write strands
+ * gap-analysis and ships a scored report with no test details.
+ *
+ * This guard re-reads the file from disk — the same read gap-analysis performs
+ * — and fails loud when a non-empty extraction did not round-trip. The check is
+ * deliberately narrow: it fires only on the catastrophic "extracted N>0,
+ * persisted 0" divergence, never on a genuinely judgment-free run.
+ */
+import { existsSync, readFileSync } from "node:fs";
+/**
+ * Parse `grader-judgments.json` and return its array length. Every "no usable
+ * judgments" shape (missing, unreadable, invalid JSON, non-array) collapses to
+ * 0 — mirroring how the downstream enrichment precondition reads the same file.
+ */
+function defaultCountPersisted(path) {
+    if (!existsSync(path))
+        return 0;
+    try {
+        const parsed = JSON.parse(readFileSync(path, "utf-8"));
+        return Array.isArray(parsed) ? parsed.length : 0;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Fail loud when a non-empty grader-judgment extraction did not round-trip to
+ * disk. No-ops when nothing was extracted — a judgment-free run (all api-errors
+ * / no llm-rubric) is valid and persists nothing by design.
+ *
+ * @param extractedCount  Judgments returned by extraction, captured BEFORE any
+ *   in-place mutation (e.g. the borderline-consensus pass) so the count
+ *   reflects what extraction actually produced.
+ * @param judgmentsPath  Absolute path to `grader-judgments.json`.
+ * @param deps  Injectable disk reader; defaults to the real filesystem.
+ * @throws {Error} when `extractedCount > 0` but the persisted file holds 0.
+ */
+export function assertGraderJudgmentsPersisted(extractedCount, judgmentsPath, deps = { countPersisted: defaultCountPersisted }) {
+    if (extractedCount <= 0)
+        return;
+    const persisted = deps.countPersisted(judgmentsPath);
+    if (persisted <= 0) {
+        throw new Error(`Grader judgments extract/persist divergence: extracted ${extractedCount} ` +
+            `judgment(s) but grader-judgments.json persisted 0. Refusing to finish ` +
+            `scoring — a scored report with no grader judgments would strand ` +
+            `gap-analysis and ship with no test details.`);
+    }
+}

package/dist/pipeline/cache-hit-restore.d.ts

@@ -8,7 +8,7 @@
  * @see docs/decisions/D0040-artifact-ref-source-run-id.md
  * @see docs/design-docs/cache-hit-artifact-restoration.md
  */
-import { type ArtifactManifest, type RunId } from "../_vendor/ailf-core/index.d.ts";
+import { type ArtifactManifest, type ArtifactObjectChecker, type RunId } from "../_vendor/ailf-core/index.d.ts";
 /**
  * Copy an artifact manifest verbatim and stamp `sourceRunId` on every ref
  * that doesn't already carry one.
@@ -47,3 +47,39 @@ import { type ArtifactManifest, type RunId } from "../_vendor/ailf-core/index.d.
 export declare function remapToCacheHitRefs(source: ArtifactManifest, opts: {
     sourceRunId: RunId;
 }): ArtifactManifest;
+/** Outcome of `pruneToResolvableRefs`. */
+export interface PruneResult {
+    /** Manifest with over-claimed entries/refs removed. */
+    readonly manifest: ArtifactManifest;
+    /** Per-entry entries dropped because their object did not exist. */
+    readonly droppedEntries: number;
+    /** Refs dropped entirely (bulk missing, or per-entry left with no entries). */
+    readonly droppedRefs: number;
+}
+/**
+ * Drop artifact refs (and per-entry entries) a cached report over-claimed —
+ * entries whose backing object was never written under the source run's
+ * storage prefix (D0040 / D0057, W0350).
+ *
+ * A degraded source run can publish a manifest that lists `rawResults`
+ * entries with no GCS object behind them; `remapToCacheHitRefs` copies those
+ * phantom entries forward into the new run's manifest, and the read side then
+ * signs URLs that 404 ("the specified key does not exist"). Pruning here, at
+ * the cache-hit restore boundary, removes the over-claim at the source so the
+ * written manifest's `entryCount` / `entries[]` reflect only artifacts that
+ * actually resolve — instead of pushing per-object HEAD checks onto the hot
+ * signing path (W0350 AC 3).
+ *
+ * Resolution mirrors the gateway: a per-entry object lives at
+ * `descriptor.objectPath(sourceRunId, entry.key)`, where `sourceRunId` is the
+ * runId encoded in `ref.path` (preferred — structurally tied to where bytes
+ * physically live) falling back to `ref.sourceRunId` (the lineage hint).
+ *
+ * **Fail open.** `checker.objectExists` throws when existence can't be
+ * determined (auth / network / quota). A throw KEEPS the ref/entry — we never
+ * drop a real artifact on a transient blip; the read side already tolerates a
+ * rare residual 404 (W0349). Only a definitive `false` drops an entry.
+ *
+ * Pure w.r.t. its inputs: returns a fresh manifest, never mutates `source`.
+ */
+export declare function pruneToResolvableRefs(source: ArtifactManifest, checker: ArtifactObjectChecker): Promise<PruneResult>;

package/dist/pipeline/cache-hit-restore.js

@@ -8,7 +8,7 @@
  * @see docs/decisions/D0040-artifact-ref-source-run-id.md
  * @see docs/design-docs/cache-hit-artifact-restoration.md
  */
-import { ARTIFACT_REGISTRY, } from "../_vendor/ailf-core/index.js";
+import { ARTIFACT_REGISTRY, runId as parseRunId, } from "../_vendor/ailf-core/index.js";
 /**
  * Copy an artifact manifest verbatim and stamp `sourceRunId` on every ref
  * that doesn't already carry one.
@@ -60,3 +60,110 @@ export function remapToCacheHitRefs(source, opts) {
     }
     return out;
 }
+/**
+ * Drop artifact refs (and per-entry entries) a cached report over-claimed —
+ * entries whose backing object was never written under the source run's
+ * storage prefix (D0040 / D0057, W0350).
+ *
+ * A degraded source run can publish a manifest that lists `rawResults`
+ * entries with no GCS object behind them; `remapToCacheHitRefs` copies those
+ * phantom entries forward into the new run's manifest, and the read side then
+ * signs URLs that 404 ("the specified key does not exist"). Pruning here, at
+ * the cache-hit restore boundary, removes the over-claim at the source so the
+ * written manifest's `entryCount` / `entries[]` reflect only artifacts that
+ * actually resolve — instead of pushing per-object HEAD checks onto the hot
+ * signing path (W0350 AC 3).
+ *
+ * Resolution mirrors the gateway: a per-entry object lives at
+ * `descriptor.objectPath(sourceRunId, entry.key)`, where `sourceRunId` is the
+ * runId encoded in `ref.path` (preferred — structurally tied to where bytes
+ * physically live) falling back to `ref.sourceRunId` (the lineage hint).
+ *
+ * **Fail open.** `checker.objectExists` throws when existence can't be
+ * determined (auth / network / quota). A throw KEEPS the ref/entry — we never
+ * drop a real artifact on a transient blip; the read side already tolerates a
+ * rare residual 404 (W0349). Only a definitive `false` drops an entry.
+ *
+ * Pure w.r.t. its inputs: returns a fresh manifest, never mutates `source`.
+ */
+export async function pruneToResolvableRefs(source, checker) {
+    const out = {};
+    let droppedEntries = 0;
+    let droppedRefs = 0;
+    for (const [type, ref] of Object.entries(source)) {
+        if (!ref)
+            continue;
+        const artifactType = type;
+        const descriptor = ARTIFACT_REGISTRY[artifactType];
+        // Bulk: a single object at ref.path.
+        if (ref.layout === "bulk") {
+            if (await existsOrKeep(checker, ref.path))
+                out[artifactType] = ref;
+            else
+                droppedRefs++;
+            continue;
+        }
+        // Per-entry: each entry is its own object under the source run prefix.
+        const sourceRunId = resolveSourceRunId(ref);
+        const entries = ref.entries ?? [];
+        if (!descriptor || sourceRunId === undefined || entries.length === 0) {
+            // Can't resolve per-entry object paths — fail open, keep verbatim.
+            out[artifactType] = ref;
+            continue;
+        }
+        const keptFlags = await Promise.all(entries.map(async (entry) => {
+            let objectPath;
+            try {
+                objectPath = descriptor.objectPath(sourceRunId, entry.key);
+            }
+            catch {
+                return true; // malformed key — fail open rather than drop
+            }
+            return existsOrKeep(checker, objectPath);
+        }));
+        const kept = entries.filter((_, i) => keptFlags[i]);
+        if (kept.length === entries.length) {
+            out[artifactType] = ref;
+            continue;
+        }
+        droppedEntries += entries.length - kept.length;
+        if (kept.length === 0) {
+            droppedRefs++; // nothing resolvable — drop the whole over-claimed ref
+            continue;
+        }
+        out[artifactType] = {
+            ...ref,
+            entries: kept,
+            entryCount: kept.length,
+            bytes: kept.reduce((sum, e) => sum + (e.bytes ?? 0), 0),
+        };
+    }
+    return { manifest: out, droppedEntries, droppedRefs };
+}
+/**
+ * Resolve where a ref's bytes physically live. Prefers the runId encoded in
+ * `ref.path` (validated through the canonical parser so a malformed manifest
+ * path can't propagate into a synthesized object name) over the
+ * `ref.sourceRunId` lineage hint — matching the gateway's resolution order.
+ */
+function resolveSourceRunId(ref) {
+    const fromPath = /^runs\/([^/]+)/.exec(ref.path)?.[1];
+    if (fromPath) {
+        const parsed = parseRunId(fromPath);
+        if (parsed.ok)
+            return parsed.value;
+    }
+    return ref.sourceRunId;
+}
+/**
+ * `checker.objectExists` wrapper that returns `true` (keep) on a thrown,
+ * indeterminate result — only a definitive `false` drops the artifact.
+ */
+async function existsOrKeep(checker, path) {
+    try {
+        return await checker.objectExists(path);
+    }
+    catch {
+        return true;
+    }
+}

package/dist/pipeline/calculate-scores.js

@@ -42,6 +42,7 @@ import { loadSource } from "../sources.js";
 import { LiteracyVariant } from "./normalize-mode.js";
 import { scoreTestGroup, } from "./compiler/scoring-bridge.js";
 import { extractGraderJudgmentsResilient, } from "./extract-grader-judgments-resilient.js";
+import { assertGraderJudgmentsPersisted } from "./assert-grader-judgments-persisted.js";
 // Re-export from core for backward compatibility.
 // Existing imports from this file continue to work unchanged.
 export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
@@ -1544,6 +1545,7 @@ export async function calculateAndWriteScores(options) {
         log.info("Score summary written to results/latest/score-summary.json");
         // Extract and persist grader judgments
         const judgments = await extractGraderJudgmentsResilient([baselineResultsPath], undefined, log, { deps: resilientJudgmentDeps });
+        const extractedJudgmentCount = judgments.length;
         const borderlineConsistency = await runBorderlinePass(judgments, [
             baselineResultsPath,
         ]);
@@ -1555,6 +1557,10 @@ export async function calculateAndWriteScores(options) {
             writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
             log.info(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
         }
+        // Fail loud if a non-empty extraction did not round-trip to disk (a
+        // transient divergence at the persist junction): otherwise gap-analysis
+        // skips and the report ships a score with no test details.
+        assertGraderJudgmentsPersisted(extractedJudgmentCount, join(outDir, "grader-judgments.json"));
         // Extract and persist per-test results (D0029: model output + metadata)
         // Agent-harness produces a single profile shared across detected variants
         // (the docs/no-docs split doesn't apply — there is no gold/baseline pair).
@@ -1607,6 +1613,7 @@ export async function calculateAndWriteScores(options) {
         writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
         log.info("Score summary written to results/latest/score-summary.json");
         const judgments = await extractGraderJudgmentsResilient([baselineResultsPath], undefined, log, { deps: resilientJudgmentDeps });
+        const extractedJudgmentCount = judgments.length;
         const borderlineConsistency = await runBorderlinePass(judgments, [
             baselineResultsPath,
         ]);
@@ -1618,6 +1625,10 @@ export async function calculateAndWriteScores(options) {
             writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
             log.info(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
         }
+        // Fail loud if a non-empty extraction did not round-trip to disk (a
+        // transient divergence at the persist junction): otherwise gap-analysis
+        // skips and the report ships a score with no test details.
+        assertGraderJudgmentsPersisted(extractedJudgmentCount, join(outDir, "grader-judgments.json"));
         // Knowledge-probe deletes vars.docs in the compiler, so every entry's
         // detected variant is "baseline" — supply the probe profile under both
         // keys so the composite is populated regardless of detection.
@@ -1744,6 +1755,9 @@ export async function calculateAndWriteScores(options) {
         ? [baselineResultsPath, agenticResultsPath]
         : [baselineResultsPath];
     const judgments = await extractGraderJudgmentsResilient(judgmentResultPaths, { reliability, ...(options.runId ? { runId: options.runId } : {}) }, log, { deps: resilientJudgmentDeps });
+    // Capture the extracted count before the borderline pass mutates the array
+    // in place — the persist guard below compares it against what lands on disk.
+    const extractedJudgmentCount = judgments.length;
     // Borderline-consensus pass — re-grade the ±5 borderline subset N times
     // and merge medians back into the canonical judgments BEFORE
     // `validateGraderJudgmentsCalibration` runs, so the calibration counter
@@ -1774,6 +1788,10 @@ export async function calculateAndWriteScores(options) {
             });
         }
     }
+    // Fail loud if a non-empty extraction did not round-trip to disk (a transient
+    // divergence at the persist junction): otherwise gap-analysis skips and the
+    // report ships a score with no test details.
+    assertGraderJudgmentsPersisted(extractedJudgmentCount, join(outDir, "grader-judgments.json"));
     // Extract and persist per-test results (D0029: model output + metadata).
     // Literacy gold (with-docs) entries score against the default profile;
     // baseline (without-docs) entries score against the output-only profile.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "7.2.1",
+  "version": "7.2.3",
   "private": false,
   "publishConfig": {
     "access": "public"