@forkpoint/agent-lighthouse-core 3.0.0 → 3.1.0

package/dist/index.js

@@ -68,6 +68,7 @@ __export(index_exports, {
   TAG_SKIPPED_PAGE_TYPE: () => TAG_SKIPPED_PAGE_TYPE,
   allEvidenceMet: () => allEvidenceMet,
   allJsonLdNodes: () => allJsonLdNodes,
+  boundedDispatcher: () => boundedDispatcher,
   buildCategoryResult: () => buildCategoryResult,
   buildScanEvidence: () => buildScanEvidence,
   calculateCategoryScore: () => calculateCategoryScore,
@@ -324,8 +325,40 @@ async function isSafeUrl(url) {
     return false;
   }
 }
-function createFetcher() {
+function createGate(limit) {
+  let inFlight = 0;
+  const waiting = [];
+  const release2 = () => {
+    inFlight -= 1;
+    const next = waiting.shift();
+    if (next) next();
+  };
+  return {
+    acquire: async () => {
+      if (inFlight >= limit) {
+        await new Promise((resolve4) => waiting.push(resolve4));
+      }
+      inFlight += 1;
+      return release2;
+    }
+  };
+}
+function boundedDispatcher(connections) {
+  return new import_undici.Agent({ connections });
+}
+function createFetcher(fetcherOptions = {}) {
+  const dispatcher = fetcherOptions.dispatcher ?? noRedirectAgent;
+  const gate = fetcherOptions.maxConcurrent && fetcherOptions.maxConcurrent > 0 ? createGate(Math.floor(fetcherOptions.maxConcurrent)) : void 0;
   async function fetch(options) {
+    if (!gate) return issue(options);
+    const release2 = await gate.acquire();
+    try {
+      return await issue(options);
+    } finally {
+      release2();
+    }
+  }
+  async function issue(options) {
     const {
       url,
       timeout = REQUEST_TIMEOUT_MS,
@@ -365,7 +398,7 @@ function createFetcher() {
         headers: reqHeaders,
         body: currentBody,
         signal,
-        dispatcher: noRedirectAgent
+        dispatcher
       });
       let gateArmed;
       let hops = 0;
@@ -413,7 +446,7 @@ function createFetcher() {
           headers: reqHeaders,
           body: currentBody,
           signal,
-          dispatcher: noRedirectAgent
+          dispatcher
         });
       }
       ttfbMs = performance.now() - start;
@@ -1218,6 +1251,197 @@ function gatedMassShare(checks2) {
   return total === 0 ? 0 : gated / total;
 }
 
+// src/gatherers/domains.ts
+var MULTI_SUFFIX = /* @__PURE__ */ new Set([
+  "co.uk",
+  "org.uk",
+  "ac.uk",
+  "gov.uk",
+  "me.uk",
+  "net.uk",
+  "com.au",
+  "net.au",
+  "org.au",
+  "edu.au",
+  "gov.au",
+  "co.nz",
+  "co.jp",
+  "or.jp",
+  "ne.jp",
+  "co.za",
+  "co.kr",
+  "co.il",
+  "co.id",
+  "co.th",
+  "com.br",
+  "com.mx",
+  "com.ar",
+  "com.co",
+  "com.pe",
+  "co.in",
+  "com.sg",
+  "com.tr",
+  "com.cn",
+  "com.hk",
+  "com.tw",
+  "com.my",
+  "com.ph",
+  "com.ua",
+  "com.pl",
+  "com.es",
+  "com.pt",
+  "com.gr"
+]);
+function registrableDomain(host) {
+  const parts = host.toLowerCase().replace(/\.$/, "").split(".").filter(Boolean);
+  if (parts.length <= 2) return parts.join(".");
+  const lastTwo = parts.slice(-2).join(".");
+  return MULTI_SUFFIX.has(lastTwo) ? parts.slice(-3).join(".") : lastTwo;
+}
+function registrableOf(url) {
+  try {
+    return registrableDomain(new URL(url).hostname);
+  } catch {
+    return "";
+  }
+}
+
+// src/scan-evidence.ts
+var ALL_PAGE_TYPES = ["homepage", "category", "product", "content"];
+var HTML_TYPES = ["text/html", "application/xhtml+xml"];
+var PERMANENT_REDIRECT = /* @__PURE__ */ new Set([301, 308]);
+function bareHost(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+  } catch {
+    return "";
+  }
+}
+function registrableName(url) {
+  const domain = registrableOf(url);
+  if (!domain) return "";
+  const parts = domain.split(".");
+  return parts.length > 1 ? parts.slice(0, -1).join(".") : domain;
+}
+function reachedTheRequestedSite(requestedUrl, result) {
+  const requested = bareHost(requestedUrl);
+  const final = bareHost(result.finalUrl || result.url);
+  if (!final) return { ok: false, reason: `The homepage response carried no usable URL.` };
+  if (requested === final) return { ok: true };
+  const requestedDomain = registrableOf(requestedUrl);
+  const finalDomain = registrableOf(result.finalUrl || result.url);
+  if (requestedDomain && requestedDomain === finalDomain) return { ok: true };
+  const requestedName = registrableName(requestedUrl);
+  if (requestedName && requestedName === registrableName(result.finalUrl || result.url)) {
+    return { ok: true };
+  }
+  const chain = result.redirectChain ?? [];
+  const leaving = chain.filter((hop) => registrableOf(hop.from) !== registrableOf(hop.to));
+  if (leaving.length > 0 && leaving.every((hop) => PERMANENT_REDIRECT.has(hop.status))) {
+    return { ok: true };
+  }
+  return {
+    ok: false,
+    reason: `The requested host redirected to ${final}, a different site, without a permanent redirect.`
+  };
+}
+function originReachable(requestedUrl, result) {
+  if (result.error) {
+    return { met: false, reason: `The homepage could not be fetched: ${result.error}.` };
+  }
+  if (result.status < 200 || result.status > 299) {
+    return { met: false, reason: `The homepage answered HTTP ${result.status}.` };
+  }
+  const type = (result.contentType || "").toLowerCase();
+  if (!HTML_TYPES.some((html) => type.includes(html))) {
+    return {
+      met: false,
+      reason: `The homepage served ${result.contentType || "no content type"}, not HTML.`
+    };
+  }
+  const reached = reachedTheRequestedSite(requestedUrl, result);
+  return reached.ok ? { met: true } : { met: false, reason: reached.reason };
+}
+function unblockedFetches(homepageResult, waf) {
+  if (waf?.isBlocked) {
+    return waf.isRateLimit ? {
+      met: false,
+      reason: `The scan was throttled (${waf.name}): ${waf.reason}.`
+    } : { met: false, reason: `${waf.name} refused the scan: ${waf.reason}.` };
+  }
+  if (homepageResult.status === 429) {
+    return { met: false, reason: "The homepage answered HTTP 429: the scan was throttled." };
+  }
+  return { met: true };
+}
+function pageRendersText(page) {
+  const text3 = getRenderedText(page.$);
+  const wordCount2 = text3.split(/\s+/).filter(Boolean).length;
+  return wordCount2 > 50 || text3.length > 200;
+}
+function buildScanEvidence(input) {
+  const origin = originReachable(input.requestedUrl, input.homepageResult);
+  const unblocked = unblockedFetches(input.homepageResult, input.wafProtection);
+  const renderedByPage = {};
+  const usablePageTypes = /* @__PURE__ */ new Set();
+  for (const page of input.pages) {
+    const rendered = pageRendersText(page);
+    renderedByPage[page.url] = rendered;
+    if (rendered) usablePageTypes.add(page.pageType);
+  }
+  const renderedCount = Object.values(renderedByPage).filter(Boolean).length;
+  const met = {
+    "origin-reachable": origin.met,
+    "unblocked-fetches": unblocked.met,
+    "rendered-body": renderedCount > 0,
+    "sample-adequate": usablePageTypes.size > 0
+  };
+  const reasons = {};
+  if (origin.reason) reasons["origin-reachable"] = origin.reason;
+  if (unblocked.reason) reasons["unblocked-fetches"] = unblocked.reason;
+  if (!met["rendered-body"]) {
+    reasons["rendered-body"] = input.pages.length === 0 ? "The scan fetched no pages." : `None of the ${input.pages.length} fetched page(s) served readable text.`;
+  }
+  if (!met["sample-adequate"]) {
+    reasons["sample-adequate"] = input.pages.length === 0 ? "The scan fetched no pages." : "No fetched page of any type served readable text.";
+  }
+  return {
+    met,
+    reasons,
+    renderedByPage,
+    usablePageTypes,
+    // A shell site was seen. What it serves is a finding about it, so
+    // `rendered-body` and `sample-adequate` do not clear `judgeable`.
+    judgeable: met["origin-reachable"] && met["unblocked-fetches"]
+  };
+}
+function scanReadTheSite(evidence) {
+  return evidence.judgeable;
+}
+function unreadSiteReason(evidence) {
+  return evidence.reasons["origin-reachable"] ?? evidence.reasons["unblocked-fetches"] ?? "The scan obtained no response it could attribute to this site.";
+}
+function scanReadPageText(evidence) {
+  return evidence.met["rendered-body"];
+}
+function unreadPageTextReason(evidence) {
+  return evidence.reasons["rendered-body"] ?? "No fetched page served text a non-JS consumer can read.";
+}
+function allEvidenceMet() {
+  return {
+    met: {
+      "origin-reachable": true,
+      "unblocked-fetches": true,
+      "rendered-body": true,
+      "sample-adequate": true
+    },
+    reasons: {},
+    renderedByPage: {},
+    usablePageTypes: new Set(ALL_PAGE_TYPES),
+    judgeable: true
+  };
+}
+
 // src/audits/access-crawl-control/no-nofollow.ts
 var NoNofollowAudit = class _NoNofollowAudit extends Audit {
   static meta = {
@@ -1231,8 +1455,9 @@ var NoNofollowAudit = class _NoNofollowAudit extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/access-crawl-control/no-nofollow.md",
-    // Gate exemption: being refused is what this category reports.
-    requires: ["origin-reachable", "rendered-body", "sample-adequate"],
+    // Gate exemption: being refused is what this category reports, and the meta tag and
+    // header this audit reads are served by a page whose body renders nothing.
+    requires: ["origin-reachable"],
     defaultPriority: "high",
     guidance: {
       impact: "A nofollow directive prevents AI crawlers from following links on your pages, effectively hiding all linked content from AI indexing. Your deeper pages become invisible to AI search engines, drastically reducing discoverability.",
@@ -1244,6 +1469,13 @@ var NoNofollowAudit = class _NoNofollowAudit extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its nofollow directives were not judged.",
+        "No site-wide nofollow directives",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (ctx.pages.length === 0) {
       return this.fail(
         "No pages scanned.",
@@ -1314,8 +1546,10 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/access-crawl-control/no-redirect-chains.md",
-    // Gate exemption: being refused is what this category reports.
-    requires: ["origin-reachable", "rendered-body", "sample-adequate"],
+    // Gate exemption: a hop that left the site is this audit's subject, and leaving the
+    // site is exactly what denies `origin-reachable`. It reads request URL against final
+    // URL, which every response carries, and reports "no pages scanned" itself.
+    requires: [],
     defaultPriority: "medium",
     guidance: {
       impact: "Redirect chains slow down AI crawlers and waste their limited crawl budget. Each extra redirect adds latency and increases the chance a crawler gives up before reaching the final page, leaving content unindexed.",
@@ -1326,17 +1560,6 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
     }
   };
   audit(ctx) {
-    if (ctx.pages.length === 0) {
-      return this.fail(
-        "No pages scanned.",
-        "No redirect chains (URL equals finalUrl or single redirect)",
-        "No pages scanned",
-        {
-          priority: "medium",
-          description: _NoRedirectChainsAudit.meta.description
-        }
-      );
-    }
     const redirected = [];
     for (const page of ctx.pages) {
       const requestUrl = page.fetchResult.url;
@@ -1346,6 +1569,24 @@ var NoRedirectChainsAudit = class _NoRedirectChainsAudit extends Audit {
       }
     }
     if (redirected.length === 0) {
+      if (!scanReadTheSite(ctx.evidence)) {
+        return this.notApplicable(
+          "No page here can be attributed to this site, so its redirect behaviour was not judged.",
+          "No redirect chains",
+          unreadSiteReason(ctx.evidence)
+        );
+      }
+      if (ctx.pages.length === 0) {
+        return this.fail(
+          "No pages scanned.",
+          "No redirect chains (URL equals finalUrl or single redirect)",
+          "No pages scanned",
+          {
+            priority: "medium",
+            description: _NoRedirectChainsAudit.meta.description
+          }
+        );
+      }
       return this.pass(
         `All ${ctx.pages.length} page(s) resolve without redirects.`,
         "No redirect chains",
@@ -1475,6 +1716,13 @@ var CanonicalLinksAudit = class extends Audit {
   };
   audit(ctx) {
     const expected = "Each page declares a canonical URL that points at itself";
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its canonical links were not judged.",
+        expected,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (ctx.pages.length === 0) {
       return this.notApplicable(
         "No pages were scanned, so no canonical links could be read.",
@@ -2688,6 +2936,13 @@ var NoBlanketBlockAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so its robots.txt was not judged.",
+        "User-agent: * does not Disallow: / entirely",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const robotsFile = ctx.rootFiles["/robots.txt"];
     if (!robotsFile || robotsFile.status !== 200 || !robotsFile.body) {
       return this.warn(
@@ -2939,6 +3194,13 @@ var CrawlDelayAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so its robots.txt was not judged.",
+        "If Crawl-delay is present, it is <= 10 seconds",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const robotsFile = ctx.rootFiles["/robots.txt"];
     if (!robotsFile || robotsFile.status !== 200 || !robotsFile.body) {
       return this.warn(
@@ -3069,8 +3331,9 @@ var MetaRobotsNotBlockingAudit = class extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/access-crawl-control/robots-directives.md",
-    // Gate exemption: being refused is what this category reports.
-    requires: ["origin-reachable", "rendered-body", "sample-adequate"],
+    // Gate exemption: being refused is what this category reports, and robots directives
+    // live in the head and the headers, which arrive whether or not the body renders.
+    requires: ["origin-reachable"],
     defaultPriority: "high",
     guidance: {
       impact: 'A content page carrying "noindex" (in a robots meta tag, a per-bot meta tag, or the X-Robots-Tag response header) is dropped from the search index, and Google documents that a page must be indexed to appear in AI Overviews or AI Mode. "nosnippet", "noarchive" and "max-snippet:0" keep the page indexed but stop its text being used as a direct input for AI answers.',
@@ -3082,6 +3345,13 @@ var MetaRobotsNotBlockingAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its robots directives were not judged.",
+        "No blocking robots directive on content pages",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (!ctx.pages || ctx.pages.length === 0) {
       return this.notApplicable(
         "No pages were scanned, so no robots directives could be read.",
@@ -3159,8 +3429,10 @@ var NoBotDetectionAudit = class extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/access-crawl-control/no-bot-detection.md",
-    // Gate exemption: being refused is what this category reports.
-    requires: ["origin-reachable", "rendered-body", "sample-adequate"],
+    // Gate exemption: being refused is what this category reports, and this audit names
+    // the firewall from `wafProtection` alone. Evidence a wall destroys is not evidence
+    // the wall finding needs.
+    requires: [],
     defaultPriority: "high",
     guidance: {
       impact: "Bot-detection services like Cloudflare Turnstile, DataDome, and reCAPTCHA can block legitimate AI agents from accessing your content. When agents are challenged, they cannot complete page fetches, making your content inaccessible to AI-powered search and assistants.",
@@ -3190,6 +3462,13 @@ var NoBotDetectionAudit = class extends Audit {
         }
       );
     }
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its scripts were not judged.",
+        "No JavaScript-based bot challenges that would block legitimate AI agents",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (!ctx.pages || ctx.pages.length === 0) {
       return this.warn(
         "No pages were scanned to check for bot-detection scripts.",
@@ -3215,6 +3494,13 @@ var NoBotDetectionAudit = class extends Audit {
       }
     }
     if (detectedServices.size === 0) {
+      if (!scanReadPageText(ctx.evidence)) {
+        return this.notApplicable(
+          "The scanned page served no readable text, so its scripts were not judged.",
+          "No JavaScript-based bot challenges that would block legitimate AI agents",
+          unreadPageTextReason(ctx.evidence)
+        );
+      }
       return this.pass(
         "No aggressive bot-detection scripts found on scanned pages.",
         "No JavaScript-based bot challenges that would block legitimate AI agents",
@@ -3337,6 +3623,13 @@ var TdmRepAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so no TDM-Rep declaration was read.",
+        EXPECTED3,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const pageUrl = ctx.pages[0]?.url;
     const header = readHeader(ctx);
     if (header) {
@@ -3624,6 +3917,13 @@ var AiContentDeclarationAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so no AI-usage declaration was read.",
+        EXPECTED4,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const found = survey(ctx);
     if (found.aipref) {
       return this.pass(
@@ -3674,8 +3974,9 @@ var HttpsEnabledAudit = class extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/access-crawl-control/https-enabled.md",
-    // Gate exemption: being refused is what this category reports.
-    requires: ["origin-reachable", "rendered-body", "sample-adequate"],
+    // Gate exemption: a base URL on plain HTTP is proven by the request, with no response
+    // at all, and that fail is worth reporting on a site whose homepage never answered.
+    requires: [],
     defaultPriority: "critical",
     guidance: {
       impact: "HTTP-only sites are completely excluded from all major AI systems. GPTBot, ClaudeBot, Perplexity, and enterprise RAG pipelines refuse to connect to non-HTTPS origins due to security policies. Your entire site is invisible to AI-generated answers, product recommendations, and agentic workflows.",
@@ -3690,35 +3991,42 @@ var HttpsEnabledAudit = class extends Audit {
     const isHttps = ctx.baseUrl.startsWith("https://");
     const page = ctx.pages?.[0];
     const status200 = page?.fetchResult.status === 200;
-    if (isHttps && status200) {
-      return this.pass(
-        "Site is served over HTTPS with a valid TLS connection.",
+    if (!isHttps) {
+      return this.fail(
+        "Site is not served over HTTPS. AI agents require secure connections.",
         "Base URL uses https:// and homepage returns 200",
-        `${ctx.baseUrl} \u2014 status ${page?.fetchResult.status}`,
+        `Base URL: ${ctx.baseUrl}`,
+        {
+          priority: "critical",
+          description: "Enterprise AI frameworks refuse to interact with non-HTTPS sites due to security policies. GPTBot, ClaudeBot, and enterprise RAG systems all skip HTTP-only sites entirely, making your content invisible to AI-generated answers. Enable HTTPS with a valid TLS certificate.",
+          code: "# For nginx:\nserver {\n  listen 443 ssl;\n  ssl_certificate /path/to/cert.pem;\n  ssl_certificate_key /path/to/key.pem;\n}"
+        },
         page?.url
       );
     }
-    if (isHttps && !status200) {
-      return this.warn(
-        `Site uses HTTPS but homepage returned status ${page?.fetchResult.status ?? "unknown"}. Possible TLS or server error.`,
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No homepage here can be attributed to this site, so its transport was not judged.",
         "Base URL uses https:// and homepage returns 200",
-        `${ctx.baseUrl} \u2014 status ${page?.fetchResult.status ?? "N/A"}`,
-        {
-          priority: "high",
-          description: "Enterprise AI frameworks refuse to interact with sites that have TLS errors. A non-200 HTTPS response prevents AI agents from ingesting your content, effectively making your site invisible to all AI systems. Fix server or TLS configuration to return a clean 200.",
-          code: "# Verify TLS with: curl -vI https://yoursite.com\n# Check for certificate expiry, chain issues, or redirect loops"
-        },
+        unreadSiteReason(ctx.evidence)
+      );
+    }
+    if (status200) {
+      return this.pass(
+        "Site is served over HTTPS with a valid TLS connection.",
+        "Base URL uses https:// and homepage returns 200",
+        `${ctx.baseUrl} \u2014 status ${page?.fetchResult.status}`,
         page?.url
       );
     }
-    return this.fail(
-      "Site is not served over HTTPS. AI agents require secure connections.",
+    return this.warn(
+      "Site uses HTTPS and the homepage answered, but the response carried no document, so an agent has nothing to read over that connection.",
       "Base URL uses https:// and homepage returns 200",
-      `Base URL: ${ctx.baseUrl}`,
+      `${ctx.baseUrl} \u2014 a 2xx response that carried no document`,
       {
-        priority: "critical",
-        description: "Enterprise AI frameworks refuse to interact with non-HTTPS sites due to security policies. GPTBot, ClaudeBot, and enterprise RAG systems all skip HTTP-only sites entirely, making your content invisible to AI-generated answers. Enable HTTPS with a valid TLS certificate.",
-        code: "# For nginx:\nserver {\n  listen 443 ssl;\n  ssl_certificate /path/to/cert.pem;\n  ssl_certificate_key /path/to/key.pem;\n}"
+        priority: "high",
+        description: "The homepage answered over HTTPS and returned no document \u2014 an empty 200 body, or a 2xx status that carries none. An AI agent that follows a link to this origin receives nothing, so nothing about the site can be indexed or quoted. Check the origin, the CDN cache entry and any edge rule that can strip a response body.",
+        code: "# Reproduce with:\ncurl -sSi https://yoursite.com | head -20"
       },
       page?.url
     );
@@ -3825,7 +4133,9 @@ var RobotsAiGroupShadowingAudit = class _RobotsAiGroupShadowingAudit extends Aud
     tier: "scored",
     dossier: "docs/evidence/audits/access-crawl-control/robots-ai-group-shadowing.md",
     // Gate exemption: being refused is what this category reports.
-    requires: ["origin-reachable", "rendered-body", "sample-adequate"],
+    // Gate exemption: the verdict comes from robots.txt. The scanned pages only widen
+    // the probe path set, so a shell narrows the probe and changes nothing judged.
+    requires: ["origin-reachable"],
     defaultPriority: "high",
     guidance: {
       impact: "RFC 9309 \xA72.2.1 states the wildcard group is consulted only 'if no matching group exists'. Therefore, for any site with a named AI-bot group, the wildcard group's Disallow rules provably do not apply to that bot, and the operator's stated intent (expressed once in `*`) diverges from the enforced policy by exactly the symmetric difference of the two rule sets. Falsifiable by construction: given robots.txt R and token T, the set of paths where R_T and R_star disagree is computable and either empty or not.",
@@ -3844,6 +4154,13 @@ var RobotsAiGroupShadowingAudit = class _RobotsAiGroupShadowingAudit extends Aud
     };
   }
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so its robots groups were not judged.",
+        EXPECTED5,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const robots = ctx.rootFiles["/robots.txt"];
     if (!robots || robots.status !== 200 || !robots.body.trim()) {
       return this.notApplicable(
@@ -4814,6 +5131,13 @@ var AiUsageSignalCoherenceAcrossChannelsAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so its AI-usage channels were not compared.",
+        "Every channel that carries an AI-usage signal says the same thing",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (ctx.pages.length === 0 && (ctx.rootFiles["/robots.txt"]?.status ?? 0) !== 200) {
       return this.notApplicable(
         "The scan read no page and no robots.txt, so no channel could carry a signal.",
@@ -5028,6 +5352,13 @@ var AiprefContentUsageDeclarationValidityAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so no Content-Usage declaration was read.",
+        "Every Content-Usage declaration parses as an RFC 8941 dictionary of AIPREF categories, attaches to a crawlable path, and agrees with the other channel",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const robots = ctx.rootFiles["/robots.txt"];
     const robotsBody = robots?.status === 200 ? robots.body : "";
     const groups = robotsBody === "" ? [] : parseRobots(robotsBody);
@@ -5915,7 +6246,9 @@ var ServerResponsivenessAudit = class extends Audit {
     evidenceGrade: "B",
     tier: "scored",
     dossier: "docs/evidence/audits/content-extraction/server-responsiveness.md",
-    requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
+    // Gate exemption: TTFB is measured from the response, and a shell answers as fast
+    // or as slow as anything else the origin serves.
+    requires: ["origin-reachable", "unblocked-fetches"],
     defaultPriority: "medium",
     guidance: {
       impact: 'Google documents that crawl capacity falls when a host slows down ("if the site slows down\u2026 the limit goes down and Google crawls less"), and slow origins are where logged HTTP 499 client-closed-request clusters from AI fetchers appear. A slow origin therefore gets less of its content into the indexes AI answers are drawn from.',
@@ -5934,6 +6267,13 @@ var ServerResponsivenessAudit = class extends Audit {
         `Blocked by ${ctx.wafProtection.name}`
       );
     }
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its response time was not judged.",
+        EXPECTED8,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const measured = ctx.pages.filter((p) => !p.fetchResult.error && p.fetchResult.status !== 0);
     const unmeasured = ctx.pages.length - measured.length;
     if (measured.length === 0) {
@@ -5994,7 +6334,8 @@ var LanguageAttributeAudit = class extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/content-extraction/language-attribute.md",
-    requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
+    // Gate exemption: `<html lang>` is served before any body renders.
+    requires: ["origin-reachable", "unblocked-fetches"],
     defaultPriority: "high",
     guidance: {
       impact: "AI agents use the lang attribute to select the correct language model and tokenizer when processing your content. Without it, agents may misinterpret content language, leading to poor translations or incorrect answers in multilingual AI systems.",
@@ -6006,6 +6347,13 @@ var LanguageAttributeAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its language attribute was not judged.",
+        '<html lang="..."> with a non-empty language code',
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     const $ = page?.$;
     const lang = $?.("html").attr("lang") ?? "";
@@ -6497,6 +6845,13 @@ var SingleH1Audit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its headings were not judged.",
+        "Exactly one <h1> on the homepage",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const homepage = ctx.pages[0];
     if (!homepage) {
       return this.fail(
@@ -6650,6 +7005,13 @@ var MainElementAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its landmarks were not judged.",
+        "<main> element present on all pages",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     let pagesWithMain = 0;
     for (const page of ctx.pages) {
       if (page.$("main").length > 0) pagesWithMain++;
@@ -6709,6 +7071,13 @@ var ArticleElementAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its landmarks were not judged.",
+        "<article> elements on content pages",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     let pagesWithArticle = 0;
     for (const page of ctx.pages) {
       if (page.$("article").length > 0) pagesWithArticle++;
@@ -6767,6 +7136,13 @@ var HeaderFooterAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its landmarks were not judged.",
+        "Both <header> and <footer> present on all pages",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     let pagesWithBoth = 0;
     let pagesWithHeader = 0;
     let pagesWithFooter = 0;
@@ -7193,6 +7569,13 @@ var DataTablesAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its tables were not judged.",
+        "Tables have <thead> and <th> elements",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     let totalTables = 0;
     let properTables = 0;
     for (const page of ctx.pages) {
@@ -7205,6 +7588,13 @@ var DataTablesAudit = class extends Audit {
       });
     }
     if (totalTables === 0) {
+      if (!scanReadPageText(ctx.evidence)) {
+        return this.notApplicable(
+          "The scanned page served no readable text, so it held no tables to judge.",
+          "Tables have <thead> and <th> elements",
+          unreadPageTextReason(ctx.evidence)
+        );
+      }
       return this.pass(
         "No data tables found \u2014 check not applicable.",
         "Tables have <thead> and <th> elements",
@@ -7401,6 +7791,13 @@ var ContentDepthAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its content depth was not judged.",
+        "More than 300 words of content per page",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     let pagesAboveThreshold = 0;
     const wordCounts = [];
     for (const page of ctx.pages) {
@@ -7573,6 +7970,13 @@ var FigureFigcaptionAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its figures were not judged.",
+        "Images with context wrapped in <figure> with <figcaption>",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     let totalFigures = 0;
     let figuresWithCaption = 0;
     for (const page of ctx.pages) {
@@ -7601,6 +8005,13 @@ var FigureFigcaptionAudit = class extends Audit {
           }
         );
       }
+      if (!scanReadPageText(ctx.evidence)) {
+        return this.notApplicable(
+          "The scanned page served no readable text, so it held no images or figures to judge.",
+          "Images with context wrapped in <figure> with <figcaption>",
+          unreadPageTextReason(ctx.evidence)
+        );
+      }
       return this.pass(
         "No images or <figure> elements found \u2014 check not applicable.",
         "Images with context wrapped in <figure> with <figcaption>",
@@ -13106,6 +13517,13 @@ var TokenRatioAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its token mix was not measured.",
+        "A homepage from this site whose token mix can be measured",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     const rawHtml = page?.fetchResult.body ?? "";
     if (!page || rawHtml.trim().length === 0) {
@@ -13239,6 +13657,13 @@ var FakeHeadingsAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its headings were not judged.",
+        "All heading-like text uses semantic <h1>-<h6> elements",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const found = [];
     for (const page of ctx.pages) {
       const $ = page.$;
@@ -13262,6 +13687,13 @@ var FakeHeadingsAudit = class extends Audit {
     const foundSummary = found.slice(0, 5).map((f) => `${f.url}: ${describe5(f.heading)}`).join("; ");
     const expected = "All heading-like text uses semantic <h1>-<h6> elements";
     if (found.length === 0) {
+      if (!scanReadPageText(ctx.evidence)) {
+        return this.notApplicable(
+          "The scanned page served no readable text, so it held no headings to judge.",
+          expected,
+          unreadPageTextReason(ctx.evidence)
+        );
+      }
       return this.pass(
         "No fake headings detected \u2014 heading-like text uses semantic heading elements.",
         expected,
@@ -13290,185 +13722,6 @@ var FakeHeadingsAudit = class extends Audit {
   }
 };
 
-// src/gatherers/domains.ts
-var MULTI_SUFFIX = /* @__PURE__ */ new Set([
-  "co.uk",
-  "org.uk",
-  "ac.uk",
-  "gov.uk",
-  "me.uk",
-  "net.uk",
-  "com.au",
-  "net.au",
-  "org.au",
-  "edu.au",
-  "gov.au",
-  "co.nz",
-  "co.jp",
-  "or.jp",
-  "ne.jp",
-  "co.za",
-  "co.kr",
-  "co.il",
-  "co.id",
-  "co.th",
-  "com.br",
-  "com.mx",
-  "com.ar",
-  "com.co",
-  "com.pe",
-  "co.in",
-  "com.sg",
-  "com.tr",
-  "com.cn",
-  "com.hk",
-  "com.tw",
-  "com.my",
-  "com.ph",
-  "com.ua",
-  "com.pl",
-  "com.es",
-  "com.pt",
-  "com.gr"
-]);
-function registrableDomain(host) {
-  const parts = host.toLowerCase().replace(/\.$/, "").split(".").filter(Boolean);
-  if (parts.length <= 2) return parts.join(".");
-  const lastTwo = parts.slice(-2).join(".");
-  return MULTI_SUFFIX.has(lastTwo) ? parts.slice(-3).join(".") : lastTwo;
-}
-function registrableOf(url) {
-  try {
-    return registrableDomain(new URL(url).hostname);
-  } catch {
-    return "";
-  }
-}
-
-// src/scan-evidence.ts
-var ALL_PAGE_TYPES = ["homepage", "category", "product", "content"];
-var HTML_TYPES = ["text/html", "application/xhtml+xml"];
-var PERMANENT_REDIRECT = /* @__PURE__ */ new Set([301, 308]);
-function bareHost(url) {
-  try {
-    return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
-  } catch {
-    return "";
-  }
-}
-function registrableName(url) {
-  const domain = registrableOf(url);
-  if (!domain) return "";
-  const parts = domain.split(".");
-  return parts.length > 1 ? parts.slice(0, -1).join(".") : domain;
-}
-function reachedTheRequestedSite(requestedUrl, result) {
-  const requested = bareHost(requestedUrl);
-  const final = bareHost(result.finalUrl || result.url);
-  if (!final) return { ok: false, reason: `The homepage response carried no usable URL.` };
-  if (requested === final) return { ok: true };
-  const requestedDomain = registrableOf(requestedUrl);
-  const finalDomain = registrableOf(result.finalUrl || result.url);
-  if (requestedDomain && requestedDomain === finalDomain) return { ok: true };
-  const requestedName = registrableName(requestedUrl);
-  if (requestedName && requestedName === registrableName(result.finalUrl || result.url)) {
-    return { ok: true };
-  }
-  const chain = result.redirectChain ?? [];
-  const leaving = chain.filter((hop) => registrableOf(hop.from) !== registrableOf(hop.to));
-  if (leaving.length > 0 && leaving.every((hop) => PERMANENT_REDIRECT.has(hop.status))) {
-    return { ok: true };
-  }
-  return {
-    ok: false,
-    reason: `The requested host redirected to ${final}, a different site, without a permanent redirect.`
-  };
-}
-function originReachable(requestedUrl, result) {
-  if (result.error) {
-    return { met: false, reason: `The homepage could not be fetched: ${result.error}.` };
-  }
-  if (result.status < 200 || result.status > 299) {
-    return { met: false, reason: `The homepage answered HTTP ${result.status}.` };
-  }
-  const type = (result.contentType || "").toLowerCase();
-  if (!HTML_TYPES.some((html) => type.includes(html))) {
-    return {
-      met: false,
-      reason: `The homepage served ${result.contentType || "no content type"}, not HTML.`
-    };
-  }
-  const reached = reachedTheRequestedSite(requestedUrl, result);
-  return reached.ok ? { met: true } : { met: false, reason: reached.reason };
-}
-function unblockedFetches(homepageResult, waf) {
-  if (waf?.isBlocked) {
-    return waf.isRateLimit ? {
-      met: false,
-      reason: `The scan was throttled (${waf.name}): ${waf.reason}.`
-    } : { met: false, reason: `${waf.name} refused the scan: ${waf.reason}.` };
-  }
-  if (homepageResult.status === 429) {
-    return { met: false, reason: "The homepage answered HTTP 429: the scan was throttled." };
-  }
-  return { met: true };
-}
-function pageRendersText(page) {
-  const text3 = getRenderedText(page.$);
-  const wordCount2 = text3.split(/\s+/).filter(Boolean).length;
-  return wordCount2 > 50 || text3.length > 200;
-}
-function buildScanEvidence(input) {
-  const origin = originReachable(input.requestedUrl, input.homepageResult);
-  const unblocked = unblockedFetches(input.homepageResult, input.wafProtection);
-  const renderedByPage = {};
-  const usablePageTypes = /* @__PURE__ */ new Set();
-  for (const page of input.pages) {
-    const rendered = pageRendersText(page);
-    renderedByPage[page.url] = rendered;
-    if (rendered) usablePageTypes.add(page.pageType);
-  }
-  const renderedCount = Object.values(renderedByPage).filter(Boolean).length;
-  const met = {
-    "origin-reachable": origin.met,
-    "unblocked-fetches": unblocked.met,
-    "rendered-body": renderedCount > 0,
-    "sample-adequate": usablePageTypes.size > 0
-  };
-  const reasons = {};
-  if (origin.reason) reasons["origin-reachable"] = origin.reason;
-  if (unblocked.reason) reasons["unblocked-fetches"] = unblocked.reason;
-  if (!met["rendered-body"]) {
-    reasons["rendered-body"] = input.pages.length === 0 ? "The scan fetched no pages." : `None of the ${input.pages.length} fetched page(s) served readable text.`;
-  }
-  if (!met["sample-adequate"]) {
-    reasons["sample-adequate"] = input.pages.length === 0 ? "The scan fetched no pages." : "No fetched page of any type served readable text.";
-  }
-  return {
-    met,
-    reasons,
-    renderedByPage,
-    usablePageTypes,
-    // A shell site was seen. What it serves is a finding about it, so
-    // `rendered-body` and `sample-adequate` do not clear `judgeable`.
-    judgeable: met["origin-reachable"] && met["unblocked-fetches"]
-  };
-}
-function allEvidenceMet() {
-  return {
-    met: {
-      "origin-reachable": true,
-      "unblocked-fetches": true,
-      "rendered-body": true,
-      "sample-adequate": true
-    },
-    reasons: {},
-    renderedByPage: {},
-    usablePageTypes: new Set(ALL_PAGE_TYPES),
-    judgeable: true
-  };
-}
-
 // src/audits/content-extraction/server-rendered.ts
 function withDetails(result, details) {
   return { ...result, details: { ...result.details ?? {}, ...details } };
@@ -13498,6 +13751,13 @@ var ServerRenderedAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its served HTML was not judged.",
+        "Every fetched page serves > 50 words or > 200 characters of readable text",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const pages = ctx.pages ?? [];
     if (pages.length === 0) {
       return this.notApplicable(
@@ -13812,6 +14072,13 @@ var CssHiddenGhostContentAudit = class _CssHiddenGhostContentAudit extends Audit
     };
   }
   async audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its hidden text was not measured.",
+        EXPECTED11,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const s = await survey2(ctx);
     const partial = s.crossOrigin > 0 ? `; ${s.crossOrigin} cross-origin stylesheet not fetched` : "";
     if (s.totalChars === 0) {
@@ -14148,6 +14415,13 @@ var PreambleTaxTokensBeforeTheFirstContentTokenAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its preamble was not measured.",
+        "A page from this site whose preamble can be measured",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     if (!page) {
       return this.notApplicable(
@@ -14408,6 +14682,13 @@ var ExtractionDeterminismAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so no extraction could be compared.",
+        "A page from this site whose extraction can be compared",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     if (!page) {
       return this.notApplicable(
@@ -14911,6 +15192,13 @@ var LlmsFullTxtAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so no llms-full.txt was judged.",
+        "GET /llms-full.txt returns 200",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const result = ctx.rootFiles["/llms-full.txt"];
     if (!result || !isOk5(result)) {
       return this.fail(
@@ -15494,6 +15782,13 @@ var RssFeedAudit = class extends Audit {
     }
   };
   async audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so no feed was judged.",
+        "Feed returns HTTP 200",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const links = autodiscoveryLinks(ctx);
     const feed = await findFeedResult(ctx, links);
     const linkNote = links.length > 0 ? `autodiscovery <link> present (${links[0].url})` : "no autodiscovery <link> in <head>";
@@ -16201,6 +16496,15 @@ var NoBrokenAiEndpointsAudit = class extends Audit {
     for (const url of allUrls) {
       if (await isSafeUrl(url)) urls.push(url);
     }
+    if (urls.length === 0) {
+      return this.warn(
+        `${allUrls.length} AI endpoint URL(s) are listed, and none of them could be requested: each names localhost, a private address, or a host that does not resolve.`,
+        "All URLs from AI-related files return 200",
+        `${allUrls.length} URL(s) listed, 0 reachable to check`,
+        void 0,
+        page?.url
+      );
+    }
     const results = await Promise.all(
       urls.map(async (url) => {
         try {
@@ -20470,6 +20774,13 @@ var UniqueMetaAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its metadata was not judged.",
+        "Each page has a unique title + description combination",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const canonicalGroups = /* @__PURE__ */ new Map();
     for (const page of ctx.pages) {
       let canon = (page.meta?.["canonical"] || page.url).trim();
@@ -20486,7 +20797,7 @@ var UniqueMetaAudit = class extends Audit {
     }
     const uniquePages = Array.from(canonicalGroups.values());
     if (uniquePages.length < 2) {
-      return this.pass(
+      return this.notApplicable(
         "Only one distinct canonical page scanned; uniqueness check not applicable.",
         "Each page has a unique title + description combination",
         "1 distinct page scanned"
@@ -21542,6 +21853,13 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its teasers were not judged.",
+        'No "click to read more" or "contact us to learn" teasers dominating the page',
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     if (!page) {
       return this.fail(
@@ -21598,6 +21916,13 @@ var ContentWithoutClickthroughAudit = class _ContentWithoutClickthroughAudit ext
           );
         }
       }
+      if (!scanReadPageText(ctx.evidence)) {
+        return this.notApplicable(
+          "The scanned page served no readable text, so there was no content to judge for teasers.",
+          'No "click to read more" or "contact us to learn" teasers dominating the page',
+          unreadPageTextReason(ctx.evidence)
+        );
+      }
       return this.pass(
         "No excessive click-through teasers found.",
         'No "click to read more" or "contact us to learn" teasers dominating the page',
@@ -23057,7 +23382,8 @@ var DescriptiveUrlsAudit = class extends Audit {
     evidenceGrade: "C",
     tier: "informative",
     dossier: "docs/evidence/audits/answer-readiness/descriptive-urls.md",
-    requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
+    // Gate exemption: a URL is readable whether or not the page behind it rendered text.
+    requires: ["origin-reachable", "unblocked-fetches"],
     defaultPriority: "high",
     guidance: {
       impact: "AI engines use URL text as a pre-fetch topic signal and display URLs in generated citations. Non-descriptive slugs with UUIDs or numeric IDs provide no topical context, reducing your content's relevance score before the page is even crawled.",
@@ -23068,6 +23394,13 @@ var DescriptiveUrlsAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its URLs were not judged.",
+        "Page URLs use readable slugs (no UUIDs, no /post-123/, no encoded params)",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     if (!page) {
       return this.fail(
@@ -23332,6 +23665,13 @@ var SnippetGateCoverageAudit = class _SnippetGateCoverageAudit extends Audit {
     };
   }
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so snippet permissions were not resolved.",
+        EXPECTED22,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     if (!page) {
       return this.notApplicable(
@@ -23806,6 +24146,13 @@ function chainOf($, el) {
   const parents = $(el).parents().toArray().filter((parent) => !["html", "body"].includes(parent.tagName)).reverse().map(describe3);
   return [...parents, describe3(el)].join(" > ");
 }
+function lastElementContaining($, text3) {
+  const all = $("*").toArray();
+  for (let i = all.length - 1; i >= 0; i--) {
+    if ($(all[i]).text().includes(text3)) return all[i];
+  }
+  return void 0;
+}
 function needleOf(text3) {
   return normalizeText(text3).split(" ").slice(0, SPAN_WORDS).join(" ");
 }
@@ -23838,8 +24185,8 @@ function keySpans(page) {
       if (typeof value === "string") {
         const needle = needleOf(value);
         if (needle.split(" ").length >= 3 && bodyText.includes(needle)) {
-          const host = $(`:contains("${value.slice(0, 40).replace(/"/g, "")}")`).last();
-          push("json-ld", host[0] ?? $("body")[0], value);
+          const host = lastElementContaining($, value.slice(0, 40));
+          push("json-ld", host ?? $("body")[0], value);
         }
         return;
       }
@@ -23883,6 +24230,13 @@ var ExtractorSurvivalRecallAudit = class extends Audit {
     }
   };
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so no key span was measured.",
+        "A page from this site whose key spans can be measured",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const page = ctx.pages[0];
     if (!page) {
       return this.notApplicable(
@@ -31320,8 +31674,10 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
     evidenceGrade: "A",
     tier: "scored",
     dossier: "docs/evidence/audits/operability-safety/no-blocking-captcha.md",
-    // Gate exemption: A captcha wall is what this audit reports.
-    requires: ["origin-reachable"],
+    // Gate exemption: a captcha wall is what this audit reports, and a wall denies
+    // `origin-reachable` — gating on it made the finding unreachable for the 403 that
+    // produced it. The wall branch reads `wafProtection`, not any response body.
+    requires: [],
     defaultPriority: "high",
     guidance: {
       impact: 'Blocking CAPTCHAs completely prevent AI agents from submitting forms on behalf of users. When a user asks an agent to "fill out the contact form," the CAPTCHA blocks the action entirely, forcing the user to do it manually or go to a competitor.',
@@ -31354,6 +31710,13 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
         ctx.baseUrl
       );
     }
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so no form was inspected for a CAPTCHA.",
+        "No recaptcha, hcaptcha, or turnstile script includes detected",
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (ctx.pages.length === 0) {
       return this.notApplicable(
         "No page was fetched, so no form could be inspected for a blocking CAPTCHA.",
@@ -31371,6 +31734,13 @@ var NoBlockingCaptchaAudit = class _NoBlockingCaptchaAudit extends Audit {
       }
     }
     if (detectedCaptchas.length === 0) {
+      if (!scanReadPageText(ctx.evidence)) {
+        return this.notApplicable(
+          "The scanned page served no readable text, so no form was inspected for a CAPTCHA.",
+          "No recaptcha, hcaptcha, or turnstile script includes detected",
+          unreadPageTextReason(ctx.evidence)
+        );
+      }
       return this.pass(
         "No blocking CAPTCHA scripts detected on scanned pages.",
         "No recaptcha, hcaptcha, or turnstile script includes detected",
@@ -33208,6 +33578,13 @@ var InvisibleInstructionScanAudit = class _InvisibleInstructionScanAudit extends
     };
   }
   async audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its hidden text was not judged.",
+        EXPECTED50,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const s = await survey9(ctx);
     const partial = s.crossOrigin > 0 ? `; ${s.crossOrigin} cross-origin stylesheet not fetched` : "";
     if (s.textNodesSeen === 0) {
@@ -33485,6 +33862,13 @@ var AriaLayerInjectionScanAudit = class _AriaLayerInjectionScanAudit extends Aud
     };
   }
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its non-visual values were not judged.",
+        EXPECTED51,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const s = survey10(ctx);
     if (s.valuesSeen === 0) {
       return this.notApplicable(
@@ -33530,6 +33914,13 @@ var AriaLayerInjectionScanAudit = class _AriaLayerInjectionScanAudit extends Aud
         warnings[0].pageUrl
       );
     }
+    if (!scanReadPageText(ctx.evidence)) {
+      return this.notApplicable(
+        "The scanned page served no readable text, so its accessibility layer was not judged.",
+        EXPECTED51,
+        unreadPageTextReason(ctx.evidence)
+      );
+    }
     return this.pass(
       `All ${s.valuesSeen} non-visual value(s) are descriptions that agree with their element and carry no instruction addressed to an AI.`,
       EXPECTED51,
@@ -33649,6 +34040,13 @@ var GhostClickableElementRatioAudit = class _GhostClickableElementRatioAudit ext
     };
   }
   async audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its click targets were not counted.",
+        EXPECTED52,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const s = await survey11(ctx);
     const total = s.semantic + s.ghosts.length;
     const partial = s.crossOrigin > 0 ? `; ${s.crossOrigin} cross-origin stylesheet not fetched` : "";
@@ -35013,6 +35411,13 @@ var UnicodeCovertChannelScanAudit = class _UnicodeCovertChannelScanAudit extends
     };
   }
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No response here can be attributed to this site, so its codepoints were not judged.",
+        EXPECTED58,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     const hits2 = [];
     for (const page of ctx.pages) hits2.push(...scanPage(page));
     for (const path of ROOT_FILES) {
@@ -35039,6 +35444,16 @@ var UnicodeCovertChannelScanAudit = class _UnicodeCovertChannelScanAudit extends
       fillerCount: filler
     };
     if (hits2.length === 0) {
+      if (!scanReadPageText(ctx.evidence)) {
+        return {
+          ...this.notApplicable(
+            "The scanned page served no readable text, so its codepoints were not judged.",
+            EXPECTED58,
+            unreadPageTextReason(ctx.evidence)
+          ),
+          details
+        };
+      }
       return {
         ...this.pass(
           "No invisible codepoint carries text on the scanned pages or in the root files.",
@@ -35234,7 +35649,10 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
     evidenceGrade: "B",
     tier: "scored",
     dossier: "docs/evidence/audits/operability-safety/third-party-dom-write-blast-radius.md",
-    requires: ["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"],
+    // Gate exemption: every origin the served HTML names is counted whether or not the
+    // body renders, so a page that ships a vendor script statically is still reported.
+    // The empty census is the case a shell cannot support, and `audit()` declines it.
+    requires: ["origin-reachable", "unblocked-fetches"],
     defaultPriority: "high",
     guidance: {
       impact: 'An agent reads the DOM as one document with one level of trust. It has no way to tell text the site wrote from text a vendor script injected after load, so every third-party origin that can write to the page can write instructions the agent will read as the site\'s own. The count is the risk: eleven uncontrolled origins is eleven independent companies \u2014 and their own supply chains \u2014 with the same authority over what an agent believes about the site. A Content-Security-Policy with a nonce, a hash or `strict-dynamic` is what turns that list from "whoever" into "these, and only these". A policy whose sources include `https:` or `*` is present in the response and constrains nothing.',
@@ -35253,6 +35671,13 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
     };
   }
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its third-party surface was not measured.",
+        EXPECTED59,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (ctx.pages.length === 0) {
       return this.notApplicable(
         "No page was fetched, so there is no third-party surface to measure.",
@@ -35292,6 +35717,17 @@ var ThirdPartyDomWriteBlastRadiusAudit = class _ThirdPartyDomWriteBlastRadiusAud
           details
         };
       }
+      if (!scanReadPageText(ctx.evidence)) {
+        return {
+          ...this.notApplicable(
+            "The scanned page served no readable text, so the origins writing into it were not counted.",
+            EXPECTED59,
+            unreadPageTextReason(ctx.evidence)
+          ),
+          displayValue: found,
+          details
+        };
+      }
       return {
         ...this.pass(
           "No third-party origin ships executable code into the page, so nothing but the site itself writes what an agent reads.",
@@ -35449,6 +35885,13 @@ var UnsafeAgentTriggerableAffordancesAudit = class _UnsafeAgentTriggerableAfford
     };
   }
   audit(ctx) {
+    if (!scanReadTheSite(ctx.evidence)) {
+      return this.notApplicable(
+        "No page here can be attributed to this site, so its links were not inspected.",
+        EXPECTED60,
+        unreadSiteReason(ctx.evidence)
+      );
+    }
     if (ctx.pages.length === 0) {
       return this.notApplicable(
         "No page was fetched, so there is no link to inspect.",
@@ -35467,6 +35910,16 @@ var UnsafeAgentTriggerableAffordancesAudit = class _UnsafeAgentTriggerableAfford
       urls: findings.slice(0, 10).map((f) => f.href)
     };
     if (findings.length === 0) {
+      if (!scanReadPageText(ctx.evidence)) {
+        return {
+          ...this.notApplicable(
+            "The scanned page served no readable text, so it exposed no links or forms to inspect.",
+            EXPECTED60,
+            unreadPageTextReason(ctx.evidence)
+          ),
+          details
+        };
+      }
       return {
         ...this.pass(
           "No link or GET form on the scanned pages changes state when it is fetched.",
@@ -37982,7 +38435,10 @@ async function runScan(url, options) {
   const signal = options?.signal;
   const tracker = new ProgressTracker((event) => onEvent?.(event));
   const start = performance.now();
-  const fetcher = createFetcher();
+  const fetcher = createFetcher({
+    dispatcher: options?.dispatcher,
+    maxConcurrent: options?.maxConcurrent
+  });
   const baseUrl = new URL(url).origin;
   const domain = new URL(url).hostname;
   const displayUrl = splitCredentials(url).url;
@@ -38040,13 +38496,18 @@ async function runScan(url, options) {
   ];
   logger.debug({ count: rootFilePaths.length }, "[orchestrator] Phase 1: Fetching root files");
   tracker.phaseStart("fetch-root", rootFilePaths.length);
+  const prefetchedRobots = options?.robotsTxt;
   const rootResults = await Promise.all(
-    rootFilePaths.map(
-      (path) => fetcher.fetch({ url: `${baseUrl}${path}`, signal }).then((result) => {
+    rootFilePaths.map((path) => {
+      if (path === "/robots.txt" && prefetchedRobots) {
+        tracker.unitDone(path);
+        return Promise.resolve(prefetchedRobots);
+      }
+      return fetcher.fetch({ url: `${baseUrl}${path}`, signal }).then((result) => {
         tracker.unitDone(path);
         return result;
-      })
-    )
+      });
+    })
   );
   const rootFiles = {};
   rootFilePaths.forEach((path, i) => {
@@ -38398,6 +38859,7 @@ function loadConfigFile(customPath) {
   TAG_SKIPPED_PAGE_TYPE,
   allEvidenceMet,
   allJsonLdNodes,
+  boundedDispatcher,
   buildCategoryResult,
   buildScanEvidence,
   calculateCategoryScore,