@goflag/cli 0.2.14 → 0.2.15

package/README.md

@@ -37,7 +37,7 @@ here and carries no code.
 
 ## What it does
 
-- **Broken links** — scrapes every crawled page, dedupes targets globally (a footer link on 500 pages is probed once), and checks each with `HEAD`→`GET` fallback, redirect-chain/loop detection, soft-404 and anti-bot (403/429) triage. Broken links are mapped back to the pages that reference them.
+- **Broken links** — reads the links of every crawled page from the copy the crawl kept, dedupes targets globally (a footer link on 500 pages is probed once, and a link to a page the crawl fetched without a redirect is judged from that answer rather than asked again), and checks each with `HEAD`→`GET` fallback, redirect-chain/loop detection, soft-404 and anti-bot (403/429) triage. Broken links are mapped back to the pages that reference them.
 - **Missing translation pages** — builds a `route × locale` matrix and flags every route that exists in one locale but is missing in another, plus hreflang reciprocity gaps (`A → B` with no `B → A`), missing `x-default`, and locale tags naming a language, script or region that does not exist. Rows are keyed by pathname, unless the site declares its translation clusters — either with `xhtml:link` in the sitemap, or with reciprocal `hreflang` between two crawled pages' `<head>`. Then those declarations decide which URLs are one page, whatever their slugs, and the run reports `diagnostics.declaredClusters`. The sitemap source survives `--coverage structural`, where the two locales of a slug-translating family are rarely both sampled; the `<head>` source needs both pages in hand and is what fixes the common site that declares properly there and nothing in its sitemap.
 - **robots.txt policy** — the file is parsed to RFC 9309, not scanned: every group, every rule, every line it could not read, each with its number. goflag flags a site that forbids crawling while its pages ask to be indexed — site-wide, and now per path, which is the quiet version nothing was watching. It also reports the file that errors (a 5xx there is read as a site-wide ban), one over the 500 KiB a parser must honour, a typo that silently deletes the rule you meant to write, a redirect handing your crawl policy to another origin, and a `Sitemap:` that is not a full URL.
 - **SEO metadata** — lints each page's `<head>` for the handful of mistakes that actually hurt in search and social and are invisible in a browser: missing/oversized `<title>` and description, missing/relative canonical, missing `og:title`/`og:description`/`og:image`, missing viewport, and contradictory `robots`/`googlebot`/`X-Robots-Tag` directives.
@@ -558,7 +558,7 @@ Three rules, whichever you use:
 stages: [build, deploy, audit]
 
 variables:
-  GOFLAG_VERSION: "0.2.14"
+  GOFLAG_VERSION: "0.2.15"
 
 # Before the merge: build the branch, let goflag boot it, audit that.
 seo:mr:
@@ -610,7 +610,7 @@ jobs:
       - run: corepack enable && pnpm install --frozen-lockfile
       - run: pnpm build
       - run: |
-          npx --yes @goflag/cli@0.2.14 http://localhost:3000 \
+          npx --yes @goflag/cli@0.2.15 http://localhost:3000 \
             --start "pnpm start" --static --no-external \
             --baseline .goflag/baseline.json --regressions-only --max-debt 41 \
             --report goflag-report.json

package/dist/{chunk-YQPL67HX.js → chunk-KHIN5TNX.js}

@@ -25,6 +25,25 @@ function fingerprint(category, ...parts) {
   return `${category}-${digest}`;
 }
 
+// src/lib/core/canonicalise.ts
+function canonicaliseUrl(href, base, options = {}) {
+  let url;
+  try {
+    url = new URL(href, base);
+  } catch {
+    return null;
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+  url.hash = "";
+  if (!options.preserveTrailingSlash && url.pathname.length > 1 && url.pathname.endsWith("/")) {
+    url.pathname = url.pathname.slice(0, -1);
+  }
+  return url.toString();
+}
+
+// src/lib/core/inspect.ts
+import * as cheerio3 from "cheerio";
+
 // src/lib/core/extract/static.ts
 import * as cheerio from "cheerio";
 
@@ -105,7 +124,9 @@ function decodeHtmlEntities(s) {
 
 // src/lib/core/extract/static.ts
 function extractStatic(html, options) {
-  const $ = cheerio.load(html);
+  return extractStaticFrom(cheerio.load(html), options);
+}
+function extractStaticFrom($, options) {
   const base = resolveBase($, options.baseUrl);
   const raw = extractRaw($);
   const meta = parseGenericMeta(raw, base);
@@ -952,12 +973,98 @@ function countEntries(body) {
   return url + sm;
 }
 
+// src/lib/core/links/extract.ts
+import * as cheerio2 from "cheerio";
+var ANCHOR_SPEC = { selector: "a[href]", attr: "href", source: "a" };
+var ASSET_SPECS = [
+  { selector: "img[src]", attr: "src", source: "img" },
+  { selector: "script[src]", attr: "src", source: "script" },
+  { selector: "link[href]", attr: "href", source: "link" },
+  { selector: "iframe[src]", attr: "src", source: "iframe" }
+];
+function extractLinks(html, options) {
+  return extractLinksFrom(cheerio2.load(html), options);
+}
+function extractLinksFrom($, options) {
+  const base = resolveBase2($, options.baseUrl);
+  const origin = safeOrigin(options.baseUrl);
+  const specs = options.includeAssets ? [ANCHOR_SPEC, ...ASSET_SPECS] : [ANCHOR_SPEC];
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const spec of specs) {
+    $(spec.selector).each((_, el) => {
+      const raw = $(el).attr(spec.attr);
+      if (raw === void 0) return;
+      const rawHref = raw.trim();
+      if (!rawHref) return;
+      const ref = buildRef($, el, rawHref, spec, base, origin);
+      if (!ref) return;
+      const dedupeKey = `${ref.source}|${ref.url}`;
+      if (seen.has(dedupeKey)) return;
+      seen.add(dedupeKey);
+      out.push(ref);
+    });
+  }
+  return out;
+}
+function buildRef($, el, rawHref, spec, base, origin) {
+  const fragment = extractFragment(rawHref);
+  const canonical = canonicaliseUrl(rawHref, base, { preserveTrailingSlash: true });
+  const url = canonical ?? rawHref;
+  const kind = canonical ? classify(canonical, origin) : "external";
+  const ref = {
+    rawHref,
+    url,
+    kind,
+    source: spec.source,
+    rel: spec.source === "a" || spec.source === "link" ? parseRel($(el).attr("rel")) : []
+  };
+  if (spec.source === "a") {
+    const text = $(el).text().trim();
+    if (text) ref.anchorText = text;
+  }
+  if (fragment) ref.fragment = fragment;
+  return ref;
+}
+function extractFragment(rawHref) {
+  const hash = rawHref.indexOf("#");
+  if (hash === -1) return void 0;
+  const frag = rawHref.slice(hash);
+  return frag.length > 1 ? frag : void 0;
+}
+function parseRel(rel) {
+  if (!rel) return [];
+  return rel.split(/\s+/).map((t) => t.trim().toLowerCase()).filter(Boolean);
+}
+function classify(url, origin) {
+  if (!origin) return "external";
+  return safeOrigin(url) === origin ? "internal" : "external";
+}
+function safeOrigin(url) {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return null;
+  }
+}
+function resolveBase2($, pageUrl) {
+  const declared = $("base[href]").first().attr("href")?.trim();
+  if (declared) {
+    try {
+      return new URL(declared, pageUrl).toString();
+    } catch {
+    }
+  }
+  return pageUrl;
+}
+
 // src/lib/core/inspect.ts
 async function inspect(url, options = {}) {
   const {
     probes: enableProbes = true,
     mode = "auto",
     headless: headlessOpts,
+    linkRefs: wantLinkRefs = false,
     ...fetchOptions
   } = options;
   const fetchedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -971,7 +1078,8 @@ async function inspect(url, options = {}) {
     });
   }
   const { meta: staticMeta, body: staticBody } = await fetchStatic(url, fetchOptions);
-  const staticParsed = extractStatic(staticBody, { baseUrl: staticMeta.finalUrl });
+  const { staticParsed, linkRefs } = parseStatic(staticBody, staticMeta.finalUrl, wantLinkRefs);
+  const withLinkRefs = linkRefs ? { linkRefs } : {};
   if (mode === "static") {
     const probes2 = await runProbes({
       enabled: enableProbes,
@@ -985,7 +1093,8 @@ async function inspect(url, options = {}) {
       fetch: staticMeta,
       extractor: { mode: "static", escalated: false },
       html: { static: staticBody },
-      probes: probes2
+      probes: probes2,
+      ...withLinkRefs
     };
   }
   const heuristic = looksClientRendered(staticParsed);
@@ -1002,7 +1111,8 @@ async function inspect(url, options = {}) {
       fetch: staticMeta,
       extractor: { mode: "static", escalated: false },
       html: { static: staticBody },
-      probes: probes2
+      probes: probes2,
+      ...withLinkRefs
     };
   }
   let renderedBody;
@@ -1034,7 +1144,8 @@ async function inspect(url, options = {}) {
         escalationBlocked: err.message
       },
       html: { static: staticBody },
-      probes: probes2
+      probes: probes2,
+      ...withLinkRefs
     };
   }
   const renderedParsed = extractStatic(renderedBody, { baseUrl: renderedFetch.finalUrl });
@@ -1056,9 +1167,16 @@ async function inspect(url, options = {}) {
     },
     html: { static: staticBody, rendered: renderedBody },
     hydration,
-    probes
+    probes,
+    ...withLinkRefs
   };
 }
+function parseStatic(html, baseUrl, wantLinkRefs) {
+  const $ = cheerio3.load(html);
+  const staticParsed = extractStaticFrom($, { baseUrl });
+  if (!wantLinkRefs) return { staticParsed };
+  return { staticParsed, linkRefs: extractLinksFrom($, { baseUrl }) };
+}
 async function inspectHeadlessOnly(args) {
   const { url, fetchedAt, enableProbes, fetchOptions, headlessOpts } = args;
   const result = await extractHeadless(url, {
@@ -1209,60 +1327,74 @@ async function crawl(options) {
   const recovered = [];
   let truncated = false;
   const seenFinal = /* @__PURE__ */ new Set();
-  while (queue.length > 0) {
+  const inspectItem = async (item) => {
+    try {
+      const page = await inspect(item.url, options.inspectOptions);
+      return { kind: "ok", item, page };
+    } catch (err) {
+      return { kind: "err", item, error: err instanceof Error ? err.message : String(err) };
+    }
+  };
+  const pending = [];
+  const maxPending = concurrency * 4;
+  let active = 0;
+  let requestDone = () => void 0;
+  const start = (item) => {
+    active += 1;
+    pending.push(
+      inspectItem(item).finally(() => {
+        active -= 1;
+        requestDone();
+      })
+    );
+  };
+  while (queue.length > 0 || pending.length > 0) {
     if (options.inspectOptions?.signal?.aborted) break;
-    const wave = queue.splice(0, concurrency);
-    const inspections = wave.map(async (item) => {
-      try {
-        const page = await inspect(item.url, options.inspectOptions);
-        return { kind: "ok", item, page };
-      } catch (err) {
-        return {
-          kind: "err",
-          item,
-          error: err instanceof Error ? err.message : String(err)
-        };
-      }
+    while (active < concurrency && pending.length < maxPending && queue.length > 0) {
+      start(queue.shift());
+    }
+    const oldest = pending[0];
+    const finished = new Promise((resolve3) => {
+      requestDone = resolve3;
     });
-    const results = await Promise.all(inspections);
-    for (const result of results) {
-      if (result.kind === "err") {
-        const attempt = (result.item.attempt ?? 0) + 1;
-        if (attempt < MAX_ATTEMPTS) {
-          queue.push({ ...result.item, attempt });
-          continue;
-        }
-        errors.push({ url: result.item.url, message: result.error });
+    const oldestReady = await Promise.race([oldest.then(() => true), finished.then(() => false)]);
+    if (!oldestReady) continue;
+    const result = await pending.shift();
+    if (result.kind === "err") {
+      const attempt = (result.item.attempt ?? 0) + 1;
+      if (attempt < MAX_ATTEMPTS) {
+        queue.push({ ...result.item, attempt });
         continue;
       }
-      if (result.item.attempt) recovered.push(result.item.url);
-      const finalUrl = canonicaliseUrl(result.page.fetch.finalUrl) ?? result.page.fetch.finalUrl;
-      if (seenFinal.has(finalUrl)) continue;
-      seenFinal.add(finalUrl);
-      pages.push(result.page);
-      if (options.onPage) {
-        options.onPage(result.page, { visited: pages.length, queued: queue.length });
-      }
-      if (pages.length >= maxPages) {
-        truncated = true;
-        break;
-      }
-      if (result.item.depth >= depthLimit) continue;
-      const candidates = extractCandidateLinks(result.page);
-      for (const candidate of candidates) {
-        const normalised = canonicaliseUrl(candidate.href, result.page.fetch.finalUrl);
-        if (!normalised) continue;
-        if (visitedSet.has(normalised)) continue;
-        const parsed = new URL(normalised);
-        if (parsed.origin !== origin) continue;
-        const passesIncludeExclude = filterAllows(parsed.pathname, include, exclude);
-        const isHreflang = candidate.source === "hreflang";
-        if (!passesIncludeExclude && !(followHreflang && isHreflang)) continue;
-        visitedSet.add(normalised);
-        queue.push({ url: normalised, depth: result.item.depth + 1 });
-      }
+      errors.push({ url: result.item.url, message: result.error });
+      continue;
+    }
+    if (result.item.attempt) recovered.push(result.item.url);
+    const finalUrl = canonicaliseUrl(result.page.fetch.finalUrl) ?? result.page.fetch.finalUrl;
+    if (seenFinal.has(finalUrl)) continue;
+    seenFinal.add(finalUrl);
+    pages.push(result.page);
+    if (options.onPage) {
+      options.onPage(result.page, { visited: pages.length, queued: queue.length });
+    }
+    if (pages.length >= maxPages) {
+      truncated = true;
+      break;
+    }
+    if (result.item.depth >= depthLimit) continue;
+    const candidates = extractCandidateLinks(result.page);
+    for (const candidate of candidates) {
+      const normalised = canonicaliseUrl(candidate.href, result.page.fetch.finalUrl);
+      if (!normalised) continue;
+      if (visitedSet.has(normalised)) continue;
+      const parsed = new URL(normalised);
+      if (parsed.origin !== origin) continue;
+      const passesIncludeExclude = filterAllows(parsed.pathname, include, exclude);
+      const isHreflang = candidate.source === "hreflang";
+      if (!passesIncludeExclude && !(followHreflang && isHreflang)) continue;
+      visitedSet.add(normalised);
+      queue.push({ url: normalised, depth: result.item.depth + 1 });
     }
-    if (truncated) break;
   }
   return {
     pages,
@@ -1277,20 +1409,6 @@ function filterAllows(pathname, include, exclude) {
   if (include.length === 0) return true;
   return matchesAny(pathname, include);
 }
-function canonicaliseUrl(href, base, options = {}) {
-  let url;
-  try {
-    url = new URL(href, base);
-  } catch {
-    return null;
-  }
-  if (url.protocol !== "http:" && url.protocol !== "https:") return null;
-  url.hash = "";
-  if (!options.preserveTrailingSlash && url.pathname.length > 1 && url.pathname.endsWith("/")) {
-    url.pathname = url.pathname.slice(0, -1);
-  }
-  return url.toString();
-}
 
 // src/lib/rules/evaluate.ts
 function bandFor(value, bands) {
@@ -4183,7 +4301,7 @@ function suggestedLocales(axis) {
 import { gunzipSync } from "zlib";
 
 // src/lib/core/sitemap/parse.ts
-import * as cheerio2 from "cheerio";
+import * as cheerio4 from "cheerio";
 function alternatesOf(node) {
   const out = [];
   const seen = /* @__PURE__ */ new Set();
@@ -4211,7 +4329,7 @@ function parseSitemap(body) {
   }
   let $;
   try {
-    $ = cheerio2.load(trimmed, { xmlMode: true });
+    $ = cheerio4.load(trimmed, { xmlMode: true });
   } catch {
     return { kind: "unknown", wellFormed: false, urls: [], sitemaps: [] };
   }
@@ -4608,6 +4726,10 @@ function normaliseContentType(raw) {
   const semi = raw.indexOf(";");
   return (semi === -1 ? raw : raw.slice(0, semi)).trim().toLowerCase() || void 0;
 }
+function capText(text, maxBytes) {
+  if (Buffer.byteLength(text, "utf8") <= maxBytes) return text;
+  return Buffer.from(text.slice(0, maxBytes), "utf8").subarray(0, maxBytes).toString("utf8");
+}
 async function readCappedText(res, maxBytes) {
   const reader = res.body?.getReader();
   if (!reader) {
@@ -4740,6 +4862,34 @@ async function checkLink(url, options = {}) {
     durationMs: Date.now() - started
   };
 }
+function checkFromAnswer(url, answer, options = {}) {
+  if (answer.status < 200 || answer.status >= 300) return void 0;
+  const outcome = {
+    status: answer.status,
+    finalUrl: url,
+    redirectChain: [],
+    loop: false,
+    body: capText(answer.body, options.softBodyBytes ?? DEFAULTS2.softBodyBytes),
+    method: "GET"
+  };
+  const softNotFound = isSoftNotFound(outcome);
+  return {
+    url,
+    finalUrl: url,
+    status: outcome.status,
+    verdict: classifyLink({
+      status: outcome.status,
+      redirected: false,
+      softNotFound,
+      loop: false
+    }),
+    method: "GET",
+    redirectChain: [],
+    reason: describeReason(outcome, softNotFound),
+    checkedAt: answer.checkedAt,
+    durationMs: answer.durationMs
+  };
+}
 async function probeWithRetry(url, options) {
   const retries = options.retries ?? DEFAULTS2.retries;
   const sleep = options.sleep ?? defaultSleep;
@@ -5608,89 +5758,6 @@ function collectSiteAdvisories(site, rules = SITE_PROSE_RULES) {
   return out;
 }
 
-// src/lib/core/links/extract.ts
-import * as cheerio3 from "cheerio";
-var ANCHOR_SPEC = { selector: "a[href]", attr: "href", source: "a" };
-var ASSET_SPECS = [
-  { selector: "img[src]", attr: "src", source: "img" },
-  { selector: "script[src]", attr: "src", source: "script" },
-  { selector: "link[href]", attr: "href", source: "link" },
-  { selector: "iframe[src]", attr: "src", source: "iframe" }
-];
-function extractLinks(html, options) {
-  const $ = cheerio3.load(html);
-  const base = resolveBase2($, options.baseUrl);
-  const origin = safeOrigin(options.baseUrl);
-  const specs = options.includeAssets ? [ANCHOR_SPEC, ...ASSET_SPECS] : [ANCHOR_SPEC];
-  const out = [];
-  const seen = /* @__PURE__ */ new Set();
-  for (const spec of specs) {
-    $(spec.selector).each((_, el) => {
-      const raw = $(el).attr(spec.attr);
-      if (raw === void 0) return;
-      const rawHref = raw.trim();
-      if (!rawHref) return;
-      const ref = buildRef($, el, rawHref, spec, base, origin);
-      if (!ref) return;
-      const dedupeKey = `${ref.source}|${ref.url}`;
-      if (seen.has(dedupeKey)) return;
-      seen.add(dedupeKey);
-      out.push(ref);
-    });
-  }
-  return out;
-}
-function buildRef($, el, rawHref, spec, base, origin) {
-  const fragment = extractFragment(rawHref);
-  const canonical = canonicaliseUrl(rawHref, base, { preserveTrailingSlash: true });
-  const url = canonical ?? rawHref;
-  const kind = canonical ? classify(canonical, origin) : "external";
-  const ref = {
-    rawHref,
-    url,
-    kind,
-    source: spec.source,
-    rel: spec.source === "a" || spec.source === "link" ? parseRel($(el).attr("rel")) : []
-  };
-  if (spec.source === "a") {
-    const text = $(el).text().trim();
-    if (text) ref.anchorText = text;
-  }
-  if (fragment) ref.fragment = fragment;
-  return ref;
-}
-function extractFragment(rawHref) {
-  const hash = rawHref.indexOf("#");
-  if (hash === -1) return void 0;
-  const frag = rawHref.slice(hash);
-  return frag.length > 1 ? frag : void 0;
-}
-function parseRel(rel) {
-  if (!rel) return [];
-  return rel.split(/\s+/).map((t) => t.trim().toLowerCase()).filter(Boolean);
-}
-function classify(url, origin) {
-  if (!origin) return "external";
-  return safeOrigin(url) === origin ? "internal" : "external";
-}
-function safeOrigin(url) {
-  try {
-    return new URL(url).origin;
-  } catch {
-    return null;
-  }
-}
-function resolveBase2($, pageUrl) {
-  const declared = $("base[href]").first().attr("href")?.trim();
-  if (declared) {
-    try {
-      return new URL(declared, pageUrl).toString();
-    } catch {
-    }
-  }
-  return pageUrl;
-}
-
 // src/lib/core/links/types.ts
 function emptyVerdictSummary() {
   return { ok: 0, redirect: 0, broken: 0, blocked: 0, warning: 0, skipped: 0 };
@@ -5722,8 +5789,16 @@ async function runLinkAudit(discovery, options = {}) {
   let pagesScanned = 0;
   let pagesFailed = 0;
   let scanDone = 0;
+  const maxBytes = options.maxScanBytes ?? DEFAULTS3.maxScanBytes;
+  const handedPages = [];
+  const toFetch = [];
+  for (const pageUrl of pageUrls) {
+    const handed = options.pages?.get(pageUrl);
+    if (handed) handedPages.push([pageUrl, handed]);
+    else toFetch.push(pageUrl);
+  }
   await mapWithHostCaps(
-    pageUrls,
+    toFetch,
     {
       concurrency: options.scanConcurrency ?? DEFAULTS3.scanConcurrency,
       perHost: Infinity,
@@ -5735,7 +5810,7 @@ async function runLinkAudit(discovery, options = {}) {
         timeoutMs: options.timeoutMs ?? DEFAULTS3.timeoutMs,
         allowInsecureTls: options.allowInsecureTls,
         userAgent: options.userAgent,
-        maxBytes: options.maxScanBytes ?? DEFAULTS3.maxScanBytes,
+        maxBytes,
         signal: options.signal
       });
       if (res.status === 0 || res.status >= 400 || res.body === void 0) {
@@ -5753,6 +5828,17 @@ async function runLinkAudit(discovery, options = {}) {
     },
     () => "scan"
   );
+  for (const [pageUrl, handed] of handedPages) {
+    if (options.signal?.aborted) break;
+    const refs = handed.links && !options.includeAssets && Buffer.byteLength(handed.html) <= maxBytes ? handed.links : extractLinks(capText(handed.html, maxBytes), {
+      baseUrl: pageUrl,
+      includeAssets: options.includeAssets
+    });
+    for (const ref of refs) occurrences.push({ pageUrl, ref });
+    pagesScanned += 1;
+    scanDone += 1;
+    options.onProgress?.({ phase: "scan", done: scanDone, total: pageUrls.length });
+  }
   const uniqueUrls = [];
   const seen = /* @__PURE__ */ new Set();
   for (const occ of occurrences) {
@@ -5769,8 +5855,20 @@ async function runLinkAudit(discovery, options = {}) {
   }
   const checks = {};
   let checkDone = 0;
+  const toProbe = [];
+  for (const url of uniqueUrls) {
+    const answer = options.answers?.get(url);
+    const check = answer ? checkFromAnswer(url, answer) : void 0;
+    if (!check) {
+      toProbe.push(url);
+      continue;
+    }
+    checks[url] = check;
+    checkDone += 1;
+    options.onProgress?.({ phase: "check", done: checkDone, total: uniqueUrls.length });
+  }
   await mapWithHostCaps(
-    uniqueUrls,
+    toProbe,
     {
       concurrency: options.checkConcurrency ?? DEFAULTS3.checkConcurrency,
       perHost: options.maxPerHost ?? DEFAULTS3.maxPerHost,
@@ -6136,6 +6234,7 @@ async function runAudit(inputUrl, options = {}) {
     inspectOptions: {
       mode: options.static ? "static" : "auto",
       probes: false,
+      linkRefs: true,
       allowInsecureTls: options.allowInsecureTls,
       timeoutMs: options.timeoutMs,
       signal: options.signal
@@ -6276,6 +6375,16 @@ async function runAudit(inputUrl, options = {}) {
     checkExternal,
     timeoutMs: options.timeoutMs,
     maxPages: effectiveMaxPages,
+    pages: new Map(
+      okPages.filter(isHtmlPage).map((page) => [
+        page.fetch.finalUrl,
+        {
+          html: page.html.static,
+          links: page.extractor.escalated ? void 0 : page.linkRefs
+        }
+      ])
+    ),
+    answers: crawlAnswers(okPages),
     signal: options.signal,
     onProgress: (p) => options.onProgress?.({
       phase: p.phase === "scan" ? "scan" : "links",
@@ -6487,6 +6596,21 @@ function sameOriginAs(url, origin) {
     return false;
   }
 }
+function crawlAnswers(pages) {
+  const answers = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    if (!isOkStatus(page.fetch.status) || !isHtmlPage(page)) continue;
+    if (page.extractor.escalated) continue;
+    if (page.fetch.redirectCount > 0 || page.fetch.finalUrl !== page.fetch.requestedUrl) continue;
+    answers.set(page.fetch.requestedUrl, {
+      status: page.fetch.status,
+      body: page.html.static,
+      checkedAt: page.fetchedAt,
+      durationMs: page.fetch.durationMs
+    });
+  }
+  return answers;
+}
 function syntheticDiscovery(origin, baseUrl, pages, truncated) {
   return {
     origin,
@@ -7835,4 +7959,4 @@ export {
   summarize,
   Logger
 };
-//# sourceMappingURL=chunk-YQPL67HX.js.map
+//# sourceMappingURL=chunk-KHIN5TNX.js.map