mcp-scraper 0.76.0 → 0.78.0

package/dist/index.cjs

@@ -128,11 +128,23 @@ var MapsSearchOptionsSchema = import_zod.z.object({
   headless: import_zod.z.boolean().default(true)
 });
 var RawPAAItemSchema = import_zod.z.object({
+  questionId: import_zod.z.string().min(1),
+  googleLinkId: import_zod.z.string().nullable(),
+  googleEvidenceId: import_zod.z.string().nullable(),
   question: import_zod.z.string().min(1),
   answer: import_zod.z.string().optional(),
   sourceTitle: import_zod.z.string().optional(),
   sourceSite: import_zod.z.string().optional(),
-  sourceCite: import_zod.z.string().optional()
+  sourceCite: import_zod.z.string().optional(),
+  sources: import_zod.z.array(import_zod.z.object({
+    title: import_zod.z.string().nullable(),
+    site: import_zod.z.string().nullable(),
+    url: import_zod.z.string().min(1),
+    rawUrl: import_zod.z.string().min(1),
+    resolvedUrl: import_zod.z.string().nullable(),
+    linkType: import_zod.z.enum(["plain", "google_url_redirect", "google_goto_redirect"]),
+    resolutionStatus: import_zod.z.enum(["not_needed", "resolved", "unresolved", "rejected"])
+  })).default([])
 });
 var RawMapsOverviewSchema = import_zod.z.object({
   name: import_zod.z.string().nullable(),
@@ -373,6 +385,9 @@ var RequestAbortedError = class extends Error {
     super(message);
   }
 };
+var PaaInteractionInterruptedError = class extends Error {
+  name = "PaaInteractionInterruptedError";
+};
 var LocationMismatchError = class extends Error {
   name = "LocationMismatchError";
   constructor(message = "Google returned results for a different location than requested") {
@@ -2254,16 +2269,15 @@ async function extractAISurfacesFromDocument(config) {
     if (textOf(heading) === "AI Overview") return true;
     const header = el.querySelector(selectors.aio.header);
     if (textOf(header).split(/\n|\s{2,}/).some((part) => part.trim() === "AI Overview")) return true;
-    return textOf(el).includes("AI Overview");
+    return false;
   }
   function findAIORoot() {
     const primaryRoots = Array.from(document.querySelectorAll(selectors.aio.root));
     const labeledPrimary = primaryRoots.find(hasAIOverviewLabel);
     if (labeledPrimary) return labeledPrimary;
-    if (primaryRoots.length > 0) return primaryRoots[0];
     if (selectors.aio.legacyRoot) {
       const legacy = document.querySelector(selectors.aio.legacyRoot);
-      if (legacy) return legacy;
+      if (legacy && hasAIOverviewLabel(legacy)) return legacy;
     }
     const headings = document.querySelectorAll(`${selectors.aio.heading}, h1, h2, h3, [role="heading"]`);
     for (const h of headings) {
@@ -2342,26 +2356,12 @@ async function extractAISurfacesFromDocument(config) {
     if (!raw || /not available|try again|can't generate/i.test(raw)) return null;
     return raw;
   }
-  function normalizeHref(rawHref) {
+  function absoluteHref(rawHref) {
     if (!rawHref || rawHref.startsWith("javascript:")) return null;
-    let href = rawHref;
     try {
       const absolute = new URL(rawHref, window.location.href);
-      const q = absolute.searchParams.get("q") ?? absolute.searchParams.get("url");
-      if (/(\.|^)google\./i.test(absolute.hostname) && q?.startsWith("http")) {
-        href = q;
-      } else {
-        href = absolute.href;
-      }
-    } catch {
-      return null;
-    }
-    if (!/^https?:\/\//i.test(href)) return null;
-    try {
-      const url = new URL(href);
-      const isGoogleInternal = /(\.|^)google\./i.test(url.hostname);
-      if (isGoogleInternal) return null;
-      return url.href;
+      if (!["http:", "https:"].includes(absolute.protocol) || absolute.username || absolute.password) return null;
+      return { href: absolute.href, rawHref: absolute.href };
     } catch {
       return null;
     }
@@ -2371,17 +2371,18 @@ async function extractAISurfacesFromDocument(config) {
     const seen = /* @__PURE__ */ new Set();
     const citations = [];
     for (const a of Array.from(root.querySelectorAll("a[href]"))) {
-      const href = normalizeHref(a.getAttribute("href") ?? "");
-      if (!href || seen.has(href)) continue;
-      seen.add(href);
+      const link = absoluteHref(a.getAttribute("href") ?? "");
+      if (!link || seen.has(link.rawHref)) continue;
+      seen.add(link.rawHref);
       let fallbackHost = "";
       try {
-        fallbackHost = new URL(href).hostname.replace(/^www\./, "");
+        fallbackHost = new URL(link.href).hostname.replace(/^www\./, "");
       } catch {
       }
       citations.push({
-        text: textOf(a) || fallbackHost || href,
-        href
+        text: textOf(a) || fallbackHost || link.href,
+        href: link.href,
+        rawHref: link.rawHref
       });
     }
     return citations;
@@ -2436,6 +2437,286 @@ async function extractAISurfacesFromDocument(config) {
   };
 }
 
+// src/extractor/google-outbound-links.ts
+var import_node_crypto3 = require("crypto");
+var MAX_LINK_LENGTH = 8e3;
+var DEFAULT_TIMEOUT_MS = 3e3;
+var DEFAULT_TOTAL_BUDGET_MS = 1e4;
+var DEFAULT_HTTP_CONCURRENCY = 8;
+function isGoogleHost(hostname) {
+  return /(^|\.)google\.(?:com|[a-z]{2,3}(?:\.[a-z]{2})?)$/i.test(hostname);
+}
+function safeHttpUrl(value) {
+  if (!value || value.length > MAX_LINK_LENGTH) return null;
+  try {
+    const url = new URL(value);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+    if (url.username || url.password) return null;
+    return url;
+  } catch {
+    return null;
+  }
+}
+function absoluteHttpUrl(value, baseUrl) {
+  if (!value || value.length > MAX_LINK_LENGTH || /^javascript:/i.test(value)) return null;
+  try {
+    const absolute = new URL(value, baseUrl);
+    return safeHttpUrl(absolute.href);
+  } catch {
+    return null;
+  }
+}
+function parsedGoogleGoto(value) {
+  const url = safeHttpUrl(value);
+  if (!url || url.protocol !== "https:" || !isGoogleHost(url.hostname) || url.pathname !== "/goto") return null;
+  return url.searchParams.get("url") ? url : null;
+}
+function questionIdFor(question) {
+  const normalized = question.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu, "").replace(/\s+/g, " ").trim();
+  return `paa_${(0, import_node_crypto3.createHash)("sha256").update(normalized).digest("hex").slice(0, 16)}`;
+}
+function classifyGoogleOutboundLink(rawHref, baseUrl) {
+  const absolute = absoluteHttpUrl(rawHref, baseUrl);
+  if (!absolute) return null;
+  const rawUrl = absolute.href;
+  if (!isGoogleHost(absolute.hostname)) {
+    return {
+      url: rawUrl,
+      rawUrl,
+      resolvedUrl: rawUrl,
+      linkType: "plain",
+      resolutionStatus: "not_needed"
+    };
+  }
+  if (absolute.pathname === "/url") {
+    const target = safeHttpUrl(absolute.searchParams.get("q") ?? absolute.searchParams.get("url") ?? "");
+    if (!target || isGoogleHost(target.hostname)) return null;
+    return {
+      url: target.href,
+      rawUrl,
+      resolvedUrl: target.href,
+      linkType: "google_url_redirect",
+      resolutionStatus: "resolved"
+    };
+  }
+  if (absolute.pathname === "/goto" && absolute.searchParams.get("url")) {
+    return {
+      url: rawUrl,
+      rawUrl,
+      resolvedUrl: null,
+      linkType: "google_goto_redirect",
+      resolutionStatus: "unresolved"
+    };
+  }
+  return null;
+}
+function tokenFor(value) {
+  return parsedGoogleGoto(value)?.searchParams.get("url") ?? null;
+}
+function externalRedirectTarget(source, location) {
+  try {
+    const sourceUrl = new URL(source);
+    const target = safeHttpUrl(new URL(location, sourceUrl).href);
+    if (!target || isGoogleHost(target.hostname)) return null;
+    if (!target.hash && sourceUrl.hash) target.hash = sourceUrl.hash;
+    return target.href;
+  } catch {
+    return null;
+  }
+}
+async function resolveGoogleGotoUrls(rawUrls, options = {}) {
+  const inputs = [...new Set([...rawUrls].flatMap((value) => parsedGoogleGoto(value) ? [value] : []))];
+  const records = new Map(inputs.map((rawUrl) => [rawUrl, {
+    rawUrl,
+    resolvedUrl: null,
+    resolutionStatus: "unresolved"
+  }]));
+  const fallbackFetch = typeof globalThis.fetch === "function" ? globalThis.fetch.bind(globalThis) : null;
+  const fetchImpl = options.fetchImpl ?? fallbackFetch;
+  if (inputs.length === 0 || !fetchImpl) {
+    return { resolutions: [...records.values()], nonGoogleRequestsObserved: 0 };
+  }
+  const timeoutMs = Math.max(1, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+  const deadlineMs = Date.now() + Math.max(1, options.totalBudgetMs ?? DEFAULT_TOTAL_BUDGET_MS);
+  const concurrency = Math.max(1, Math.min(16, Math.floor(options.concurrency ?? DEFAULT_HTTP_CONCURRENCY)));
+  let cursor = 0;
+  const worker = async () => {
+    while (cursor < inputs.length) {
+      const rawUrl = inputs[cursor++];
+      const source = parsedGoogleGoto(rawUrl);
+      const expectedToken = source?.searchParams.get("url");
+      if (!source || !expectedToken) {
+        records.set(rawUrl, { rawUrl, resolvedUrl: null, resolutionStatus: "rejected" });
+        continue;
+      }
+      source.hash = "";
+      let currentUrl = source.href;
+      for (let hop = 0; hop < 2; hop++) {
+        const remainingMs = deadlineMs - Date.now();
+        if (remainingMs <= 0) return;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), Math.min(timeoutMs, remainingMs));
+        let response;
+        try {
+          response = await fetchImpl(currentUrl, {
+            method: "GET",
+            redirect: "manual",
+            headers: {
+              accept: "*/*",
+              "user-agent": "Mozilla/5.0"
+            },
+            signal: controller.signal
+          });
+        } catch {
+          break;
+        } finally {
+          clearTimeout(timer);
+        }
+        if (response.status < 300 || response.status >= 400) break;
+        const location = response.headers.get("location");
+        if (!location) break;
+        const destination = externalRedirectTarget(rawUrl, location);
+        if (destination) {
+          records.set(rawUrl, { rawUrl, resolvedUrl: destination, resolutionStatus: "resolved" });
+          break;
+        }
+        let next;
+        try {
+          next = new URL(location, currentUrl);
+        } catch {
+          records.set(rawUrl, { rawUrl, resolvedUrl: null, resolutionStatus: "rejected" });
+          break;
+        }
+        if (!isGoogleHost(next.hostname) || tokenFor(next.href) !== expectedToken) {
+          records.set(rawUrl, { rawUrl, resolvedUrl: null, resolutionStatus: "rejected" });
+          break;
+        }
+        currentUrl = next.href;
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, inputs.length) }, () => worker()));
+  return { resolutions: [...records.values()], nonGoogleRequestsObserved: 0 };
+}
+async function resolveGoogleOutboundLinks(page, values, options = {}) {
+  const byRawUrl = /* @__PURE__ */ new Map();
+  for (const value of values) byRawUrl.set(value.rawUrl, value);
+  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect");
+  if (inputs.length === 0) return byRawUrl;
+  const deadlineMs = Date.now() + (options.totalBudgetMs ?? DEFAULT_TOTAL_BUDGET_MS);
+  const direct = await resolveGoogleGotoUrls(inputs.map((value) => value.rawUrl), options);
+  for (const record of direct.resolutions) {
+    const input = byRawUrl.get(record.rawUrl);
+    if (!input) continue;
+    byRawUrl.set(record.rawUrl, record.resolutionStatus === "resolved" && record.resolvedUrl ? { ...input, url: record.resolvedUrl, resolvedUrl: record.resolvedUrl, resolutionStatus: "resolved" } : { ...input, resolutionStatus: record.resolutionStatus });
+  }
+  const unresolvedInputs = inputs.filter((input) => byRawUrl.get(input.rawUrl)?.resolutionStatus === "unresolved");
+  if (unresolvedInputs.length === 0 || options.browserFallback === false || Date.now() >= deadlineMs) {
+    options.onNetworkAudit?.({ nonGoogleRequestsObserved: 0 });
+    return byRawUrl;
+  }
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const resolverPage = await page.context().newPage().catch(() => null);
+  if (!resolverPage) {
+    options.onNetworkAudit?.({ nonGoogleRequestsObserved: 0 });
+    return byRawUrl;
+  }
+  let active = null;
+  let nonGoogleRequestsObserved = 0;
+  try {
+    await resolverPage.route("**/*", async (route) => {
+      const requestUrl = route.request().url();
+      try {
+        if (!isGoogleHost(new URL(requestUrl).hostname)) nonGoogleRequestsObserved += 1;
+      } catch {
+      }
+      const requestToken = tokenFor(requestUrl);
+      if (!active || requestToken !== active.token) {
+        await route.abort("blockedbyclient").catch(() => {
+        });
+        return;
+      }
+      const target = active;
+      try {
+        const response = await route.fetch({ maxRedirects: 0, timeout: target.timeoutMs });
+        target.location = response.headers().location ?? null;
+      } finally {
+        await route.abort("blockedbyclient").catch(() => {
+        });
+        target.finish();
+      }
+    });
+    for (const input of unresolvedInputs) {
+      const remainingMs = deadlineMs - Date.now();
+      if (remainingMs <= 0) break;
+      const source = parsedGoogleGoto(input.rawUrl);
+      const expectedToken = source?.searchParams.get("url");
+      if (!source || !expectedToken) {
+        byRawUrl.set(input.rawUrl, { ...input, resolutionStatus: "rejected" });
+        continue;
+      }
+      const attemptTimeoutMs = Math.min(timeoutMs, remainingMs);
+      source.hash = "";
+      let currentUrl = source.href;
+      let rejected = false;
+      for (let hop = 0; hop < 2; hop++) {
+        let finish;
+        const intercepted = new Promise((resolve) => {
+          finish = resolve;
+        });
+        const frameId = `mcp-goto-${Math.random().toString(36).slice(2)}`;
+        active = { token: expectedToken, timeoutMs: attemptTimeoutMs, location: null, finish };
+        await resolverPage.evaluate(({ url, id }) => {
+          const frame = document.createElement("iframe");
+          frame.id = id;
+          frame.hidden = true;
+          frame.src = url;
+          document.body.append(frame);
+        }, { url: currentUrl, id: frameId }).catch(() => null);
+        await Promise.race([
+          intercepted,
+          new Promise((resolve) => setTimeout(resolve, attemptTimeoutMs))
+        ]);
+        const location = active.location;
+        active = null;
+        await resolverPage.evaluate((id) => document.getElementById(id)?.remove(), frameId).catch(() => null);
+        if (!location) break;
+        const destination = externalRedirectTarget(input.rawUrl, location);
+        if (destination) {
+          byRawUrl.set(input.rawUrl, {
+            ...input,
+            url: destination,
+            resolvedUrl: destination,
+            resolutionStatus: "resolved"
+          });
+          break;
+        }
+        let next;
+        try {
+          next = new URL(location, currentUrl);
+        } catch {
+          rejected = true;
+          break;
+        }
+        if (!isGoogleHost(next.hostname) || tokenFor(next.href) !== expectedToken) {
+          rejected = true;
+          break;
+        }
+        currentUrl = next.href;
+      }
+      if (rejected && byRawUrl.get(input.rawUrl)?.resolutionStatus !== "resolved") {
+        byRawUrl.set(input.rawUrl, { ...input, resolutionStatus: "rejected" });
+      }
+    }
+  } finally {
+    active = null;
+    await resolverPage.close().catch(() => {
+    });
+    options.onNetworkAudit?.({ nonGoogleRequestsObserved });
+  }
+  return byRawUrl;
+}
+
 // src/extractor/PAAExtractor.ts
 var PAAExtractor = class {
   constructor(driver, reporter, onPaaProgress) {
@@ -2453,10 +2734,118 @@ var PAAExtractor = class {
     aioShareCaptured: null
   };
   aioShareUrlEarly = null;
+  linkDiagnostics = this.emptyLinkDiagnostics();
   collectedRows = [];
   partialContext = null;
   progressSequence = 0;
   clickedQuestions = /* @__PURE__ */ new Set();
+  paaLifecycle = this.emptyPaaLifecycle(0);
+  checkpointMaterial = null;
+  emptyPaaLifecycle(requestedQuestions) {
+    return {
+      requestedQuestions,
+      questionsFound: 0,
+      targetStatus: "not_met",
+      discoveryStatus: "in_progress",
+      materialStatus: "partial",
+      completionReason: "PAA expansion is still running.",
+      automaticRetries: 0,
+      knownQuestions: 0,
+      processedQuestions: 0,
+      failedInteractions: 0,
+      noGrowthConfirmations: 0
+    };
+  }
+  emptyLinkDiagnostics() {
+    return {
+      linkCandidatesObserved: 0,
+      plainLinksObserved: 0,
+      googleUrlRedirectsObserved: 0,
+      googleGotoRedirectsObserved: 0,
+      redirectsResolved: 0,
+      redirectsUnresolved: 0,
+      redirectsRejected: 0,
+      resolverNonGoogleRequestsObserved: 0,
+      paaSourcesPreserved: 0,
+      aiOverviewLabelValidated: false
+    };
+  }
+  mergePaaSources(existing = [], incoming = []) {
+    const byRawUrl = /* @__PURE__ */ new Map();
+    for (const source of [...existing, ...incoming]) {
+      const current = byRawUrl.get(source.rawUrl);
+      if (!current) {
+        byRawUrl.set(source.rawUrl, source);
+        continue;
+      }
+      const selected = source.resolutionStatus === "resolved" && current.resolutionStatus !== "resolved" ? source : current;
+      byRawUrl.set(source.rawUrl, { ...selected, title: selected.title ?? current.title ?? source.title, site: selected.site ?? current.site ?? source.site });
+    }
+    return [...byRawUrl.values()];
+  }
+  normalizeAICitations(pageUrl, citations) {
+    const seen = /* @__PURE__ */ new Set();
+    return citations.flatMap((citation) => {
+      const link = classifyGoogleOutboundLink(citation.rawHref ?? citation.href, pageUrl);
+      if (!link || seen.has(link.rawUrl)) return [];
+      seen.add(link.rawUrl);
+      const { url, ...metadata } = link;
+      return [{ ...metadata, text: citation.text, href: url }];
+    });
+  }
+  async resolveMaterialLinks(page, flat, organicResults, aiOverview, aiMode) {
+    const candidates = [
+      ...flat.flatMap((row) => row.sources ?? []),
+      ...organicResults,
+      ...aiOverview.citations.map(({ text: _text, href, ...link }) => ({ ...link, url: href })),
+      ...aiMode.citations.map(({ text: _text, href, ...link }) => ({ ...link, url: href }))
+    ];
+    const unique = new Map(candidates.map((link) => [link.rawUrl, link]));
+    let resolverNonGoogleRequestsObserved = 0;
+    const resolved = await resolveGoogleOutboundLinks(page, unique.values(), {
+      onNetworkAudit: (audit) => {
+        resolverNonGoogleRequestsObserved = audit.nonGoogleRequestsObserved;
+      }
+    });
+    const replace = (link) => ({ ...link, ...resolved.get(link.rawUrl) ?? link });
+    for (const row of flat) {
+      row.sources = (row.sources ?? []).map(replace);
+      const primary = row.sources[0];
+      row.source_title = primary?.title ?? row.source_title;
+      row.source_site = primary?.site ?? row.source_site;
+      row.source_cite = primary?.url ?? row.source_cite;
+    }
+    for (let index = 0; index < organicResults.length; index++) {
+      const next = replace(organicResults[index]);
+      if (next.resolvedUrl) {
+        try {
+          next.domain = new URL(next.resolvedUrl).hostname.replace(/^www\./, "");
+        } catch {
+        }
+      }
+      organicResults[index] = next;
+    }
+    const replaceCitation = (citation) => {
+      const link = replace({ url: citation.href, rawUrl: citation.rawUrl, resolvedUrl: citation.resolvedUrl, linkType: citation.linkType, resolutionStatus: citation.resolutionStatus });
+      const { url, ...metadata } = link;
+      return { ...metadata, text: citation.text, href: url };
+    };
+    aiOverview.citations = aiOverview.citations.map(replaceCitation);
+    aiMode.citations = aiMode.citations.map(replaceCitation);
+    const values = [...resolved.values()];
+    this.linkDiagnostics = {
+      linkCandidatesObserved: unique.size,
+      plainLinksObserved: values.filter((link) => link.linkType === "plain").length,
+      googleUrlRedirectsObserved: values.filter((link) => link.linkType === "google_url_redirect").length,
+      googleGotoRedirectsObserved: values.filter((link) => link.linkType === "google_goto_redirect").length,
+      redirectsResolved: values.filter((link) => link.linkType !== "plain" && link.resolutionStatus === "resolved").length,
+      redirectsUnresolved: values.filter((link) => link.resolutionStatus === "unresolved").length,
+      redirectsRejected: values.filter((link) => link.resolutionStatus === "rejected").length,
+      resolverNonGoogleRequestsObserved,
+      paaSourcesPreserved: flat.reduce((total, row) => total + (row.sources ?? []).length, 0),
+      aiOverviewLabelValidated: aiOverview.detected
+    };
+  }
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -2493,34 +2882,6 @@ var PAAExtractor = class {
       function cleanSourceLabel(value) {
         return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
       }
-      function isGoogleHost(host) {
-        const normalized = host.toLowerCase().replace(/\.$/, "");
-        return normalized === "google.com" || normalized.endsWith(".google.com") || /^google\.[a-z.]+$/.test(normalized) || /\.google\.[a-z.]+$/.test(normalized) || normalized === "googleusercontent.com" || normalized.endsWith(".googleusercontent.com");
-      }
-      function safeCitationTarget(rawHref) {
-        let parsed;
-        try {
-          parsed = new URL(rawHref);
-        } catch {
-          return null;
-        }
-        if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
-        if (parsed.username || parsed.password) return null;
-        if (isGoogleHost(parsed.hostname)) {
-          if (parsed.pathname !== "/url") return null;
-          const redirected = parsed.searchParams.get("q") ?? parsed.searchParams.get("url");
-          if (!redirected) return null;
-          try {
-            parsed = new URL(redirected);
-          } catch {
-            return null;
-          }
-          if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
-          if (parsed.username || parsed.password) return null;
-        }
-        if (isGoogleHost(parsed.hostname)) return null;
-        return { href: parsed.href, host: parsed.hostname };
-      }
       return Array.from(document.querySelectorAll(selectors.item)).filter((pair) => !onlyQuestion2 || (pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "") === onlyQuestion2).map((pair) => {
         const clickTarget = pair.querySelector(selectors.clickTarget);
         const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
@@ -2528,50 +2889,62 @@ var PAAExtractor = class {
         const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
         const answerElement = aiAnswerBody ?? answerContainer;
         const answerText = cleanText(answerElement);
-        const hasLoadedAnswer = expanded || answerText.length > 30 && !/an error has occurred/i.test(answerText.slice(0, 80));
+        const citationCandidates = [
+          ...Array.from((answerElement ?? pair).querySelectorAll("a[href]")),
+          ...Array.from(pair.querySelectorAll(selectors.aiCitation)),
+          ...Array.from(pair.querySelectorAll(selectors.aiSourceCard))
+        ];
+        const hasLoadedAnswer = expanded || answerText.length > 30 && !/an error has occurred/i.test(answerText.slice(0, 80)) && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href));
         let sourceTitle = hasLoadedAnswer ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
         let sourceSite = hasLoadedAnswer ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
-        let sourceCite = hasLoadedAnswer ? pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0 : void 0;
-        if (hasLoadedAnswer && !sourceCite) {
+        const sources = [];
+        if (hasLoadedAnswer) {
           const seenHrefs = /* @__PURE__ */ new Set();
-          const anchors = [];
-          const anchorRoot = answerElement ?? pair;
-          const prioritized = [
-            ...Array.from(anchorRoot.querySelectorAll(selectors.aiCitation)),
-            ...Array.from(anchorRoot.querySelectorAll(selectors.aiSourceCard)),
-            ...Array.from(anchorRoot.querySelectorAll("a[href]"))
-          ];
-          for (const a of prioritized) {
-            const target = safeCitationTarget(a.href);
-            if (!target || seenHrefs.has(target.href)) continue;
-            seenHrefs.add(target.href);
-            anchors.push({
-              text: (a.textContent || "").trim(),
-              label: cleanSourceLabel(a.getAttribute("aria-label")),
-              href: target.href,
-              host: target.host
-            });
-          }
-          const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
-          if (primary) {
-            sourceTitle = sourceTitle || primary.label || primary.text || void 0;
-            sourceSite = sourceSite || primary.host.replace(/^www\./, "");
-            sourceCite = primary.href;
+          for (const a of citationCandidates) {
+            if (!/^https?:\/\//.test(a.href) || seenHrefs.has(a.href)) continue;
+            seenHrefs.add(a.href);
+            sources.push({ title: cleanSourceLabel(a.getAttribute("aria-label")) || (a.textContent || "").trim(), rawHref: a.href });
           }
+          const primary = sources.find((x) => x.title.length > 2) ?? sources[0];
+          if (primary) sourceTitle = sourceTitle || primary.title || void 0;
         }
         return {
+          googleLinkId: pair.getAttribute("data-lk") || pair.querySelector("[data-lk]")?.getAttribute("data-lk") || null,
+          googleEvidenceId: pair.getAttribute("data-ved") || pair.querySelector("[data-ved]")?.getAttribute("data-ved") || null,
           question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
           answer: hasLoadedAnswer ? answerText || void 0 : void 0,
           sourceTitle,
           sourceSite,
-          sourceCite
+          sources
         };
       });
     }, { selectors: sels, onlyQuestion });
+    const baseUrl = typeof page.url === "function" ? page.url() : "https://www.google.com/";
     return raw.flatMap((item) => {
+      const sources = [];
+      const seen = /* @__PURE__ */ new Set();
+      for (const candidate of item.sources) {
+        const link = classifyGoogleOutboundLink(candidate.rawHref, baseUrl);
+        if (!link || seen.has(link.rawUrl)) continue;
+        seen.add(link.rawUrl);
+        let site = item.sourceSite ?? null;
+        try {
+          site ||= new URL(link.resolvedUrl ?? link.rawUrl).hostname.replace(/^www\./, "");
+        } catch {
+        }
+        sources.push({ ...link, title: candidate.title || item.sourceTitle || null, site });
+      }
+      const primary = sources[0];
       const cleaned = {
-        ...item,
-        answer: cleanPAAAnswerText(item.answer, item.question, item.sourceTitle)
+        questionId: questionIdFor(item.question),
+        googleLinkId: item.googleLinkId,
+        googleEvidenceId: item.googleEvidenceId,
+        question: item.question,
+        answer: cleanPAAAnswerText(item.answer, item.question, item.sourceTitle),
+        sourceTitle: primary?.title ?? item.sourceTitle,
+        sourceSite: primary?.site ?? item.sourceSite,
+        sourceCite: primary?.url,
+        sources
       };
       const result = RawPAAItemSchema.safeParse(cleaned);
       if (!result.success) {
@@ -2596,11 +2969,15 @@ var PAAExtractor = class {
   toFlatRow(item, depth, parentQuestion, seed) {
     return {
       seed_query: seed,
+      question_id: item.questionId ?? questionIdFor(item.question),
+      google_link_id: item.googleLinkId ?? "",
+      google_evidence_id: item.googleEvidenceId ?? "",
       question: item.question,
       answer: item.answer ?? "",
       source_title: item.sourceTitle ?? "",
       source_site: item.sourceSite ?? "",
       source_cite: item.sourceCite ?? "",
+      sources: item.sources ?? [],
       depth,
       parent_question: parentQuestion ?? "",
       extracted_at: (/* @__PURE__ */ new Date()).toISOString()
@@ -2621,20 +2998,23 @@ var PAAExtractor = class {
       source_title: row.source_title || existing.source_title,
       source_site: row.source_site || existing.source_site,
       source_cite: row.source_cite || existing.source_cite,
+      sources: this.mergePaaSources(existing.sources, row.sources),
       extracted_at: row.extracted_at || existing.extracted_at
     };
-    const changed = merged.answer !== existing.answer || merged.source_title !== existing.source_title || merged.source_site !== existing.source_site || merged.source_cite !== existing.source_cite;
+    const changed = merged.answer !== existing.answer || merged.source_title !== existing.source_title || merged.source_site !== existing.source_site || merged.source_cite !== existing.source_cite || JSON.stringify(merged.sources) !== JSON.stringify(existing.sources);
     if (changed) this.collectedRows[index] = merged;
     return changed;
   }
   async emitProgress(phase) {
-    if (!this.onPaaProgress || this.collectedRows.length === 0) return;
+    if (!this.onPaaProgress || this.collectedRows.length === 0 && !this.checkpointMaterial) return;
     const snapshot = {
       sequence: ++this.progressSequence,
       observedAt: (/* @__PURE__ */ new Date()).toISOString(),
       clickedQuestions: [...this.clickedQuestions],
       phase,
-      records: this.collectedRows.map((row) => ({ ...row }))
+      records: this.collectedRows.map((row) => ({ ...row })),
+      lifecycle: { ...this.paaLifecycle },
+      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {}
     };
     try {
       await this.onPaaProgress(snapshot);
@@ -2649,10 +3029,11 @@ var PAAExtractor = class {
   }
   getPartialResult() {
     const ctx = this.partialContext;
-    if (!ctx || this.collectedRows.length === 0) return null;
+    if (!ctx || this.collectedRows.length === 0 && !this.checkpointMaterial) return null;
     const flat = this.collectedRows.map((row) => ({ ...row }));
+    const material = this.checkpointMaterial;
     const paaWithoutAnswer = ctx.questionsOnly ? 0 : flat.filter((row) => !row.answer?.trim()).length;
-    const paaWithoutSource = ctx.questionsOnly ? 0 : flat.filter((row) => !row.source_cite?.trim()).length;
+    const paaWithoutSource = ctx.questionsOnly ? 0 : flat.filter((row) => (row.sources ?? []).length === 0).length;
     const stats = {
       seed: ctx.seed,
       totalQuestions: flat.length,
@@ -2670,6 +3051,14 @@ var PAAExtractor = class {
         resultQuality: "partial",
         degradedResult: false,
         retryRecommended: true,
+        paaLifecycle: {
+          ...this.paaLifecycle,
+          questionsFound: flat.length,
+          targetStatus: flat.length >= (ctx.maxQuestions ?? 0) ? "met" : "not_met",
+          discoveryStatus: "interrupted",
+          materialStatus: paaWithoutAnswer > 0 || paaWithoutSource > 0 ? "partial" : "complete",
+          completionReason: "The browser attempt ended before frontier exhaustion could be proven."
+        },
         completeness: {
           ...this.completeness,
           paaWithoutAnswer,
@@ -2686,17 +3075,17 @@ var PAAExtractor = class {
         }]
       },
       totalQuestions: flat.length,
-      surface: "web",
-      aiOverview: { detected: false, text: null, citations: [] },
-      aiMode: { detected: false, text: null, citations: [] },
-      whatPeopleSaying: [],
+      surface: material?.surface ?? "web",
+      aiOverview: material?.aiOverview ?? { detected: false, text: null, citations: [] },
+      aiMode: material?.aiMode ?? { detected: false, text: null, citations: [] },
+      whatPeopleSaying: material?.whatPeopleSaying ?? [],
       tree: this.buildTree(flat, ctx.seed),
       flat,
-      videos: [],
-      forums: [],
-      organicResults: [],
-      localPack: [],
-      entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
+      videos: material?.videos ?? [],
+      forums: material?.forums ?? [],
+      organicResults: material?.organicResults ?? [],
+      localPack: material?.localPack ?? [],
+      entityIds: material?.entityIds ?? { entities: [], kgIds: [], cids: [], gcids: [] },
       stats
     };
   }
@@ -2705,12 +3094,14 @@ var PAAExtractor = class {
     const seenKeys = /* @__PURE__ */ new Set();
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
-    this.collectedRows = [];
-    this.progressSequence = 0;
     this.clickedQuestions = /* @__PURE__ */ new Set();
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
-    const clickedOnceEver = /* @__PURE__ */ new Set();
+    const clickAttempts = /* @__PURE__ */ new Map();
+    const processedQuestions = /* @__PURE__ */ new Set();
+    const failedInteractions = /* @__PURE__ */ new Set();
     const capturedItems = /* @__PURE__ */ new Map();
+    const MAX_CLICK_ATTEMPTS = 2;
+    const EXHAUSTION_CONFIRMATIONS = 3;
     const itemArgs = {
       sel: PAASelectors.item,
       dataQ: PAASelectors.itemDataQ,
@@ -2721,15 +3112,15 @@ var PAAExtractor = class {
       clickTarget: PAASelectors.clickTarget
     };
     const readItemStates = () => page.evaluate(
-      ({ sel, dataQ, dataInitQ, questionEl, expandedClass }) => {
+      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel }) => {
         return Array.from(document.querySelectorAll(sel)).map((el) => ({
           q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
-          done: el.classList.contains(expandedClass)
+          done: el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true" || Boolean((el.querySelector(answerSel)?.textContent ?? "").trim())
         })).filter((s) => s.q);
       },
       itemArgs
     );
-    const expandOneItemSerially = async (q) => {
+    const expandOneItemSerially = async (q, beforeCount) => {
       const clickStartedMs = Date.now();
       const items = page.locator(itemArgs.sel);
       const itemIndex = await items.evaluateAll(
@@ -2744,49 +3135,118 @@ var PAAExtractor = class {
         await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
         return true;
       })().catch(() => false) : false;
-      if (!clicked) return "failed";
+      if (!clicked) return { dispatched: false, confirmed: false };
       const remainingHumanClickDelayMs = 700 - (Date.now() - clickStartedMs);
       if (remainingHumanClickDelayMs > 0) await page.waitForTimeout(remainingHumanClickDelayMs);
-      return "ok";
+      const confirmed = await page.waitForFunction(
+        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, target, min }) => {
+          const items2 = Array.from(document.querySelectorAll(sel));
+          const item = items2.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
+          if (!item) return items2.length > min;
+          return item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true" || Boolean((item.querySelector(answerSel)?.textContent ?? "").trim()) || items2.length > min;
+        },
+        { ...itemArgs, target: q, min: beforeCount },
+        { timeout: 2500, polling: 100 }
+      ).catch(() => null);
+      return { dispatched: true, confirmed: Boolean(confirmed) };
     };
     let round = 0;
-    let growthWaits = 0;
-    while (true) {
-      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
+    expansion: while (true) {
+      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) {
+        this.paaLifecycle = {
+          ...this.paaLifecycle,
+          discoveryStatus: "interrupted",
+          completionReason: "The execution deadline arrived before frontier exhaustion could be proven."
+        };
+        throw new PaaInteractionInterruptedError(this.paaLifecycle.completionReason);
+      }
       this.throwIfAborted(signal);
       const states = await readItemStates();
       let discovered = false;
-      for (const s of states) {
-        if (!seenQs.has(s.q)) {
-          seenQs.add(s.q);
-          orderedQs.push(s.q);
+      for (const state of states) {
+        if (!seenQs.has(state.q)) {
+          seenQs.add(state.q);
+          orderedQs.push(state.q);
           if (orderedQs.length <= options.maxQuestions) {
-            discovered = this.upsertCollectedRow(this.toFlatRow({ question: s.q }, 1, null, options.query)) || discovered;
+            discovered = this.upsertCollectedRow(this.toFlatRow({ questionId: questionIdFor(state.q), googleLinkId: null, googleEvidenceId: null, question: state.q, sources: [] }, 1, null, options.query)) || discovered;
           }
         }
+        if (state.done) processedQuestions.add(state.q);
       }
+      this.paaLifecycle = {
+        ...this.paaLifecycle,
+        questionsFound: Math.min(orderedQs.length, options.maxQuestions),
+        targetStatus: orderedQs.length >= options.maxQuestions ? "met" : "not_met",
+        knownQuestions: orderedQs.length,
+        processedQuestions: processedQuestions.size,
+        failedInteractions: failedInteractions.size
+      };
       if (discovered) await this.emitProgress("discovered");
-      if (options.questionsOnly && seenQs.size >= options.maxQuestions) break;
+      if (options.questionsOnly && seenQs.size >= options.maxQuestions) {
+        this.paaLifecycle = {
+          ...this.paaLifecycle,
+          discoveryStatus: "target_reached",
+          targetStatus: "met",
+          completionReason: "The requested unique-question count was observed."
+        };
+        break;
+      }
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
-      const clickable = states.filter((s) => kept.has(s.q) && !clickedOnceEver.has(s.q) && !abandonedNeverReclick.has(s.q));
-      const target = clickable.find((s) => !s.done) ?? (seenQs.size < options.maxQuestions ? clickable[0] : void 0);
+      const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
+      const target = clickable[0];
       if (!target) {
-        if (seenQs.size >= options.maxQuestions) break;
-        if (++growthWaits > 1) break;
-        const grew = await page.waitForFunction(
-          ({ sel, min }) => document.querySelectorAll(sel).length > min,
-          { sel: PAASelectors.item, min: states.length },
-          { timeout: 2e3, polling: 100 }
-        ).catch(() => null);
-        if (!grew) continue;
-        continue;
+        if (seenQs.size >= options.maxQuestions) {
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            discoveryStatus: "target_reached",
+            targetStatus: "met",
+            completionReason: "The requested unique-question count was observed."
+          };
+          break;
+        }
+        const visible = new Set(states.map((state) => state.q));
+        const unresolved2 = orderedQs.slice(0, options.maxQuestions).filter(
+          (question) => !processedQuestions.has(question) && (failedInteractions.has(question) || !visible.has(question))
+        );
+        if (unresolved2.length > 0) {
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            discoveryStatus: "interrupted",
+            failedInteractions: (/* @__PURE__ */ new Set([...failedInteractions, ...unresolved2])).size,
+            completionReason: "At least one known PAA control failed or disappeared before it could be confirmed."
+          };
+          await this.emitProgress("expansion_finished");
+          throw new PaaInteractionInterruptedError(this.paaLifecycle.completionReason);
+        }
+        for (let confirmation = 1; confirmation <= EXHAUSTION_CONFIRMATIONS; confirmation++) {
+          this.throwIfAborted(signal);
+          await this.throwIfCaptcha(page, "Google PAA exhaustion confirmation");
+          await page.mouse.wheel(0, 320);
+          const grew = await page.waitForFunction(
+            ({ sel, min }) => document.querySelectorAll(sel).length > min,
+            { sel: PAASelectors.item, min: states.length },
+            { timeout: 1500, polling: 100 }
+          ).catch(() => null);
+          this.paaLifecycle = { ...this.paaLifecycle, noGrowthConfirmations: confirmation };
+          if (grew) continue expansion;
+          const confirmationStates = await readItemStates();
+          if (confirmationStates.some((state) => !seenQs.has(state.q))) continue expansion;
+        }
+        this.paaLifecycle = {
+          ...this.paaLifecycle,
+          discoveryStatus: "frontier_exhausted",
+          targetStatus: "not_met",
+          failedInteractions: 0,
+          noGrowthConfirmations: EXHAUSTION_CONFIRMATIONS,
+          completionReason: "Every known eligible PAA control was processed and three healthy no-growth confirmations found no additional questions."
+        };
+        break;
       }
-      growthWaits = 0;
       this.reporter.onDepth(++round);
       await this.throwIfCaptcha(page, "Google PAA expansion");
-      clickedOnceEver.add(target.q);
-      this.clickedQuestions.add(target.q);
-      const expansionStatus = await expandOneItemSerially(target.q);
+      clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
+      const expansionResult = await expandOneItemSerially(target.q, states.length);
+      if (expansionResult.dispatched) this.clickedQuestions.add(target.q);
       if (!options.questionsOnly) {
         let clickedItem = (await this.extractVisibleItems(page, target.q))[0];
         if (!clickedItem?.answer) {
@@ -2800,27 +3260,21 @@ var PAAExtractor = class {
           }
         }
       }
-      if (expansionStatus === "failed" && !capturedItems.has(target.q)) {
+      const captured = capturedItems.get(target.q);
+      if (expansionResult.confirmed || Boolean(captured?.answer?.trim())) {
+        processedQuestions.add(target.q);
+        failedInteractions.delete(target.q);
+      } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
+        failedInteractions.add(target.q);
         abandonedNeverReclick.add(target.q);
       }
-    }
-    if (options.questionsOnly && seenQs.size < options.maxQuestions) {
-      for (let recoveryRound = 0; recoveryRound < options.questionGrowthRecoveryRounds && seenQs.size < options.maxQuestions; recoveryRound++) {
-        if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
-        const reExpanded = await this.reExpandCollapsedItems(page).catch(() => false);
-        if (!reExpanded) break;
-        await page.waitForTimeout(900);
-        for (const state of await readItemStates()) {
-          if (!seenQs.has(state.q)) {
-            seenQs.add(state.q);
-            orderedQs.push(state.q);
-            if (orderedQs.length <= options.maxQuestions) {
-              this.upsertCollectedRow(this.toFlatRow({ question: state.q }, 1, null, options.query));
-            }
-          }
-        }
-        await this.emitProgress("discovered");
-      }
+      this.paaLifecycle = {
+        ...this.paaLifecycle,
+        processedQuestions: processedQuestions.size,
+        failedInteractions: failedInteractions.size,
+        noGrowthConfirmations: 0
+      };
+      await this.emitProgress("click_observed");
     }
     const itemMap = options.questionsOnly ? /* @__PURE__ */ new Map() : new Map(capturedItems);
     if (!options.questionsOnly) {
@@ -2837,24 +3291,34 @@ var PAAExtractor = class {
         const row = this.toFlatRow(item, 1, null, options.query);
         this.upsertCollectedRow(row);
         results.push(row);
-        this.reporter.onQuestion({ question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, depth: 1, parentQuestion: null, children: [] });
+        this.reporter.onQuestion({ questionId: item.questionId, googleLinkId: item.googleLinkId, googleEvidenceId: item.googleEvidenceId, question: item.question, answer: item.answer ?? null, sourceTitle: item.sourceTitle ?? null, sourceSite: item.sourceSite ?? null, sourceCite: item.sourceCite ?? null, sources: item.sources, depth: 1, parentQuestion: null, children: [] });
       } else {
-        const row = this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query);
+        const row = this.toFlatRow({ questionId: questionIdFor(q), googleLinkId: null, googleEvidenceId: null, question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0, sources: [] }, 1, null, options.query);
         this.upsertCollectedRow(row);
         results.push(row);
       }
     }
     this.completeness.paaWithoutAnswer = options.questionsOnly ? 0 : results.filter((r) => !r.answer).length;
-    this.completeness.paaWithoutSource = options.questionsOnly ? 0 : results.filter((r) => !r.source_cite).length;
+    this.completeness.paaWithoutSource = options.questionsOnly ? 0 : results.filter((r) => (r.sources ?? []).length === 0).length;
     this.completeness.paaRequested = options.maxQuestions;
     this.completeness.paaReturned = results.length;
     this.completeness.paaUnique = seenKeys.size;
+    const materialPartial = !options.questionsOnly && (this.completeness.paaWithoutAnswer > 0 || this.completeness.paaWithoutSource > 0);
+    this.paaLifecycle = {
+      ...this.paaLifecycle,
+      questionsFound: results.length,
+      targetStatus: results.length >= options.maxQuestions ? "met" : "not_met",
+      materialStatus: materialPartial ? "partial" : "complete",
+      processedQuestions: processedQuestions.size,
+      failedInteractions: failedInteractions.size
+    };
     await this.emitProgress("fill_incomplete");
+    await this.emitProgress("expansion_finished");
     return results;
   }
   async fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick) {
     const kept = new Set(orderedQs.slice(0, options.maxQuestions));
-    const isIncomplete = (item) => !item?.answer || !(item.sourceTitle || item.sourceSite || item.sourceCite);
+    const isIncomplete = (item) => !item?.answer || (item.sources ?? []).length === 0;
     const missingBefore = new Set(
       [...kept].filter((q) => isIncomplete(itemMap.get(q)))
     );
@@ -2872,7 +3336,8 @@ var PAAExtractor = class {
         answer: item.answer ?? existing?.answer,
         sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
         sourceSite: item.sourceSite ?? existing?.sourceSite,
-        sourceCite: item.sourceCite ?? existing?.sourceCite
+        sourceCite: item.sourceCite ?? existing?.sourceCite,
+        sources: this.mergePaaSources(existing?.sources, item.sources)
       });
       const upgraded = itemMap.get(q);
       if (upgraded && this.upsertCollectedRow(this.toFlatRow(upgraded, 1, null, options.query))) {
@@ -3059,7 +3524,7 @@ var PAAExtractor = class {
   }
   async extractOrganicResults(page) {
     const sels = OrganicSelectors;
-    return page.evaluate((s) => {
+    const raw = await page.evaluate((s) => {
       const out = [];
       let pos = 0;
       document.querySelectorAll(s.result).forEach((card) => {
@@ -3085,6 +3550,17 @@ var PAAExtractor = class {
       });
       return out;
     }, sels);
+    const baseUrl = typeof page.url === "function" ? page.url() : "https://www.google.com/";
+    return raw.flatMap((item) => {
+      const link = classifyGoogleOutboundLink(item.url, baseUrl);
+      if (!link) return [];
+      let domain = item.domain;
+      try {
+        domain = new URL(link.resolvedUrl ?? link.rawUrl).hostname.replace(/^www\./, "");
+      } catch {
+      }
+      return [{ ...item, ...link, domain }];
+    });
   }
   async extractLocalPack(page) {
     const sels = LocalPackSelectors;
@@ -3236,15 +3712,21 @@ var PAAExtractor = class {
       aim: AIModeSelectors,
       expandWaitMs: 1500
     });
-    if (!surfaces.aiOverview.detected) {
+    const pageUrl = typeof page.url === "function" ? page.url() : "https://www.google.com/";
+    const normalized = {
+      surface: surfaces.surface,
+      aiOverview: { ...surfaces.aiOverview, citations: this.normalizeAICitations(pageUrl, surfaces.aiOverview.citations) },
+      aiMode: { ...surfaces.aiMode, citations: this.normalizeAICitations(pageUrl, surfaces.aiMode.citations) }
+    };
+    if (!normalized.aiOverview.detected) {
       this.completeness.aioShareCaptured = null;
-      return surfaces;
+      return normalized;
     }
     const shareUrl = this.aioShareUrlEarly ?? await this.captureAIOverviewShareUrl(page, options).catch(() => null);
     this.completeness.aioShareCaptured = shareUrl !== null;
     return {
-      ...surfaces,
-      aiOverview: { ...surfaces.aiOverview, shareUrl }
+      ...normalized,
+      aiOverview: { ...normalized.aiOverview, shareUrl }
     };
   }
   async captureAIOverviewShareUrl(page, options) {
@@ -3289,11 +3771,15 @@ var PAAExtractor = class {
     const nodeMap = /* @__PURE__ */ new Map();
     for (const row of flat) {
       const node = {
+        questionId: row.question_id,
+        googleLinkId: row.google_link_id || null,
+        googleEvidenceId: row.google_evidence_id || null,
         question: row.question,
         answer: row.answer || null,
         sourceTitle: row.source_title || null,
         sourceSite: row.source_site || null,
         sourceCite: row.source_cite || null,
+        sources: row.sources ?? [],
         depth: row.depth,
         parentQuestion: row.parent_question || null,
         children: []
@@ -3340,10 +3826,13 @@ var PAAExtractor = class {
   async extract(options, signal) {
     const startMs = Date.now();
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
+    this.linkDiagnostics = this.emptyLinkDiagnostics();
     this.aioShareUrlEarly = null;
     this.collectedRows = [];
     this.progressSequence = 0;
+    this.checkpointMaterial = null;
     this.clickedQuestions = /* @__PURE__ */ new Set();
+    this.paaLifecycle = this.emptyPaaLifecycle(options.maxQuestions);
     this.partialContext = {
       seed: options.query,
       location: options.location ?? null,
@@ -3406,8 +3895,17 @@ var PAAExtractor = class {
       this.throwIfAborted(signal);
       const page = this.driver.getPage();
       await this.throwIfCaptcha(page, "Google SERP");
+      if (!hasPaa) {
+        this.paaLifecycle = {
+          ...this.paaLifecycle,
+          discoveryStatus: "no_paa_observed",
+          materialStatus: "complete",
+          completionReason: "A readable Google SERP was observed without a People Also Ask block."
+        };
+      }
       if (options.questionsOnly) {
         const flat2 = hasPaa ? await this.runBFS(page, executionOptions, signal) : [];
+        await this.resolveMaterialLinks(page, flat2, [], emptyAiSurfaces.aiOverview, emptyAiSurfaces.aiMode);
         const stats2 = {
           seed: executionOptions.query,
           totalQuestions: flat2.length,
@@ -3424,7 +3922,9 @@ var PAAExtractor = class {
             completionStatus: hasPaa ? "paa_found" : "no_paa",
             ...!hasPaa ? { noPaaObserved: true } : {},
             problem: null,
+            paaLifecycle: { ...this.paaLifecycle },
             completeness: { ...this.completeness },
+            links: { ...this.linkDiagnostics },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, void 0, locationResolution) } : {}
           },
           totalQuestions: flat2.length,
@@ -3467,6 +3967,7 @@ var PAAExtractor = class {
             locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, allOrganic2, localPack2);
           }
         }
+        await this.resolveMaterialLinks(page, [], allOrganic2, aiSurfaces2.aiOverview, aiSurfaces2.aiMode);
         const stats2 = {
           seed: executionOptions.query,
           totalQuestions: 0,
@@ -3483,6 +3984,7 @@ var PAAExtractor = class {
             completionStatus: "serp_only",
             problem: null,
             completeness: { ...this.completeness },
+            links: { ...this.linkDiagnostics },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
           },
           totalQuestions: 0,
@@ -3528,6 +4030,7 @@ var PAAExtractor = class {
           }
         }
         const aiSurfaces2 = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
+        await this.resolveMaterialLinks(page, [], noPaaOrganic, aiSurfaces2.aiOverview, aiSurfaces2.aiMode);
         const stats2 = {
           seed: executionOptions.query,
           totalQuestions: 0,
@@ -3544,7 +4047,9 @@ var PAAExtractor = class {
             completionStatus: "no_paa",
             noPaaObserved: true,
             problem: null,
+            paaLifecycle: { ...this.paaLifecycle },
             completeness: { ...this.completeness },
+            links: { ...this.linkDiagnostics },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
           },
           totalQuestions: 0,
@@ -3562,9 +4067,21 @@ var PAAExtractor = class {
           stats: stats2
         };
       }
+      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
+      this.checkpointMaterial = {
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        whatPeopleSaying,
+        videos,
+        forums,
+        organicResults,
+        localPack,
+        entityIds
+      };
+      await this.emitProgress("serp_captured");
       const flat = await this.runBFS(page, executionOptions, signal);
       this.throwIfAborted(signal);
-      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
       const shortVidsParams = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
       if (uule) shortVidsParams.set("uule", uule);
       let shortVideos = [];
@@ -3597,6 +4114,15 @@ var PAAExtractor = class {
           locationEvidence = inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack);
         }
       }
+      await this.resolveMaterialLinks(page, flat, allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+      this.checkpointMaterial = {
+        ...this.checkpointMaterial,
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        videos: [...videos, ...shortVideos],
+        organicResults: allOrganic
+      };
       const allVideos = [...videos, ...shortVideos];
       const tree = this.buildTree(flat, executionOptions.query);
       const stats = {
@@ -3614,7 +4140,9 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          paaLifecycle: { ...this.paaLifecycle },
           completeness: { ...this.completeness },
+          links: { ...this.linkDiagnostics },
           ...diagnosticWarnings.length > 0 ? { warnings: diagnosticWarnings } : {},
           ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence, locationResolution) } : {}
         },
@@ -3634,6 +4162,13 @@ var PAAExtractor = class {
       };
     } catch (err) {
       errorCount++;
+      if (this.paaLifecycle.discoveryStatus === "in_progress") {
+        this.paaLifecycle = {
+          ...this.paaLifecycle,
+          discoveryStatus: "interrupted",
+          completionReason: "The browser attempt ended before frontier exhaustion could be proven."
+        };
+      }
       await this.emitProgress("attempt_ending").catch(() => {
       });
       this.reporter.onError(err instanceof Error ? err : new Error(String(err)));
@@ -3682,7 +4217,10 @@ var OutputSerializer = class {
     await import_node_fs.promises.mkdir(outputDir, { recursive: true });
     const seedRaw = rows[0]?.seed_query ?? "paa";
     const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
-    const csv = import_papaparse.default.unparse(rows, { header: true });
+    const csv = import_papaparse.default.unparse(rows.map(({ sources, ...row }) => ({
+      ...row,
+      sources_json: JSON.stringify(sources)
+    })), { header: true });
     const filename = `${slug}-${Date.now()}.csv`;
     const fullPath = import_node_path.default.join(outputDir, filename);
     await import_node_fs.promises.writeFile(fullPath, csv, "utf8");
@@ -3778,7 +4316,7 @@ var ProgressReporter = class {
 var import_sdk3 = __toESM(require("@onkernel/sdk"), 1);
 
 // src/backup-proxy.ts
-var import_node_crypto3 = require("crypto");
+var import_node_crypto4 = require("crypto");
 var import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
 var BACKUP_PROXY_HOST = "pr.oxylabs.io";
 var BACKUP_PROXY_PORT = 7777;
@@ -3800,7 +4338,7 @@ function backupProxyAvailable() {
 }
 function randomStickySessionId() {
   let digits = "";
-  for (let i = 0; i < 10; i++) digits += String((0, import_node_crypto3.randomInt)(0, 10));
+  for (let i = 0; i < 10; i++) digits += String((0, import_node_crypto4.randomInt)(0, 10));
   return digits;
 }
 function buildBackupUsername(opts = {}) {
@@ -4336,6 +4874,7 @@ function classifyAttemptError(err) {
   if (err instanceof CaptchaError) return "captcha";
   if (err instanceof LocationMismatchError) return "location_mismatch";
   if (err instanceof RequestAbortedError) return "request_aborted";
+  if (err instanceof PaaInteractionInterruptedError) return "browser_session_interrupted";
   if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return "timeout";
   const message = err instanceof Error ? err.message : String(err);
   if (/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message)) {
@@ -4360,7 +4899,9 @@ function withResultQuality(result, requestedPaaCount, questionsOnly) {
   const paaWithoutSource = questionsOnly ? 0 : result.flat.filter((row) => !row.source_cite?.trim()).length;
   const noPaa = result.diagnostics.completionStatus === "no_paa" && (result.diagnostics.noPaaObserved === true || hasUsableContent);
   const hasPaa = result.diagnostics.completionStatus === "paa_found";
-  const paaIncomplete = hasPaa && (uniquePaaCount < requestedPaaCount || paaWithoutAnswer > 0 || paaWithoutSource > 0 || Boolean(result.diagnostics.interruption));
+  const discoveryStatus = result.diagnostics.paaLifecycle?.discoveryStatus;
+  const discoveryIncomplete = discoveryStatus === "interrupted" || discoveryStatus === "recovery_exhausted";
+  const paaIncomplete = hasPaa && (uniquePaaCount < requestedPaaCount && discoveryStatus !== "frontier_exhausted" || paaWithoutAnswer > 0 || paaWithoutSource > 0 || Boolean(result.diagnostics.interruption) || discoveryIncomplete);
   const resultQuality = noPaa ? "complete" : hasPaa && uniquePaaCount > 0 ? paaIncomplete || hasWarnings ? "partial" : "complete" : hasUsableContent ? hasWarnings ? "partial" : "complete" : "degraded";
   const degradedResult = resultQuality === "degraded";
   result.diagnostics = {
@@ -4377,7 +4918,15 @@ function withResultQuality(result, requestedPaaCount, questionsOnly) {
       paaRequested: requestedPaaCount,
       paaReturned: result.flat.length,
       paaUnique: uniquePaaCount
-    }
+    },
+    ...result.diagnostics.paaLifecycle ? {
+      paaLifecycle: {
+        ...result.diagnostics.paaLifecycle,
+        questionsFound: uniquePaaCount,
+        targetStatus: uniquePaaCount >= requestedPaaCount ? "met" : "not_met",
+        materialStatus: paaWithoutAnswer > 0 || paaWithoutSource > 0 ? "partial" : "complete"
+      }
+    } : {}
   };
   return result;
 }