mcp-scraper 0.79.1 → 0.79.3

package/dist/index.cjs

@@ -2598,6 +2598,93 @@ async function resolveGoogleGotoUrls(rawUrls, options = {}) {
   await Promise.all(Array.from({ length: Math.min(concurrency, inputs.length) }, () => worker()));
   return { resolutions: [...records.values()], nonGoogleRequestsObserved: 0 };
 }
+function resolvedSite(link, fallback) {
+  try {
+    return new URL(link.resolvedUrl ?? link.url).hostname.replace(/^www\./, "");
+  } catch {
+    return fallback;
+  }
+}
+async function resolveHarvestResultGoogleGotoLinks(result, options = {}) {
+  const treeNodes = [];
+  const visit = (nodes) => {
+    for (const node of nodes) {
+      treeNodes.push(node);
+      visit(node.children ?? []);
+    }
+  };
+  visit(result.tree ?? []);
+  const candidates = [
+    ...result.flat.flatMap((row) => row.sources ?? []),
+    ...treeNodes.flatMap((node) => node.sources),
+    ...result.organicResults ?? [],
+    ...(result.aiOverview?.citations ?? []).map(({ text: _text, href, ...link }) => ({ ...link, url: href })),
+    ...(result.aiMode?.citations ?? []).map(({ text: _text, href, ...link }) => ({ ...link, url: href }))
+  ];
+  const unique = new Map(candidates.filter((link) => Boolean(link?.rawUrl)).map((link) => [link.rawUrl, link]));
+  const gotoUrls = [...unique.values()].filter((link) => link.linkType === "google_goto_redirect" && link.resolutionStatus !== "resolved").map((link) => link.rawUrl);
+  if (gotoUrls.length > 0) {
+    const batch = await resolveGoogleGotoUrls(gotoUrls, options);
+    for (const resolution2 of batch.resolutions) {
+      const link = unique.get(resolution2.rawUrl);
+      if (!link) continue;
+      unique.set(resolution2.rawUrl, resolution2.resolutionStatus === "resolved" && resolution2.resolvedUrl ? { ...link, url: resolution2.resolvedUrl, resolvedUrl: resolution2.resolvedUrl, resolutionStatus: "resolved" } : { ...link, resolutionStatus: resolution2.resolutionStatus });
+    }
+  }
+  const replace = (link) => ({
+    ...link,
+    ...unique.get(link.rawUrl) ?? link
+  });
+  const replaceSources = (sources) => sources.map((source) => {
+    const link = replace(source);
+    return { ...link, site: resolvedSite(link, source.site) };
+  });
+  for (const row of result.flat) {
+    row.sources = replaceSources(row.sources ?? []);
+    const primary = row.sources[0];
+    row.source_title = primary?.title ?? row.source_title;
+    row.source_site = primary?.site ?? row.source_site;
+    row.source_cite = primary?.url ?? row.source_cite;
+  }
+  for (const node of treeNodes) {
+    node.sources = replaceSources(node.sources ?? []);
+    const primary = node.sources[0];
+    node.sourceTitle = primary?.title ?? node.sourceTitle;
+    node.sourceSite = primary?.site ?? node.sourceSite;
+    node.sourceCite = primary?.url ?? node.sourceCite;
+  }
+  for (let index = 0; index < (result.organicResults?.length ?? 0); index++) {
+    const link = replace(result.organicResults[index]);
+    result.organicResults[index] = { ...link, domain: resolvedSite(link, result.organicResults[index].domain) ?? "" };
+  }
+  const replaceCitation = (citation) => {
+    const link = replace({
+      url: citation.href,
+      rawUrl: citation.rawUrl,
+      resolvedUrl: citation.resolvedUrl,
+      linkType: citation.linkType,
+      resolutionStatus: citation.resolutionStatus
+    });
+    const { url, ...metadata } = link;
+    return { ...citation, ...metadata, href: url };
+  };
+  if (result.aiOverview) result.aiOverview.citations = (result.aiOverview.citations ?? []).map(replaceCitation);
+  if (result.aiMode) result.aiMode.citations = (result.aiMode.citations ?? []).map(replaceCitation);
+  const values = [...unique.values()];
+  result.diagnostics.links = {
+    linkCandidatesObserved: unique.size,
+    plainLinksObserved: values.filter((link) => link.linkType === "plain").length,
+    googleUrlRedirectsObserved: values.filter((link) => link.linkType === "google_url_redirect").length,
+    googleGotoRedirectsObserved: values.filter((link) => link.linkType === "google_goto_redirect").length,
+    redirectsResolved: values.filter((link) => link.linkType !== "plain" && link.resolutionStatus === "resolved").length,
+    redirectsUnresolved: values.filter((link) => link.resolutionStatus === "unresolved").length,
+    redirectsRejected: values.filter((link) => link.resolutionStatus === "rejected").length,
+    resolverNonGoogleRequestsObserved: 0,
+    paaSourcesPreserved: result.flat.reduce((total, row) => total + row.sources.length, 0),
+    aiOverviewLabelValidated: result.aiOverview?.detected ?? false
+  };
+  return result;
+}
 async function resolveGoogleOutboundLinks(page, values, options = {}) {
   const byRawUrl = /* @__PURE__ */ new Map();
   for (const value of values) byRawUrl.set(value.rawUrl, value);
@@ -2718,15 +2805,24 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
 }
 
 // src/extractor/PAAExtractor.ts
+var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
+var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
+var PAA_CLICK_MID_CONFIRMATION_MS = 1e3;
+var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
+var PAA_CLICK_DISPATCH_TIMEOUT_MS = 900;
+var PAA_FRONTIER_BATCH_SIZE = 3;
+var MAX_RETAINED_INTERACTION_OBSERVATIONS = 250;
 var PAAExtractor = class {
-  constructor(driver, reporter, onPaaProgress) {
+  constructor(driver, reporter, onPaaProgress, onProviderSession) {
     this.driver = driver;
     this.reporter = reporter;
     this.onPaaProgress = onPaaProgress;
+    this.onProviderSession = onProviderSession;
   }
   driver;
   reporter;
   onPaaProgress;
+  onProviderSession;
   completeness = {
     paaWithoutAnswer: 0,
     paaWithoutSource: 0,
@@ -2741,6 +2837,7 @@ var PAAExtractor = class {
   clickedQuestions = /* @__PURE__ */ new Set();
   paaLifecycle = this.emptyPaaLifecycle(0);
   checkpointMaterial = null;
+  interactionObservations = [];
   emptyPaaLifecycle(requestedQuestions) {
     return {
       requestedQuestions,
@@ -2753,7 +2850,12 @@ var PAAExtractor = class {
       knownQuestions: 0,
       processedQuestions: 0,
       failedInteractions: 0,
-      noGrowthConfirmations: 0
+      noGrowthConfirmations: 0,
+      visibleControls: 0,
+      eligibleControls: 0,
+      dispatchedInteractions: 0,
+      confirmedInteractions: 0,
+      unconfirmedInteractions: 0
     };
   }
   emptyLinkDiagnostics() {
@@ -2862,7 +2964,7 @@ var PAAExtractor = class {
   }
   async extractVisibleItems(page, onlyQuestion) {
     const sels = PAASelectors;
-    const raw = await page.evaluate(({ selectors, onlyQuestion: onlyQuestion2 }) => {
+    const raw = await page.evaluate(({ selectors, onlyQuestion: onlyQuestion2, answerPlaceholderPattern }) => {
       function cleanText(el) {
         if (!el) return "";
         const parts = [];
@@ -2889,12 +2991,13 @@ var PAAExtractor = class {
         const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
         const answerElement = aiAnswerBody ?? answerContainer;
         const answerText = cleanText(answerElement);
+        const answerIsPlaceholder = new RegExp(answerPlaceholderPattern, "i").test(answerText);
         const citationCandidates = [
           ...Array.from((answerElement ?? pair).querySelectorAll("a[href]")),
           ...Array.from(pair.querySelectorAll(selectors.aiCitation)),
           ...Array.from(pair.querySelectorAll(selectors.aiSourceCard))
         ];
-        const hasLoadedAnswer = expanded || answerText.length > 30 && !/an error has occurred/i.test(answerText.slice(0, 80)) && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href));
+        const hasLoadedAnswer = !answerIsPlaceholder && (expanded && answerText.length >= 10 || answerText.length > 30 && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href)));
         let sourceTitle = hasLoadedAnswer ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
         let sourceSite = hasLoadedAnswer ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
         const sources = [];
@@ -2918,7 +3021,7 @@ var PAAExtractor = class {
           sources
         };
       });
-    }, { selectors: sels, onlyQuestion });
+    }, { selectors: sels, onlyQuestion, answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN });
     const baseUrl = typeof page.url === "function" ? page.url() : "https://www.google.com/";
     return raw.flatMap((item) => {
       const sources = [];
@@ -3014,7 +3117,11 @@ var PAAExtractor = class {
       phase,
       records: this.collectedRows.map((row) => ({ ...row })),
       lifecycle: { ...this.paaLifecycle },
-      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {}
+      ...this.checkpointMaterial ? { material: structuredClone(this.checkpointMaterial) } : {},
+      interactions: this.interactionObservations.map((observation) => ({
+        ...observation,
+        confirmationSignals: [...observation.confirmationSignals]
+      }))
     };
     try {
       await this.onPaaProgress(snapshot);
@@ -3095,6 +3202,7 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     this.clickedQuestions = /* @__PURE__ */ new Set();
+    this.interactionObservations = [];
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickAttempts = /* @__PURE__ */ new Map();
     const processedQuestions = /* @__PURE__ */ new Set();
@@ -3102,6 +3210,7 @@ var PAAExtractor = class {
     const capturedItems = /* @__PURE__ */ new Map();
     const MAX_CLICK_ATTEMPTS = 2;
     const EXHAUSTION_CONFIRMATIONS = 3;
+    let batchNumber = 0;
     const itemArgs = {
       sel: PAASelectors.item,
       dataQ: PAASelectors.itemDataQ,
@@ -3109,46 +3218,171 @@ var PAAExtractor = class {
       questionEl: PAASelectors.itemQuestionEl,
       answerSel: PAASelectors.answerContainer,
       expandedClass: PAASelectors.expandedClass,
-      clickTarget: PAASelectors.clickTarget
+      clickTarget: PAASelectors.clickTarget,
+      answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
+      questionsOnly: options.questionsOnly,
+      sourceSel: "a[href]"
     };
     const readItemStates = () => page.evaluate(
-      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel }) => {
-        return Array.from(document.querySelectorAll(sel)).map((el) => ({
-          q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
-          done: el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true" || Boolean((el.querySelector(answerSel)?.textContent ?? "").trim())
-        })).filter((s) => s.q);
+      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
+        const cleanVisibleMaterial = (element) => {
+          if (!element) return "";
+          const parts = [];
+          for (const node of element.childNodes) {
+            if (node.nodeType === Node.TEXT_NODE) {
+              const text2 = node.textContent?.trim();
+              if (text2) parts.push(text2);
+              continue;
+            }
+            if (node.nodeType !== Node.ELEMENT_NODE) continue;
+            const child = node;
+            if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
+            const text = cleanVisibleMaterial(child);
+            if (text) parts.push(text);
+          }
+          return parts.join(" ").replace(/\s+/g, " ").trim();
+        };
+        return Array.from(document.querySelectorAll(sel)).map((el, index) => {
+          const answerText = cleanVisibleMaterial(el.querySelector(answerSel)).replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
+          const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
+          const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
+          return {
+            q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
+            done: expanded && (questionsOnly || answerReady),
+            expanded,
+            index
+          };
+        }).filter((s) => s.q);
       },
       itemArgs
     );
-    const expandOneItemSerially = async (q, beforeCount) => {
+    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded, controlAttempt, batchNumber2) => {
       const clickStartedMs = Date.now();
-      const items = page.locator(itemArgs.sel);
-      const itemIndex = await items.evaluateAll(
-        (elements, { dataQ, dataInitQ, target }) => elements.findIndex((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ)) === target),
-        { dataQ: itemArgs.dataQ, dataInitQ: itemArgs.dataInitQ, target: q }
-      ).catch(() => -1);
-      const clickTarget = itemIndex >= 0 ? items.nth(itemIndex).locator(itemArgs.clickTarget) : null;
+      const baseObservation = {
+        id: `${batchNumber2}:${questionIdFor(q)}:${controlAttempt}`,
+        questionId: questionIdFor(q),
+        question: q,
+        batchNumber: batchNumber2,
+        controlAttempt,
+        observedAt: new Date(clickStartedMs).toISOString(),
+        beforeQuestionCount: beforeCount,
+        expandedBefore: wasExpanded
+      };
+      const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
+      let dispatchErrorCode = null;
       const clicked = clickTarget ? await (async () => {
-        await clickTarget.scrollIntoViewIfNeeded({ timeout: 5e3 });
-        const box = await clickTarget.boundingBox({ timeout: 5e3 });
-        if (!box) return false;
-        await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
+        if (wasExpanded) {
+          await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
+          await page.waitForTimeout(120);
+        }
+        await clickTarget.click({ timeout: PAA_CLICK_DISPATCH_TIMEOUT_MS, force: true });
         return true;
-      })().catch(() => false) : false;
-      if (!clicked) return { dispatched: false, confirmed: false };
-      const remainingHumanClickDelayMs = 700 - (Date.now() - clickStartedMs);
-      if (remainingHumanClickDelayMs > 0) await page.waitForTimeout(remainingHumanClickDelayMs);
-      const confirmed = await page.waitForFunction(
-        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, target, min }) => {
-          const items2 = Array.from(document.querySelectorAll(sel));
-          const item = items2.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
-          if (!item) return items2.length > min;
-          return item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true" || Boolean((item.querySelector(answerSel)?.textContent ?? "").trim()) || items2.length > min;
-        },
-        { ...itemArgs, target: q, min: beforeCount },
-        { timeout: 2500, polling: 100 }
-      ).catch(() => null);
-      return { dispatched: true, confirmed: Boolean(confirmed) };
+      })().catch((error) => {
+        dispatchErrorCode = error instanceof Error && (error.name === "TimeoutError" || /timeout/i.test(error.message)) ? "click_ack_timeout" : "click_failed";
+        return false;
+      }) : false;
+      const dispatchDurationMs = Date.now() - clickStartedMs;
+      if (!clicked && dispatchErrorCode !== "click_ack_timeout") {
+        const observation2 = {
+          ...baseObservation,
+          dispatched: false,
+          outcome: "dispatch_failed",
+          confirmationSignals: [],
+          confirmationCheckpointMs: null,
+          confirmedAtMs: null,
+          dispatchDurationMs,
+          totalDurationMs: dispatchDurationMs,
+          afterQuestionCount: beforeCount,
+          newQuestionCount: 0,
+          expandedAfter: wasExpanded,
+          answerObserved: false,
+          sourceCount: 0,
+          errorCode: itemIndex < 0 ? "control_missing" : dispatchErrorCode ?? "click_failed"
+        };
+        return { dispatched: false, confirmed: false, observation: observation2 };
+      }
+      let latest = {
+        exists: true,
+        expanded: wasExpanded,
+        answerReady: false,
+        sourceCount: 0,
+        questionCount: beforeCount
+      };
+      let signals = [];
+      let confirmationCheckpointMs = null;
+      let confirmedAtMs = null;
+      let confirmed = false;
+      for (const checkpointMs of [PAA_CLICK_EARLIEST_CONFIRMATION_MS, PAA_CLICK_MID_CONFIRMATION_MS, PAA_CLICK_ATTEMPT_BUDGET_MS]) {
+        const remainingMs = clickStartedMs + checkpointMs - Date.now();
+        if (remainingMs > 0) await page.waitForTimeout(remainingMs);
+        latest = await page.evaluate(
+          ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, sourceSel, target }) => {
+            const cleanVisibleMaterial = (element) => {
+              if (!element) return "";
+              const parts = [];
+              for (const node of element.childNodes) {
+                if (node.nodeType === Node.TEXT_NODE) {
+                  const text2 = node.textContent?.trim();
+                  if (text2) parts.push(text2);
+                  continue;
+                }
+                if (node.nodeType !== Node.ELEMENT_NODE) continue;
+                const child = node;
+                if (child.tagName === "STYLE" || child.tagName === "SCRIPT" || child.getAttribute("aria-hidden") === "true") continue;
+                const text = cleanVisibleMaterial(child);
+                if (text) parts.push(text);
+              }
+              return parts.join(" ").replace(/\s+/g, " ").trim();
+            };
+            const items = Array.from(document.querySelectorAll(sel));
+            const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
+            if (!item) return { exists: false, expanded: false, answerReady: false, sourceCount: 0, questionCount: items.length };
+            const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
+            const answerElement = item.querySelector(answerSel);
+            const answerText = cleanVisibleMaterial(answerElement);
+            const confirmationText = answerText.replace(/An AI Overview is not available for this search/gi, " ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi, " ").replace(/\bAI Overview\b/gi, " ").replace(/\s+/g, " ").trim();
+            const answerReady = confirmationText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(confirmationText);
+            return {
+              exists: true,
+              expanded,
+              answerReady,
+              sourceCount: answerReady ? answerElement?.querySelectorAll(sourceSel).length ?? 0 : 0,
+              questionCount: items.length
+            };
+          },
+          { ...itemArgs, target: q }
+        ).catch(() => latest);
+        signals = [
+          ...latest.expanded ? ["expanded"] : [],
+          ...latest.answerReady ? ["answer"] : [],
+          ...latest.sourceCount > 0 ? ["source"] : [],
+          ...latest.questionCount > beforeCount ? ["question_growth"] : []
+        ];
+        if (latest.answerReady || latest.sourceCount > 0 || latest.questionCount > beforeCount) {
+          confirmed = true;
+          confirmationCheckpointMs = checkpointMs;
+          confirmedAtMs = Date.now() - clickStartedMs;
+          break;
+        }
+      }
+      const totalDurationMs = Date.now() - clickStartedMs;
+      const observation = {
+        ...baseObservation,
+        dispatched: clicked,
+        outcome: confirmed ? "confirmed" : clicked ? "unconfirmed" : "dispatch_failed",
+        confirmationSignals: signals,
+        confirmationCheckpointMs,
+        confirmedAtMs,
+        dispatchDurationMs,
+        totalDurationMs,
+        afterQuestionCount: latest.questionCount,
+        newQuestionCount: Math.max(0, latest.questionCount - beforeCount),
+        expandedAfter: latest.expanded,
+        answerObserved: latest.answerReady,
+        sourceCount: latest.sourceCount,
+        errorCode: !clicked ? dispatchErrorCode : confirmed ? null : latest.exists ? "confirmation_timeout" : "control_missing"
+      };
+      return { dispatched: clicked, confirmed, observation };
     };
     let round = 0;
     expansion: while (true) {
@@ -3179,7 +3413,8 @@ var PAAExtractor = class {
         targetStatus: orderedQs.length >= options.maxQuestions ? "met" : "not_met",
         knownQuestions: orderedQs.length,
         processedQuestions: processedQuestions.size,
-        failedInteractions: failedInteractions.size
+        failedInteractions: failedInteractions.size,
+        visibleControls: states.length
       };
       if (discovered) await this.emitProgress("discovered");
       if (options.questionsOnly && seenQs.size >= options.maxQuestions) {
@@ -3191,10 +3426,44 @@ var PAAExtractor = class {
         };
         break;
       }
+      if (!options.questionsOnly && seenQs.size >= options.maxQuestions) {
+        let materialChanged = false;
+        const keptQuestions = new Set(orderedQs.slice(0, options.maxQuestions));
+        for (const item of await this.extractVisibleItems(page)) {
+          if (!keptQuestions.has(item.question)) continue;
+          const existing = capturedItems.get(item.question);
+          const merged = {
+            ...item,
+            answer: item.answer ?? existing?.answer,
+            sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
+            sourceSite: item.sourceSite ?? existing?.sourceSite,
+            sourceCite: item.sourceCite ?? existing?.sourceCite,
+            sources: this.mergePaaSources(existing?.sources, item.sources)
+          };
+          capturedItems.set(item.question, merged);
+          materialChanged = this.upsertCollectedRow(this.toFlatRow(merged, 1, null, options.query)) || materialChanged;
+        }
+        if (materialChanged) await this.emitProgress("answer_captured");
+        if ([...keptQuestions].every((question) => {
+          const item = capturedItems.get(question);
+          return Boolean(item?.answer?.trim() && item.sources.length > 0);
+        })) {
+          failedInteractions.clear();
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            discoveryStatus: "target_reached",
+            targetStatus: "met",
+            failedInteractions: 0,
+            completionReason: "The requested unique-question count and its answer/source material were observed."
+          };
+          break;
+        }
+      }
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
-      const target = clickable[0];
-      if (!target) {
+      const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
+      this.paaLifecycle = { ...this.paaLifecycle, eligibleControls: clickable.length };
+      if (targets.length === 0) {
         if (seenQs.size >= options.maxQuestions) {
           this.paaLifecycle = {
             ...this.paaLifecycle,
@@ -3242,37 +3511,64 @@ var PAAExtractor = class {
         };
         break;
       }
-      this.reporter.onDepth(++round);
       await this.throwIfCaptcha(page, "Google PAA expansion");
-      clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
-      const expansionResult = await expandOneItemSerially(target.q, states.length);
-      if (expansionResult.dispatched) this.clickedQuestions.add(target.q);
-      if (!options.questionsOnly) {
-        let clickedItem = (await this.extractVisibleItems(page, target.q))[0];
-        if (!clickedItem?.answer) {
-          await page.waitForTimeout(1300);
-          clickedItem = (await this.extractVisibleItems(page, target.q))[0];
+      batchNumber += 1;
+      for (const target of targets) {
+        this.reporter.onDepth(++round);
+        if ((clickAttempts.get(target.q) ?? 0) > 0) {
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            automaticRetries: this.paaLifecycle.automaticRetries + 1
+          };
         }
-        if (clickedItem) {
-          capturedItems.set(target.q, clickedItem);
+        clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
+      }
+      const expansionResults = await Promise.all(targets.map(async (target) => ({
+        target,
+        result: await expandOneItemSerially(
+          target.q,
+          states.length,
+          target.index,
+          target.expanded,
+          clickAttempts.get(target.q) ?? 1,
+          batchNumber
+        )
+      })));
+      for (const { target, result } of expansionResults) {
+        if (result.dispatched || result.confirmed) this.clickedQuestions.add(target.q);
+        this.interactionObservations.push(result.observation);
+      }
+      if (this.interactionObservations.length > MAX_RETAINED_INTERACTION_OBSERVATIONS) {
+        this.interactionObservations.splice(0, this.interactionObservations.length - MAX_RETAINED_INTERACTION_OBSERVATIONS);
+      }
+      if (!options.questionsOnly) {
+        const targetQuestions = new Set(targets.map((target) => target.q));
+        for (const clickedItem of await this.extractVisibleItems(page)) {
+          if (!targetQuestions.has(clickedItem.question)) continue;
+          capturedItems.set(clickedItem.question, clickedItem);
           if (this.upsertCollectedRow(this.toFlatRow(clickedItem, 1, null, options.query))) {
             await this.emitProgress("answer_captured");
           }
         }
       }
-      const captured = capturedItems.get(target.q);
-      if (expansionResult.confirmed || Boolean(captured?.answer?.trim())) {
-        processedQuestions.add(target.q);
-        failedInteractions.delete(target.q);
-      } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
-        failedInteractions.add(target.q);
-        abandonedNeverReclick.add(target.q);
+      for (const { target, result } of expansionResults) {
+        const captured = capturedItems.get(target.q);
+        if (result.confirmed || Boolean(captured?.answer?.trim())) {
+          processedQuestions.add(target.q);
+          failedInteractions.delete(target.q);
+        } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
+          failedInteractions.add(target.q);
+          abandonedNeverReclick.add(target.q);
+        }
       }
       this.paaLifecycle = {
         ...this.paaLifecycle,
         processedQuestions: processedQuestions.size,
         failedInteractions: failedInteractions.size,
-        noGrowthConfirmations: 0
+        noGrowthConfirmations: 0,
+        dispatchedInteractions: this.interactionObservations.filter((value) => value.dispatched).length,
+        confirmedInteractions: this.interactionObservations.filter((value) => value.outcome === "confirmed").length,
+        unconfirmedInteractions: this.interactionObservations.filter((value) => value.outcome !== "confirmed").length
       };
       await this.emitProgress("click_observed");
     }
@@ -3866,6 +4162,18 @@ var PAAExtractor = class {
     try {
       this.throwIfAborted(signal);
       await this.driver.launch(config);
+      const providerSession = this.driver.getProviderSessionMetadata?.();
+      if (providerSession?.providerSessionId && this.onProviderSession) {
+        try {
+          await this.onProviderSession(providerSession);
+        } catch (err) {
+          console.warn(JSON.stringify({
+            event: "paa_provider_session_sink_failed",
+            provider: providerSession.provider,
+            message: err instanceof Error ? err.message : String(err)
+          }));
+        }
+      }
       this.throwIfAborted(signal);
       const locationResolution = options.location ? resolveLocation(options.location) : null;
       const canonicalLocation = options.location ? locationResolution?.canonical ?? options.location : null;
@@ -4970,6 +5278,17 @@ function stripInternalDebug(result, keepDebug) {
   delete diagnostics.debug;
   return { ...result, diagnostics };
 }
+async function resolveSurvivingGotoLinks(result) {
+  try {
+    return await resolveHarvestResultGoogleGotoLinks(result, { totalBudgetMs: 1e4 });
+  } catch (error) {
+    console.warn(JSON.stringify({
+      event: "harvest_goto_resolution_failed",
+      message: errorMessage(error)
+    }));
+    return result;
+  }
+}
 async function cleanupDisposableProxy(kernelApiKey, proxyId) {
   if (!kernelApiKey || !proxyId) return;
   try {
@@ -4986,10 +5305,10 @@ async function cleanupDisposableProxy(kernelApiKey, proxyId) {
     }));
   }
 }
-async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress) {
+async function extractOnce(options, signal, forceManagedSerp = false, onPaaProgress, onProviderSession) {
   const driver = createSerpDriver(options, { forceManaged: forceManagedSerp });
   const reporter = new ProgressReporter();
-  const extractor = new PAAExtractor(driver, reporter, onPaaProgress);
+  const extractor = new PAAExtractor(driver, reporter, onPaaProgress, onProviderSession);
   if (signal?.aborted) {
     return {
       result: null,
@@ -5134,12 +5453,20 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
+      const onProviderSession = (metadata) => emitAttemptEvent(onAttemptEvent, {
+        type: "provider_session_started",
+        attemptNumber,
+        maxAttempts,
+        provider: metadata.provider,
+        providerSessionId: metadata.providerSessionId,
+        observedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
       const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
       const attempt = baseCtx ? await runWithCostContext({
         ...baseCtx,
         forceHeadless: allowHeadlessTest,
         forceHeadful: !allowHeadlessTest
-      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress);
+      }, () => extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession)) : await extractOnce(attemptOptions, signal, forceManagedSerp, onPaaProgress, onProviderSession);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);
@@ -5184,7 +5511,8 @@ async function harvest(rawOptions) {
         lastError = err;
         if (willRetry2) continue;
         if (attempt.result) {
-          return withResultQuality(stripInternalDebug(attempt.result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+          const resolvedResult2 = await resolveSurvivingGotoLinks(attempt.result);
+          return withResultQuality(stripInternalDebug(resolvedResult2, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
         }
         break;
       }
@@ -5219,7 +5547,8 @@ async function harvest(rawOptions) {
         if (willRetry2) continue;
         break;
       }
-      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+      const resolvedResult = await resolveSurvivingGotoLinks(result);
+      const finalResult = withResultQuality(stripInternalDebug(resolvedResult, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
       const outcome = classifyAttemptResult(finalResult);
       const willRetry = outcome === "degraded_result" && i < maxAttempts - 1;
       const resultError = outcome === "degraded_result" ? degradedResultMessage(finalResult) : null;