mcp-scraper 0.79.1 → 0.79.2

package/CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 
 ## [Unreleased]
 
+### Fixed
+
+- Routed canonical `/assistant` page loads to the web app and the redacted private Assistant readiness endpoint to the main API function, preventing production 404s after the 0.79.1 launch.
+
+## [0.79.2] - 2026-08-28
+
+### Fixed
+
+- Rejected Google loading and error placeholders as PAA answers, retried interrupted controls without misreporting frontier exhaustion, preserved complete answer/source material from retained DOM, and resolved surviving Google `/goto` evidence after a late browser disconnect without visiting publisher pages.
+- Advanced requested question counts through bounded newest-frontier click batches with 0.7–1.4 second observation windows, reducing a verified 35-question complete harvest from 347 seconds to 83 seconds while retaining actual multi-question click growth.
+
 ## [0.79.1] - 2026-08-28
 
 ### Fixed
@@ -1478,7 +1489,16 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 - Write actions remain unavailable until the account owner explicitly enables them.
 - Provider-specific connection data is normalized into one agent-facing contract.
 
-[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.5...HEAD
+[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.2...HEAD
+[0.79.2]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.1...v0.79.2
+[0.79.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.0...v0.79.1
+[0.79.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.78.0...v0.79.0
+[0.78.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.77.0...v0.78.0
+[0.77.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.76.0...v0.77.0
+[0.76.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.1...v0.76.0
+[0.75.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.75.0...v0.75.1
+[0.75.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.74.0...v0.75.0
+[0.74.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.5...v0.74.0
 [0.73.5]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.4...v0.73.5
 [0.73.4]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.3...v0.73.4
 [0.73.3]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.73.2...v0.73.3

package/README.md

@@ -173,7 +173,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.79.1`, SHA-256 `25c661ca24ac033667250ddb6866e1009a5ffa55789e4791475e20e2cd4103da`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.79.2`, SHA-256 `5d9ac8818c9dc6e8667b5734bd4253d3012df82a6976e45eeee22f37bb2026a1`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.js

@@ -17,8 +17,8 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-MNSYIX42.js"),
-    import("../worker-VU6JCCLG.js"),
+    import("../server-5OSHEEVT.js"),
+    import("../worker-2TYCYFPM.js"),
     import("../db-TA567JLA.js")
   ]);
   const PORT = parseInt(process.env.PORT ?? "3001");

package/dist/bin/mcp-scraper-cli.js

@@ -13,7 +13,7 @@ import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-OIGM6VKQ.js";
+} from "../chunk-2RTELL6M.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-core.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import {
   runMcpScraperStdio
-} from "../chunk-QVW5S6BD.js";
-import "../chunk-NH3EUCZL.js";
+} from "../chunk-XUM2LEGU.js";
+import "../chunk-JECPQBLZ.js";
 import "../chunk-PGJQDMC2.js";
 import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
@@ -10,10 +10,10 @@ import "../chunk-B6L73AO4.js";
 import "../chunk-VFCUGTCF.js";
 import "../chunk-OM7HVEJ3.js";
 import "../chunk-NGLZ6UJR.js";
-import "../chunk-R4YRH5PG.js";
+import "../chunk-6FL7ZZCF.js";
 import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
-import "../chunk-OIGM6VKQ.js";
+import "../chunk-2RTELL6M.js";
 import "../chunk-OU4FV2RP.js";
 
 // src/mcp/thorbit-restricted-tool-surface.ts

package/dist/bin/mcp-scraper-install.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import {
   renderInstallTerminal
-} from "../chunk-R4YRH5PG.js";
+} from "../chunk-6FL7ZZCF.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-OIGM6VKQ.js";
+} from "../chunk-2RTELL6M.js";
 
 // bin/mcp-scraper-install.ts
 var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;

package/dist/bin/mcp-stdio-server.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import {
   runMcpScraperStdio
-} from "../chunk-QVW5S6BD.js";
-import "../chunk-NH3EUCZL.js";
+} from "../chunk-XUM2LEGU.js";
+import "../chunk-JECPQBLZ.js";
 import "../chunk-PGJQDMC2.js";
 import "../chunk-DNM65UCK.js";
 import "../chunk-T3MZISOF.js";
@@ -10,10 +10,10 @@ import "../chunk-B6L73AO4.js";
 import "../chunk-VFCUGTCF.js";
 import "../chunk-OM7HVEJ3.js";
 import "../chunk-NGLZ6UJR.js";
-import "../chunk-R4YRH5PG.js";
+import "../chunk-6FL7ZZCF.js";
 import "../chunk-VXLU74YZ.js";
 import "../chunk-GGZEC22A.js";
-import "../chunk-OIGM6VKQ.js";
+import "../chunk-2RTELL6M.js";
 import "../chunk-OU4FV2RP.js";
 
 // bin/mcp-stdio-server.ts

package/dist/bin/paa-harvest.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-UFJJGXVO.js";
+} from "../chunk-WEEAXPWK.js";
 import {
   browserServiceApiKey
 } from "../chunk-OM7HVEJ3.js";

package/dist/{chunk-OIGM6VKQ.js → chunk-2RTELL6M.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.79.1";
+var PACKAGE_VERSION = "0.79.2";
 
 export {
   PACKAGE_VERSION

package/dist/{chunk-R4YRH5PG.js → chunk-6FL7ZZCF.js}

@@ -1,6 +1,6 @@
 // release-message.json
 var release_message_default = {
-  message: "PAA harvests now distinguish a proven end of the question frontier from an interrupted browser, automatically resume the same durable job, and preserve captured questions, answers, AI Overview citations, and ranked SERP links."
+  message: "PAA harvests now reject Google error placeholders, retry interrupted controls without claiming the frontier ended, advance the newest question frontier in bounded batches, preserve complete answers and sources, and resolve retained Google redirect evidence even after the browser session closes."
 };
 
 // src/install-terminal.ts

package/dist/{chunk-NH3EUCZL.js → chunk-JECPQBLZ.js}

@@ -45,7 +45,7 @@ import {
 } from "./chunk-VXLU74YZ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-OIGM6VKQ.js";
+} from "./chunk-2RTELL6M.js";
 import {
   PUBLIC_ERROR_CODES,
   buildPublicErrorEnvelope,

package/dist/{chunk-UFJJGXVO.js → chunk-WEEAXPWK.js}

@@ -2054,6 +2054,93 @@ async function resolveGoogleGotoUrls(rawUrls, options = {}) {
   await Promise.all(Array.from({ length: Math.min(concurrency, inputs.length) }, () => worker()));
   return { resolutions: [...records.values()], nonGoogleRequestsObserved: 0 };
 }
+function resolvedSite(link, fallback) {
+  try {
+    return new URL(link.resolvedUrl ?? link.url).hostname.replace(/^www\./, "");
+  } catch {
+    return fallback;
+  }
+}
+async function resolveHarvestResultGoogleGotoLinks(result, options = {}) {
+  const treeNodes = [];
+  const visit = (nodes) => {
+    for (const node of nodes) {
+      treeNodes.push(node);
+      visit(node.children ?? []);
+    }
+  };
+  visit(result.tree ?? []);
+  const candidates = [
+    ...result.flat.flatMap((row) => row.sources ?? []),
+    ...treeNodes.flatMap((node) => node.sources),
+    ...result.organicResults ?? [],
+    ...(result.aiOverview?.citations ?? []).map(({ text: _text, href, ...link }) => ({ ...link, url: href })),
+    ...(result.aiMode?.citations ?? []).map(({ text: _text, href, ...link }) => ({ ...link, url: href }))
+  ];
+  const unique = new Map(candidates.filter((link) => Boolean(link?.rawUrl)).map((link) => [link.rawUrl, link]));
+  const gotoUrls = [...unique.values()].filter((link) => link.linkType === "google_goto_redirect" && link.resolutionStatus !== "resolved").map((link) => link.rawUrl);
+  if (gotoUrls.length > 0) {
+    const batch = await resolveGoogleGotoUrls(gotoUrls, options);
+    for (const resolution2 of batch.resolutions) {
+      const link = unique.get(resolution2.rawUrl);
+      if (!link) continue;
+      unique.set(resolution2.rawUrl, resolution2.resolutionStatus === "resolved" && resolution2.resolvedUrl ? { ...link, url: resolution2.resolvedUrl, resolvedUrl: resolution2.resolvedUrl, resolutionStatus: "resolved" } : { ...link, resolutionStatus: resolution2.resolutionStatus });
+    }
+  }
+  const replace = (link) => ({
+    ...link,
+    ...unique.get(link.rawUrl) ?? link
+  });
+  const replaceSources = (sources) => sources.map((source) => {
+    const link = replace(source);
+    return { ...link, site: resolvedSite(link, source.site) };
+  });
+  for (const row of result.flat) {
+    row.sources = replaceSources(row.sources ?? []);
+    const primary = row.sources[0];
+    row.source_title = primary?.title ?? row.source_title;
+    row.source_site = primary?.site ?? row.source_site;
+    row.source_cite = primary?.url ?? row.source_cite;
+  }
+  for (const node of treeNodes) {
+    node.sources = replaceSources(node.sources ?? []);
+    const primary = node.sources[0];
+    node.sourceTitle = primary?.title ?? node.sourceTitle;
+    node.sourceSite = primary?.site ?? node.sourceSite;
+    node.sourceCite = primary?.url ?? node.sourceCite;
+  }
+  for (let index = 0; index < (result.organicResults?.length ?? 0); index++) {
+    const link = replace(result.organicResults[index]);
+    result.organicResults[index] = { ...link, domain: resolvedSite(link, result.organicResults[index].domain) ?? "" };
+  }
+  const replaceCitation = (citation) => {
+    const link = replace({
+      url: citation.href,
+      rawUrl: citation.rawUrl,
+      resolvedUrl: citation.resolvedUrl,
+      linkType: citation.linkType,
+      resolutionStatus: citation.resolutionStatus
+    });
+    const { url, ...metadata } = link;
+    return { ...citation, ...metadata, href: url };
+  };
+  if (result.aiOverview) result.aiOverview.citations = (result.aiOverview.citations ?? []).map(replaceCitation);
+  if (result.aiMode) result.aiMode.citations = (result.aiMode.citations ?? []).map(replaceCitation);
+  const values = [...unique.values()];
+  result.diagnostics.links = {
+    linkCandidatesObserved: unique.size,
+    plainLinksObserved: values.filter((link) => link.linkType === "plain").length,
+    googleUrlRedirectsObserved: values.filter((link) => link.linkType === "google_url_redirect").length,
+    googleGotoRedirectsObserved: values.filter((link) => link.linkType === "google_goto_redirect").length,
+    redirectsResolved: values.filter((link) => link.linkType !== "plain" && link.resolutionStatus === "resolved").length,
+    redirectsUnresolved: values.filter((link) => link.resolutionStatus === "unresolved").length,
+    redirectsRejected: values.filter((link) => link.resolutionStatus === "rejected").length,
+    resolverNonGoogleRequestsObserved: 0,
+    paaSourcesPreserved: result.flat.reduce((total, row) => total + row.sources.length, 0),
+    aiOverviewLabelValidated: result.aiOverview?.detected ?? false
+  };
+  return result;
+}
 async function resolveGoogleOutboundLinks(page, values, options = {}) {
   const byRawUrl = /* @__PURE__ */ new Map();
   for (const value of values) byRawUrl.set(value.rawUrl, value);
@@ -2174,6 +2261,10 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
 }
 
 // src/extractor/PAAExtractor.ts
+var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
+var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
+var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
+var PAA_FRONTIER_BATCH_SIZE = 3;
 var PAAExtractor = class {
   constructor(driver, reporter, onPaaProgress) {
     this.driver = driver;
@@ -2318,7 +2409,7 @@ var PAAExtractor = class {
   }
   async extractVisibleItems(page, onlyQuestion) {
     const sels = PAASelectors;
-    const raw = await page.evaluate(({ selectors, onlyQuestion: onlyQuestion2 }) => {
+    const raw = await page.evaluate(({ selectors, onlyQuestion: onlyQuestion2, answerPlaceholderPattern }) => {
       function cleanText(el) {
         if (!el) return "";
         const parts = [];
@@ -2345,12 +2436,13 @@ var PAAExtractor = class {
         const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
         const answerElement = aiAnswerBody ?? answerContainer;
         const answerText = cleanText(answerElement);
+        const answerIsPlaceholder = new RegExp(answerPlaceholderPattern, "i").test(answerText);
         const citationCandidates = [
           ...Array.from((answerElement ?? pair).querySelectorAll("a[href]")),
           ...Array.from(pair.querySelectorAll(selectors.aiCitation)),
           ...Array.from(pair.querySelectorAll(selectors.aiSourceCard))
         ];
-        const hasLoadedAnswer = expanded || answerText.length > 30 && !/an error has occurred/i.test(answerText.slice(0, 80)) && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href));
+        const hasLoadedAnswer = !answerIsPlaceholder && (expanded && answerText.length >= 10 || answerText.length > 30 && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href)));
         let sourceTitle = hasLoadedAnswer ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
         let sourceSite = hasLoadedAnswer ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
         const sources = [];
@@ -2374,7 +2466,7 @@ var PAAExtractor = class {
           sources
         };
       });
-    }, { selectors: sels, onlyQuestion });
+    }, { selectors: sels, onlyQuestion, answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN });
     const baseUrl = typeof page.url === "function" ? page.url() : "https://www.google.com/";
     return raw.flatMap((item) => {
       const sources = [];
@@ -2565,44 +2657,52 @@ var PAAExtractor = class {
       questionEl: PAASelectors.itemQuestionEl,
       answerSel: PAASelectors.answerContainer,
       expandedClass: PAASelectors.expandedClass,
-      clickTarget: PAASelectors.clickTarget
+      clickTarget: PAASelectors.clickTarget,
+      answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
+      questionsOnly: options.questionsOnly
     };
     const readItemStates = () => page.evaluate(
-      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel }) => {
-        return Array.from(document.querySelectorAll(sel)).map((el) => ({
-          q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
-          done: el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true" || Boolean((el.querySelector(answerSel)?.textContent ?? "").trim())
-        })).filter((s) => s.q);
+      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
+        return Array.from(document.querySelectorAll(sel)).map((el, index) => {
+          const answerText = (el.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
+          const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
+          const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
+          return {
+            q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
+            done: expanded && (questionsOnly || answerReady),
+            expanded,
+            index
+          };
+        }).filter((s) => s.q);
       },
       itemArgs
     );
-    const expandOneItemSerially = async (q, beforeCount) => {
+    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded) => {
       const clickStartedMs = Date.now();
-      const items = page.locator(itemArgs.sel);
-      const itemIndex = await items.evaluateAll(
-        (elements, { dataQ, dataInitQ, target }) => elements.findIndex((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ)) === target),
-        { dataQ: itemArgs.dataQ, dataInitQ: itemArgs.dataInitQ, target: q }
-      ).catch(() => -1);
-      const clickTarget = itemIndex >= 0 ? items.nth(itemIndex).locator(itemArgs.clickTarget) : null;
+      const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
       const clicked = clickTarget ? await (async () => {
-        await clickTarget.scrollIntoViewIfNeeded({ timeout: 5e3 });
-        const box = await clickTarget.boundingBox({ timeout: 5e3 });
-        if (!box) return false;
-        await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
+        if (wasExpanded) {
+          await clickTarget.click({ timeout: 2e3, force: true });
+          await page.waitForTimeout(250);
+        }
+        await clickTarget.click({ timeout: 2e3, force: true });
         return true;
       })().catch(() => false) : false;
       if (!clicked) return { dispatched: false, confirmed: false };
-      const remainingHumanClickDelayMs = 700 - (Date.now() - clickStartedMs);
-      if (remainingHumanClickDelayMs > 0) await page.waitForTimeout(remainingHumanClickDelayMs);
+      const remainingConfirmationMs = Math.max(100, PAA_CLICK_ATTEMPT_BUDGET_MS - (Date.now() - clickStartedMs));
       const confirmed = await page.waitForFunction(
-        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, target, min }) => {
-          const items2 = Array.from(document.querySelectorAll(sel));
-          const item = items2.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
-          if (!item) return items2.length > min;
-          return item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true" || Boolean((item.querySelector(answerSel)?.textContent ?? "").trim()) || items2.length > min;
+        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, questionsOnly, target, min, earliestConfirmationAt }) => {
+          if (Date.now() < earliestConfirmationAt) return false;
+          const items = Array.from(document.querySelectorAll(sel));
+          const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
+          if (!item) return false;
+          const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
+          if (questionsOnly) return expanded || items.length > min;
+          const answerText = (item.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
+          return answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
         },
-        { ...itemArgs, target: q, min: beforeCount },
-        { timeout: 2500, polling: 100 }
+        { ...itemArgs, target: q, min: beforeCount, earliestConfirmationAt: clickStartedMs + PAA_CLICK_EARLIEST_CONFIRMATION_MS },
+        { timeout: remainingConfirmationMs, polling: 100 }
       ).catch(() => null);
       return { dispatched: true, confirmed: Boolean(confirmed) };
     };
@@ -2647,10 +2747,43 @@ var PAAExtractor = class {
         };
         break;
       }
+      if (!options.questionsOnly && seenQs.size >= options.maxQuestions) {
+        let materialChanged = false;
+        const keptQuestions = new Set(orderedQs.slice(0, options.maxQuestions));
+        for (const item of await this.extractVisibleItems(page)) {
+          if (!keptQuestions.has(item.question)) continue;
+          const existing = capturedItems.get(item.question);
+          const merged = {
+            ...item,
+            answer: item.answer ?? existing?.answer,
+            sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
+            sourceSite: item.sourceSite ?? existing?.sourceSite,
+            sourceCite: item.sourceCite ?? existing?.sourceCite,
+            sources: this.mergePaaSources(existing?.sources, item.sources)
+          };
+          capturedItems.set(item.question, merged);
+          materialChanged = this.upsertCollectedRow(this.toFlatRow(merged, 1, null, options.query)) || materialChanged;
+        }
+        if (materialChanged) await this.emitProgress("answer_captured");
+        if ([...keptQuestions].every((question) => {
+          const item = capturedItems.get(question);
+          return Boolean(item?.answer?.trim() && item.sources.length > 0);
+        })) {
+          failedInteractions.clear();
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            discoveryStatus: "target_reached",
+            targetStatus: "met",
+            failedInteractions: 0,
+            completionReason: "The requested unique-question count and its answer/source material were observed."
+          };
+          break;
+        }
+      }
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
-      const target = clickable[0];
-      if (!target) {
+      const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
+      if (targets.length === 0) {
         if (seenQs.size >= options.maxQuestions) {
           this.paaLifecycle = {
             ...this.paaLifecycle,
@@ -2698,31 +2831,43 @@ var PAAExtractor = class {
         };
         break;
       }
-      this.reporter.onDepth(++round);
       await this.throwIfCaptcha(page, "Google PAA expansion");
-      clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
-      const expansionResult = await expandOneItemSerially(target.q, states.length);
-      if (expansionResult.dispatched) this.clickedQuestions.add(target.q);
-      if (!options.questionsOnly) {
-        let clickedItem = (await this.extractVisibleItems(page, target.q))[0];
-        if (!clickedItem?.answer) {
-          await page.waitForTimeout(1300);
-          clickedItem = (await this.extractVisibleItems(page, target.q))[0];
+      for (const target of targets) {
+        this.reporter.onDepth(++round);
+        if ((clickAttempts.get(target.q) ?? 0) > 0) {
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            automaticRetries: this.paaLifecycle.automaticRetries + 1
+          };
         }
-        if (clickedItem) {
-          capturedItems.set(target.q, clickedItem);
+        clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
+      }
+      const expansionResults = await Promise.all(targets.map(async (target) => ({
+        target,
+        result: await expandOneItemSerially(target.q, states.length, target.index, target.expanded)
+      })));
+      for (const { target, result } of expansionResults) {
+        if (result.dispatched) this.clickedQuestions.add(target.q);
+      }
+      if (!options.questionsOnly) {
+        const targetQuestions = new Set(targets.map((target) => target.q));
+        for (const clickedItem of await this.extractVisibleItems(page)) {
+          if (!targetQuestions.has(clickedItem.question)) continue;
+          capturedItems.set(clickedItem.question, clickedItem);
           if (this.upsertCollectedRow(this.toFlatRow(clickedItem, 1, null, options.query))) {
             await this.emitProgress("answer_captured");
           }
         }
       }
-      const captured = capturedItems.get(target.q);
-      if (expansionResult.confirmed || Boolean(captured?.answer?.trim())) {
-        processedQuestions.add(target.q);
-        failedInteractions.delete(target.q);
-      } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
-        failedInteractions.add(target.q);
-        abandonedNeverReclick.add(target.q);
+      for (const { target, result } of expansionResults) {
+        const captured = capturedItems.get(target.q);
+        if (result.confirmed || Boolean(captured?.answer?.trim())) {
+          processedQuestions.add(target.q);
+          failedInteractions.delete(target.q);
+        } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
+          failedInteractions.add(target.q);
+          abandonedNeverReclick.add(target.q);
+        }
       }
       this.paaLifecycle = {
         ...this.paaLifecycle,
@@ -4430,6 +4575,17 @@ function stripInternalDebug(result, keepDebug) {
   delete diagnostics.debug;
   return { ...result, diagnostics };
 }
+async function resolveSurvivingGotoLinks(result) {
+  try {
+    return await resolveHarvestResultGoogleGotoLinks(result, { totalBudgetMs: 1e4 });
+  } catch (error) {
+    console.warn(JSON.stringify({
+      event: "harvest_goto_resolution_failed",
+      message: errorMessage(error)
+    }));
+    return result;
+  }
+}
 async function cleanupDisposableProxy(kernelApiKey, proxyId) {
   if (!kernelApiKey || !proxyId) return;
   try {
@@ -4644,7 +4800,8 @@ async function harvest(rawOptions) {
         lastError = err;
         if (willRetry2) continue;
         if (attempt.result) {
-          return withResultQuality(stripInternalDebug(attempt.result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+          const resolvedResult2 = await resolveSurvivingGotoLinks(attempt.result);
+          return withResultQuality(stripInternalDebug(resolvedResult2, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
         }
         break;
       }
@@ -4679,7 +4836,8 @@ async function harvest(rawOptions) {
         if (willRetry2) continue;
         break;
       }
-      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+      const resolvedResult = await resolveSurvivingGotoLinks(result);
+      const finalResult = withResultQuality(stripInternalDebug(resolvedResult, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
       const outcome = classifyAttemptResult(finalResult);
       const willRetry = outcome === "degraded_result" && i < maxAttempts - 1;
       const resultError = outcome === "degraded_result" ? degradedResultMessage(finalResult) : null;

package/dist/{chunk-QVW5S6BD.js → chunk-XUM2LEGU.js}

@@ -12,13 +12,13 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "./chunk-NH3EUCZL.js";
+} from "./chunk-JECPQBLZ.js";
 import {
   renderInstallTerminal
-} from "./chunk-R4YRH5PG.js";
+} from "./chunk-6FL7ZZCF.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-OIGM6VKQ.js";
+} from "./chunk-2RTELL6M.js";
 
 // src/mcp/stdio-runtime.ts
 import { readFileSync } from "fs";

package/dist/index.cjs

@@ -2598,6 +2598,93 @@ async function resolveGoogleGotoUrls(rawUrls, options = {}) {
   await Promise.all(Array.from({ length: Math.min(concurrency, inputs.length) }, () => worker()));
   return { resolutions: [...records.values()], nonGoogleRequestsObserved: 0 };
 }
+function resolvedSite(link, fallback) {
+  try {
+    return new URL(link.resolvedUrl ?? link.url).hostname.replace(/^www\./, "");
+  } catch {
+    return fallback;
+  }
+}
+async function resolveHarvestResultGoogleGotoLinks(result, options = {}) {
+  const treeNodes = [];
+  const visit = (nodes) => {
+    for (const node of nodes) {
+      treeNodes.push(node);
+      visit(node.children ?? []);
+    }
+  };
+  visit(result.tree ?? []);
+  const candidates = [
+    ...result.flat.flatMap((row) => row.sources ?? []),
+    ...treeNodes.flatMap((node) => node.sources),
+    ...result.organicResults ?? [],
+    ...(result.aiOverview?.citations ?? []).map(({ text: _text, href, ...link }) => ({ ...link, url: href })),
+    ...(result.aiMode?.citations ?? []).map(({ text: _text, href, ...link }) => ({ ...link, url: href }))
+  ];
+  const unique = new Map(candidates.filter((link) => Boolean(link?.rawUrl)).map((link) => [link.rawUrl, link]));
+  const gotoUrls = [...unique.values()].filter((link) => link.linkType === "google_goto_redirect" && link.resolutionStatus !== "resolved").map((link) => link.rawUrl);
+  if (gotoUrls.length > 0) {
+    const batch = await resolveGoogleGotoUrls(gotoUrls, options);
+    for (const resolution2 of batch.resolutions) {
+      const link = unique.get(resolution2.rawUrl);
+      if (!link) continue;
+      unique.set(resolution2.rawUrl, resolution2.resolutionStatus === "resolved" && resolution2.resolvedUrl ? { ...link, url: resolution2.resolvedUrl, resolvedUrl: resolution2.resolvedUrl, resolutionStatus: "resolved" } : { ...link, resolutionStatus: resolution2.resolutionStatus });
+    }
+  }
+  const replace = (link) => ({
+    ...link,
+    ...unique.get(link.rawUrl) ?? link
+  });
+  const replaceSources = (sources) => sources.map((source) => {
+    const link = replace(source);
+    return { ...link, site: resolvedSite(link, source.site) };
+  });
+  for (const row of result.flat) {
+    row.sources = replaceSources(row.sources ?? []);
+    const primary = row.sources[0];
+    row.source_title = primary?.title ?? row.source_title;
+    row.source_site = primary?.site ?? row.source_site;
+    row.source_cite = primary?.url ?? row.source_cite;
+  }
+  for (const node of treeNodes) {
+    node.sources = replaceSources(node.sources ?? []);
+    const primary = node.sources[0];
+    node.sourceTitle = primary?.title ?? node.sourceTitle;
+    node.sourceSite = primary?.site ?? node.sourceSite;
+    node.sourceCite = primary?.url ?? node.sourceCite;
+  }
+  for (let index = 0; index < (result.organicResults?.length ?? 0); index++) {
+    const link = replace(result.organicResults[index]);
+    result.organicResults[index] = { ...link, domain: resolvedSite(link, result.organicResults[index].domain) ?? "" };
+  }
+  const replaceCitation = (citation) => {
+    const link = replace({
+      url: citation.href,
+      rawUrl: citation.rawUrl,
+      resolvedUrl: citation.resolvedUrl,
+      linkType: citation.linkType,
+      resolutionStatus: citation.resolutionStatus
+    });
+    const { url, ...metadata } = link;
+    return { ...citation, ...metadata, href: url };
+  };
+  if (result.aiOverview) result.aiOverview.citations = (result.aiOverview.citations ?? []).map(replaceCitation);
+  if (result.aiMode) result.aiMode.citations = (result.aiMode.citations ?? []).map(replaceCitation);
+  const values = [...unique.values()];
+  result.diagnostics.links = {
+    linkCandidatesObserved: unique.size,
+    plainLinksObserved: values.filter((link) => link.linkType === "plain").length,
+    googleUrlRedirectsObserved: values.filter((link) => link.linkType === "google_url_redirect").length,
+    googleGotoRedirectsObserved: values.filter((link) => link.linkType === "google_goto_redirect").length,
+    redirectsResolved: values.filter((link) => link.linkType !== "plain" && link.resolutionStatus === "resolved").length,
+    redirectsUnresolved: values.filter((link) => link.resolutionStatus === "unresolved").length,
+    redirectsRejected: values.filter((link) => link.resolutionStatus === "rejected").length,
+    resolverNonGoogleRequestsObserved: 0,
+    paaSourcesPreserved: result.flat.reduce((total, row) => total + row.sources.length, 0),
+    aiOverviewLabelValidated: result.aiOverview?.detected ?? false
+  };
+  return result;
+}
 async function resolveGoogleOutboundLinks(page, values, options = {}) {
   const byRawUrl = /* @__PURE__ */ new Map();
   for (const value of values) byRawUrl.set(value.rawUrl, value);
@@ -2718,6 +2805,10 @@ async function resolveGoogleOutboundLinks(page, values, options = {}) {
 }
 
 // src/extractor/PAAExtractor.ts
+var PAA_ANSWER_PLACEHOLDER_PATTERN = String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`;
+var PAA_CLICK_EARLIEST_CONFIRMATION_MS = 700;
+var PAA_CLICK_ATTEMPT_BUDGET_MS = 1400;
+var PAA_FRONTIER_BATCH_SIZE = 3;
 var PAAExtractor = class {
   constructor(driver, reporter, onPaaProgress) {
     this.driver = driver;
@@ -2862,7 +2953,7 @@ var PAAExtractor = class {
   }
   async extractVisibleItems(page, onlyQuestion) {
     const sels = PAASelectors;
-    const raw = await page.evaluate(({ selectors, onlyQuestion: onlyQuestion2 }) => {
+    const raw = await page.evaluate(({ selectors, onlyQuestion: onlyQuestion2, answerPlaceholderPattern }) => {
       function cleanText(el) {
         if (!el) return "";
         const parts = [];
@@ -2889,12 +2980,13 @@ var PAAExtractor = class {
         const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
         const answerElement = aiAnswerBody ?? answerContainer;
         const answerText = cleanText(answerElement);
+        const answerIsPlaceholder = new RegExp(answerPlaceholderPattern, "i").test(answerText);
         const citationCandidates = [
           ...Array.from((answerElement ?? pair).querySelectorAll("a[href]")),
           ...Array.from(pair.querySelectorAll(selectors.aiCitation)),
           ...Array.from(pair.querySelectorAll(selectors.aiSourceCard))
         ];
-        const hasLoadedAnswer = expanded || answerText.length > 30 && !/an error has occurred/i.test(answerText.slice(0, 80)) && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href));
+        const hasLoadedAnswer = !answerIsPlaceholder && (expanded && answerText.length >= 10 || answerText.length > 30 && citationCandidates.some((anchor) => /^https?:\/\//.test(anchor.href)));
         let sourceTitle = hasLoadedAnswer ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
         let sourceSite = hasLoadedAnswer ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
         const sources = [];
@@ -2918,7 +3010,7 @@ var PAAExtractor = class {
           sources
         };
       });
-    }, { selectors: sels, onlyQuestion });
+    }, { selectors: sels, onlyQuestion, answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN });
     const baseUrl = typeof page.url === "function" ? page.url() : "https://www.google.com/";
     return raw.flatMap((item) => {
       const sources = [];
@@ -3109,44 +3201,52 @@ var PAAExtractor = class {
       questionEl: PAASelectors.itemQuestionEl,
       answerSel: PAASelectors.answerContainer,
       expandedClass: PAASelectors.expandedClass,
-      clickTarget: PAASelectors.clickTarget
+      clickTarget: PAASelectors.clickTarget,
+      answerPlaceholderPattern: PAA_ANSWER_PLACEHOLDER_PATTERN,
+      questionsOnly: options.questionsOnly
     };
     const readItemStates = () => page.evaluate(
-      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel }) => {
-        return Array.from(document.querySelectorAll(sel)).map((el) => ({
-          q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
-          done: el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true" || Boolean((el.querySelector(answerSel)?.textContent ?? "").trim())
-        })).filter((s) => s.q);
+      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget, answerSel, answerPlaceholderPattern, questionsOnly }) => {
+        return Array.from(document.querySelectorAll(sel)).map((el, index) => {
+          const answerText = (el.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
+          const expanded = el.classList.contains(expandedClass) || el.querySelector(clickTarget)?.getAttribute("aria-expanded") === "true";
+          const answerReady = answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
+          return {
+            q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
+            done: expanded && (questionsOnly || answerReady),
+            expanded,
+            index
+          };
+        }).filter((s) => s.q);
       },
       itemArgs
     );
-    const expandOneItemSerially = async (q, beforeCount) => {
+    const expandOneItemSerially = async (q, beforeCount, itemIndex, wasExpanded) => {
       const clickStartedMs = Date.now();
-      const items = page.locator(itemArgs.sel);
-      const itemIndex = await items.evaluateAll(
-        (elements, { dataQ, dataInitQ, target }) => elements.findIndex((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ)) === target),
-        { dataQ: itemArgs.dataQ, dataInitQ: itemArgs.dataInitQ, target: q }
-      ).catch(() => -1);
-      const clickTarget = itemIndex >= 0 ? items.nth(itemIndex).locator(itemArgs.clickTarget) : null;
+      const clickTarget = itemIndex >= 0 ? page.locator(itemArgs.sel).nth(itemIndex).locator(itemArgs.clickTarget) : null;
       const clicked = clickTarget ? await (async () => {
-        await clickTarget.scrollIntoViewIfNeeded({ timeout: 5e3 });
-        const box = await clickTarget.boundingBox({ timeout: 5e3 });
-        if (!box) return false;
-        await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2);
+        if (wasExpanded) {
+          await clickTarget.click({ timeout: 2e3, force: true });
+          await page.waitForTimeout(250);
+        }
+        await clickTarget.click({ timeout: 2e3, force: true });
         return true;
       })().catch(() => false) : false;
       if (!clicked) return { dispatched: false, confirmed: false };
-      const remainingHumanClickDelayMs = 700 - (Date.now() - clickStartedMs);
-      if (remainingHumanClickDelayMs > 0) await page.waitForTimeout(remainingHumanClickDelayMs);
+      const remainingConfirmationMs = Math.max(100, PAA_CLICK_ATTEMPT_BUDGET_MS - (Date.now() - clickStartedMs));
       const confirmed = await page.waitForFunction(
-        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, target, min }) => {
-          const items2 = Array.from(document.querySelectorAll(sel));
-          const item = items2.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
-          if (!item) return items2.length > min;
-          return item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true" || Boolean((item.querySelector(answerSel)?.textContent ?? "").trim()) || items2.length > min;
+        ({ sel, dataQ, dataInitQ, questionEl, expandedClass, clickTarget: clickTarget2, answerSel, answerPlaceholderPattern, questionsOnly, target, min, earliestConfirmationAt }) => {
+          if (Date.now() < earliestConfirmationAt) return false;
+          const items = Array.from(document.querySelectorAll(sel));
+          const item = items.find((element) => (element.getAttribute(dataQ) || element.getAttribute(dataInitQ) || element.querySelector(questionEl)?.innerText?.trim()) === target);
+          if (!item) return false;
+          const expanded = item.classList.contains(expandedClass) || item.querySelector(clickTarget2)?.getAttribute("aria-expanded") === "true";
+          if (questionsOnly) return expanded || items.length > min;
+          const answerText = (item.querySelector(answerSel)?.textContent ?? "").replace(/\s+/g, " ").trim();
+          return answerText.length >= 10 && !new RegExp(answerPlaceholderPattern, "i").test(answerText);
         },
-        { ...itemArgs, target: q, min: beforeCount },
-        { timeout: 2500, polling: 100 }
+        { ...itemArgs, target: q, min: beforeCount, earliestConfirmationAt: clickStartedMs + PAA_CLICK_EARLIEST_CONFIRMATION_MS },
+        { timeout: remainingConfirmationMs, polling: 100 }
       ).catch(() => null);
       return { dispatched: true, confirmed: Boolean(confirmed) };
     };
@@ -3191,10 +3291,43 @@ var PAAExtractor = class {
         };
         break;
       }
+      if (!options.questionsOnly && seenQs.size >= options.maxQuestions) {
+        let materialChanged = false;
+        const keptQuestions = new Set(orderedQs.slice(0, options.maxQuestions));
+        for (const item of await this.extractVisibleItems(page)) {
+          if (!keptQuestions.has(item.question)) continue;
+          const existing = capturedItems.get(item.question);
+          const merged = {
+            ...item,
+            answer: item.answer ?? existing?.answer,
+            sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
+            sourceSite: item.sourceSite ?? existing?.sourceSite,
+            sourceCite: item.sourceCite ?? existing?.sourceCite,
+            sources: this.mergePaaSources(existing?.sources, item.sources)
+          };
+          capturedItems.set(item.question, merged);
+          materialChanged = this.upsertCollectedRow(this.toFlatRow(merged, 1, null, options.query)) || materialChanged;
+        }
+        if (materialChanged) await this.emitProgress("answer_captured");
+        if ([...keptQuestions].every((question) => {
+          const item = capturedItems.get(question);
+          return Boolean(item?.answer?.trim() && item.sources.length > 0);
+        })) {
+          failedInteractions.clear();
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            discoveryStatus: "target_reached",
+            targetStatus: "met",
+            failedInteractions: 0,
+            completionReason: "The requested unique-question count and its answer/source material were observed."
+          };
+          break;
+        }
+      }
       const kept = new Set(orderedQs.slice(0, options.maxQuestions));
       const clickable = states.filter((state) => kept.has(state.q) && !state.done && (clickAttempts.get(state.q) ?? 0) < MAX_CLICK_ATTEMPTS && !abandonedNeverReclick.has(state.q));
-      const target = clickable[0];
-      if (!target) {
+      const targets = clickable.slice(-PAA_FRONTIER_BATCH_SIZE);
+      if (targets.length === 0) {
         if (seenQs.size >= options.maxQuestions) {
           this.paaLifecycle = {
             ...this.paaLifecycle,
@@ -3242,31 +3375,43 @@ var PAAExtractor = class {
         };
         break;
       }
-      this.reporter.onDepth(++round);
       await this.throwIfCaptcha(page, "Google PAA expansion");
-      clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
-      const expansionResult = await expandOneItemSerially(target.q, states.length);
-      if (expansionResult.dispatched) this.clickedQuestions.add(target.q);
-      if (!options.questionsOnly) {
-        let clickedItem = (await this.extractVisibleItems(page, target.q))[0];
-        if (!clickedItem?.answer) {
-          await page.waitForTimeout(1300);
-          clickedItem = (await this.extractVisibleItems(page, target.q))[0];
+      for (const target of targets) {
+        this.reporter.onDepth(++round);
+        if ((clickAttempts.get(target.q) ?? 0) > 0) {
+          this.paaLifecycle = {
+            ...this.paaLifecycle,
+            automaticRetries: this.paaLifecycle.automaticRetries + 1
+          };
         }
-        if (clickedItem) {
-          capturedItems.set(target.q, clickedItem);
+        clickAttempts.set(target.q, (clickAttempts.get(target.q) ?? 0) + 1);
+      }
+      const expansionResults = await Promise.all(targets.map(async (target) => ({
+        target,
+        result: await expandOneItemSerially(target.q, states.length, target.index, target.expanded)
+      })));
+      for (const { target, result } of expansionResults) {
+        if (result.dispatched) this.clickedQuestions.add(target.q);
+      }
+      if (!options.questionsOnly) {
+        const targetQuestions = new Set(targets.map((target) => target.q));
+        for (const clickedItem of await this.extractVisibleItems(page)) {
+          if (!targetQuestions.has(clickedItem.question)) continue;
+          capturedItems.set(clickedItem.question, clickedItem);
           if (this.upsertCollectedRow(this.toFlatRow(clickedItem, 1, null, options.query))) {
             await this.emitProgress("answer_captured");
           }
         }
       }
-      const captured = capturedItems.get(target.q);
-      if (expansionResult.confirmed || Boolean(captured?.answer?.trim())) {
-        processedQuestions.add(target.q);
-        failedInteractions.delete(target.q);
-      } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
-        failedInteractions.add(target.q);
-        abandonedNeverReclick.add(target.q);
+      for (const { target, result } of expansionResults) {
+        const captured = capturedItems.get(target.q);
+        if (result.confirmed || Boolean(captured?.answer?.trim())) {
+          processedQuestions.add(target.q);
+          failedInteractions.delete(target.q);
+        } else if ((clickAttempts.get(target.q) ?? 0) >= MAX_CLICK_ATTEMPTS) {
+          failedInteractions.add(target.q);
+          abandonedNeverReclick.add(target.q);
+        }
       }
       this.paaLifecycle = {
         ...this.paaLifecycle,
@@ -4970,6 +5115,17 @@ function stripInternalDebug(result, keepDebug) {
   delete diagnostics.debug;
   return { ...result, diagnostics };
 }
+async function resolveSurvivingGotoLinks(result) {
+  try {
+    return await resolveHarvestResultGoogleGotoLinks(result, { totalBudgetMs: 1e4 });
+  } catch (error) {
+    console.warn(JSON.stringify({
+      event: "harvest_goto_resolution_failed",
+      message: errorMessage(error)
+    }));
+    return result;
+  }
+}
 async function cleanupDisposableProxy(kernelApiKey, proxyId) {
   if (!kernelApiKey || !proxyId) return;
   try {
@@ -5184,7 +5340,8 @@ async function harvest(rawOptions) {
         lastError = err;
         if (willRetry2) continue;
         if (attempt.result) {
-          return withResultQuality(stripInternalDebug(attempt.result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+          const resolvedResult2 = await resolveSurvivingGotoLinks(attempt.result);
+          return withResultQuality(stripInternalDebug(resolvedResult2, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
         }
         break;
       }
@@ -5219,7 +5376,8 @@ async function harvest(rawOptions) {
         if (willRetry2) continue;
         break;
       }
-      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
+      const resolvedResult = await resolveSurvivingGotoLinks(result);
+      const finalResult = withResultQuality(stripInternalDebug(resolvedResult, requestedDebug), attemptOptions.maxQuestions, attemptOptions.questionsOnly);
       const outcome = classifyAttemptResult(finalResult);
       const willRetry = outcome === "degraded_result" && i < maxAttempts - 1;
       const resultError = outcome === "degraded_result" ? degradedResultMessage(finalResult) : null;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   harvest
-} from "./chunk-UFJJGXVO.js";
+} from "./chunk-WEEAXPWK.js";
 import "./chunk-OM7HVEJ3.js";
 import {
   recordVendorUsage

package/dist/{server-MNSYIX42.js → server-5OSHEEVT.js}

@@ -229,7 +229,7 @@ import {
   serpArrivalUrl,
   validateProviderSessionId,
   withRuntimeUserAgent
-} from "./chunk-UFJJGXVO.js";
+} from "./chunk-WEEAXPWK.js";
 import {
   AnalyticsConfirmedPersonProjectionSchema,
   AnalyticsCrmFieldSchema,
@@ -316,7 +316,7 @@ import {
   resolveDeploymentProfile,
   resolveLocalSourcebookSchemaType,
   transcribeMediaUrl
-} from "./chunk-NH3EUCZL.js";
+} from "./chunk-JECPQBLZ.js";
 import {
   auditImageUrls,
   auditImages,
@@ -435,7 +435,7 @@ import {
 } from "./chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-OIGM6VKQ.js";
+} from "./chunk-2RTELL6M.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -77738,7 +77738,7 @@ app.get("/cron/tick", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const startedAt = Date.now();
-  const { drainQueue } = await import("./worker-VU6JCCLG.js");
+  const { drainQueue } = await import("./worker-2TYCYFPM.js");
   const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
   const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
   const unfinished = [];

package/dist/{worker-VU6JCCLG.js → worker-2TYCYFPM.js}

@@ -9,7 +9,7 @@ import {
 import "./chunk-OZJMVCDK.js";
 import {
   harvest
-} from "./chunk-UFJJGXVO.js";
+} from "./chunk-WEEAXPWK.js";
 import {
   sanitizeAttempts,
   sanitizeHarvestResult

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.79.1",
+  "version": "0.79.2",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",