mcp-scraper 0.86.5 → 0.88.0

package/dist/{chunk-NH4QAH3X.js → chunk-S24LFPL7.js}

@@ -15,7 +15,7 @@ import {
   RawPAAItemSchema,
   postToMemoryLibrary,
   recencyToTbs
-} from "./chunk-OWF2JJKN.js";
+} from "./chunk-GXBZXWXB.js";
 import {
   CaptchaError,
   ExtractionError,
@@ -134,6 +134,58 @@ var MapsSelectors = {
   expandReview: '[data-review-id] button[aria-label*="See more"], [data-review-id] button.w8nwRe'
 };
 
+// src/driver/IBrowserDriver.ts
+async function withManagedTemporaryPage(context, setup, operation, signal) {
+  let page;
+  let ended = false;
+  let timer;
+  let rejectStop = () => {
+  };
+  const aborted = () => signal?.reason instanceof DOMException && signal.reason.name === "TimeoutError" ? signal.reason : new RequestAbortedError();
+  const onAbort = () => rejectStop(aborted());
+  const stop = new Promise((_, reject) => {
+    rejectStop = reject;
+  });
+  const close = async (target) => {
+    let closeTimer;
+    try {
+      await Promise.race([
+        target.close().catch(() => {
+        }),
+        new Promise((resolve) => {
+          closeTimer = setTimeout(resolve, 2e3);
+        })
+      ]);
+    } finally {
+      if (closeTimer) clearTimeout(closeTimer);
+    }
+  };
+  if (signal?.aborted) throw aborted();
+  signal?.addEventListener("abort", onAbort, { once: true });
+  timer = setTimeout(() => rejectStop(new DOMException("Optional pagination timed out", "TimeoutError")), 3e4);
+  try {
+    return await Promise.race([
+      (async () => {
+        const created = await context.newPage();
+        if (ended) {
+          await close(created);
+          throw new RequestAbortedError();
+        }
+        page = created;
+        await setup(created);
+        if (ended) throw new RequestAbortedError();
+        return operation(created);
+      })(),
+      stop
+    ]);
+  } finally {
+    ended = true;
+    if (timer) clearTimeout(timer);
+    signal?.removeEventListener("abort", onAbort);
+    if (page) await close(page);
+  }
+}
+
 // src/lib/browser-user-agent.ts
 var RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
 async function detectPageUserAgent(page) {
@@ -523,6 +575,11 @@ var BrowserDriver = class {
       };
     });
   }
+  async withTemporaryPage(operation, signal) {
+    if (!this.context) throw new Error("Browser context is not available");
+    return withManagedTemporaryPage(this.context, async () => {
+    }, operation, signal);
+  }
   async navigateToSERP(query, uule, gl, hl, options) {
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
     if (options?.tbs) params.set("tbs", options.tbs);
@@ -927,7 +984,10 @@ var BrightDataSerpDriver = class {
     const customCdp = this.cdpSession;
     const sessionResponse = await customCdp.send("Browser.getSessionId");
     this.providerSessionId = validateProviderSessionId(sessionResponse?.sessionId);
-    await this.page.route("**/*", async (route) => {
+    await this.setupPage(this.page);
+  }
+  async setupPage(page) {
+    await page.route("**/*", async (route) => {
       const request = route.request();
       const resourceType = request.resourceType();
       let requestUrl;
@@ -952,6 +1012,10 @@ var BrightDataSerpDriver = class {
       await route.continue();
     });
   }
+  async withTemporaryPage(operation, signal) {
+    if (!this.context || this.closing) throw new Error("Browser context is not available");
+    return withManagedTemporaryPage(this.context, (page) => this.setupPage(page), operation, signal);
+  }
   async navigateToSERP(query, uule, gl, hl, options) {
     if (!this.page) throw new Error("Browser page is not available");
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
@@ -1927,6 +1991,11 @@ function parsedGoogleGoto(value) {
   if (!url || url.protocol !== "https:" || !isGoogleHost(url.hostname) || url.pathname !== "/goto") return null;
   return url.searchParams.get("url") ? url : null;
 }
+function hasResolvedGotoDestination(value) {
+  if (value.linkType !== "google_goto_redirect" || value.resolutionStatus !== "resolved" || !parsedGoogleGoto(value.rawUrl) || !value.resolvedUrl) return false;
+  const destination = safeHttpUrl(value.resolvedUrl);
+  return Boolean(destination && !isGoogleHost(destination.hostname) && destination.href === value.resolvedUrl && value.url === destination.href);
+}
 function questionIdFor(question) {
   const normalized = question.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu, "").replace(/\s+/g, " ").trim();
   return `paa_${createHash("sha256").update(normalized).digest("hex").slice(0, 16)}`;
@@ -2143,8 +2212,12 @@ async function resolveHarvestResultGoogleGotoLinks(result, options = {}) {
 }
 async function resolveGoogleOutboundLinks(page, values, options = {}) {
   const byRawUrl = /* @__PURE__ */ new Map();
-  for (const value of values) byRawUrl.set(value.rawUrl, value);
-  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect");
+  for (const value of values) {
+    const existing = byRawUrl.get(value.rawUrl);
+    if (existing && hasResolvedGotoDestination(existing)) continue;
+    byRawUrl.set(value.rawUrl, value.linkType === "google_goto_redirect" && !hasResolvedGotoDestination(value) ? { ...value, url: value.rawUrl, resolvedUrl: null, resolutionStatus: "unresolved" } : value);
+  }
+  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect" && !hasResolvedGotoDestination(value));
   if (inputs.length === 0) return byRawUrl;
   const deadlineMs = Date.now() + (options.totalBudgetMs ?? DEFAULT_TOTAL_BUDGET_MS);
   const direct = await resolveGoogleGotoUrls(inputs.map((value) => value.rawUrl), options);
@@ -2360,7 +2433,7 @@ var PAAExtractor = class {
     ];
     const unique = new Map(candidates.map((link) => [link.rawUrl, link]));
     let resolverNonGoogleRequestsObserved = 0;
-    const resolved = await resolveGoogleOutboundLinks(page, unique.values(), {
+    const resolved = await resolveGoogleOutboundLinks(page, candidates, {
       onNetworkAudit: (audit) => {
         resolverNonGoogleRequestsObserved = audit.nonGoogleRequestsObserved;
       }
@@ -2611,6 +2684,7 @@ var PAAExtractor = class {
       diagnostics: {
         completionStatus: "paa_found",
         problem: null,
+        ...material?.pagination ? { pagination: { ...material.pagination } } : {},
         resultQuality: "partial",
         degradedResult: false,
         retryRecommended: true,
@@ -3575,6 +3649,55 @@ var PAAExtractor = class {
       ...locationEvidence ? { locationEvidence } : {}
     };
   }
+  async captureSecondOrganicPage(page, options, signal) {
+    this.throwIfAborted(signal);
+    if (!this.driver.withTemporaryPage) return { organic: [], status: "failed", failureCode: "unsupported_driver" };
+    let guardedFailure;
+    try {
+      const initial = new URL(page.url());
+      const valid = (raw) => {
+        try {
+          const url = new URL(raw, initial);
+          return initial.origin === "https://www.google.com" && url.origin === initial.origin && !url.username && !url.password && url.pathname === "/search" && url.searchParams.getAll("q").length === 1 && url.searchParams.get("q") === options.query && url.searchParams.getAll("start").length === 1 && url.searchParams.get("start") === "10" && (!url.searchParams.has("num") || url.searchParams.get("num") === "10") && ["gl", "hl"].every((key) => !url.searchParams.has(key) || url.searchParams.getAll(key).length === 1 && url.searchParams.get(key) === options[key]) && ["uule", "tbs", "udm", "tbm", "pws"].every((key) => url.searchParams.getAll(key).length <= 1 && url.searchParams.get(key) === initial.searchParams.get(key));
+        } catch {
+          return false;
+        }
+      };
+      const href = await page.evaluate(() => {
+        const next = document.querySelector('a#pnnext, a[rel="next"]');
+        return next?.getAttribute("href") ?? null;
+      });
+      if (!href) return { organic: [], status: "unavailable", failureCode: "missing_next" };
+      if (!valid(href)) return { organic: [], status: "unavailable", failureCode: "invalid_next" };
+      const organic = await this.driver.withTemporaryPage(async (second) => {
+        await second.route("**/*", async (route) => {
+          const request = route.request();
+          if (request.isNavigationRequest() && request.frame() === second.mainFrame() && !valid(request.url())) {
+            guardedFailure = /^https:\/\/www\.google\.com\/sorry(?:\/|\?)/.test(request.url()) ? "captcha" : "invalid_next";
+            await route.abort("blockedbyclient");
+          } else await route.fallback();
+        });
+        await second.goto(new URL(href, initial).href, { waitUntil: "domcontentloaded", timeout: 2e4 });
+        this.throwIfAborted(signal);
+        await this.throwIfCaptcha(second, "Google SERP page 2");
+        if (!valid(second.url())) {
+          guardedFailure = "invalid_next";
+          throw new Error("Invalid pagination landing");
+        }
+        return this.extractOrganicResults(second);
+      }, signal);
+      this.throwIfAborted(signal);
+      return organic.length > 0 ? { organic: organic.map((row) => ({ ...row, position: row.position + 10 })), status: "captured" } : { organic: [], status: "unavailable", failureCode: "empty_page" };
+    } catch (err) {
+      this.throwIfAborted(signal);
+      if (err instanceof RequestAbortedError) throw err;
+      return {
+        organic: [],
+        status: "failed",
+        failureCode: guardedFailure ?? (err instanceof CaptchaError ? "captcha" : err instanceof Error && (err.name === "TimeoutError" || /timed? ?out/i.test(err.message)) ? "timeout" : "navigation_error")
+      };
+    }
+  }
   async extract(options, signal) {
     const startMs = Date.now();
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
@@ -3778,23 +3901,49 @@ var PAAExtractor = class {
       const initialLocationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults, localPack) : void 0;
       this.reporter.onVideos(videos);
       this.reporter.onForums(forums);
+      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
+      let pagination = {
+        requestedPages: (options.pages ?? 1) >= 2 ? 2 : 1,
+        capturedPages: 1,
+        page2Status: (options.pages ?? 1) >= 2 ? "not_attempted" : "not_requested",
+        page1OrganicCount: organicResults.length,
+        page2OrganicCount: 0
+      };
+      let allOrganic = organicResults;
+      this.checkpointMaterial = {
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        whatPeopleSaying,
+        videos,
+        forums,
+        organicResults: allOrganic,
+        localPack,
+        entityIds,
+        pagination
+      };
+      await this.emitProgress("serp_captured");
+      if (pagination.requestedPages === 2) {
+        const second = await this.captureSecondOrganicPage(page, executionOptions, signal);
+        allOrganic = [...organicResults, ...second.organic];
+        pagination = {
+          ...pagination,
+          capturedPages: second.status === "captured" ? 2 : 1,
+          page2Status: second.status,
+          page2OrganicCount: second.organic.length,
+          ...second.failureCode ? { failureCode: second.failureCode } : {}
+        };
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic, pagination };
+        await this.emitProgress("serp_captured");
+        await this.resolveMaterialLinks(page, [], allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic };
+        await this.emitProgress("serp_captured");
+      }
+      const locationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack) : initialLocationEvidence;
       if (!hasPaa) {
-        let noPaaOrganic = organicResults;
-        let locationEvidence2 = initialLocationEvidence;
-        if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
-          if (uule) p2params.set("uule", uule);
-          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
-          await this.throwIfCaptcha(page, "Google SERP page 2");
-          const p2organic = await this.extractOrganicResults(page);
-          noPaaOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
-          if (options.debug) {
-            locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
-          }
-        }
-        const aiSurfaces2 = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
-        await this.resolveMaterialLinks(page, [], noPaaOrganic, aiSurfaces2.aiOverview, aiSurfaces2.aiMode);
+        await this.resolveMaterialLinks(page, [], allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic };
+        await this.emitProgress("serp_captured");
         const stats2 = {
           seed: executionOptions.query,
           totalQuestions: 0,
@@ -3811,39 +3960,27 @@ var PAAExtractor = class {
             completionStatus: "no_paa",
             noPaaObserved: true,
             problem: null,
+            pagination,
             paaLifecycle: { ...this.paaLifecycle },
             completeness: { ...this.completeness },
             links: { ...this.linkDiagnostics },
-            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence, locationResolution) } : {}
           },
           totalQuestions: 0,
-          surface: aiSurfaces2.surface,
-          aiOverview: aiSurfaces2.aiOverview,
-          aiMode: aiSurfaces2.aiMode,
+          surface: aiSurfaces.surface,
+          aiOverview: aiSurfaces.aiOverview,
+          aiMode: aiSurfaces.aiMode,
           whatPeopleSaying,
           tree: [],
           flat: [],
           videos,
           forums,
-          organicResults: noPaaOrganic,
+          organicResults: allOrganic,
           localPack,
           entityIds,
           stats: stats2
         };
       }
-      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
-      this.checkpointMaterial = {
-        surface: aiSurfaces.surface,
-        aiOverview: aiSurfaces.aiOverview,
-        aiMode: aiSurfaces.aiMode,
-        whatPeopleSaying,
-        videos,
-        forums,
-        organicResults,
-        localPack,
-        entityIds
-      };
-      await this.emitProgress("serp_captured");
       const flat = await this.runBFS(page, executionOptions, signal);
       this.throwIfAborted(signal);
       const shortVidsParams = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
@@ -3864,20 +4001,6 @@ var PAAExtractor = class {
         }
       }
       this.reporter.onVideos(shortVideos);
-      let allOrganic = organicResults;
-      let locationEvidence = initialLocationEvidence;
-      if ((options.pages ?? 1) >= 2) {
-        const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-        if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
-        if (uule) p2params.set("uule", uule);
-        await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
-        await this.throwIfCaptcha(page, "Google SERP page 2");
-        const p2organic = await this.extractOrganicResults(page);
-        allOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
-        if (options.debug) {
-          locationEvidence = inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack);
-        }
-      }
       await this.resolveMaterialLinks(page, flat, allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
       this.checkpointMaterial = {
         ...this.checkpointMaterial,
@@ -3887,6 +4010,7 @@ var PAAExtractor = class {
         videos: [...videos, ...shortVideos],
         organicResults: allOrganic
       };
+      await this.emitProgress("expansion_finished");
       const allVideos = [...videos, ...shortVideos];
       const tree = this.buildTree(flat, executionOptions.query);
       const stats = {
@@ -3904,6 +4028,7 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          pagination,
           paaLifecycle: { ...this.paaLifecycle },
           completeness: { ...this.completeness },
           links: { ...this.linkDiagnostics },

package/dist/{chunk-6IOGTFNS.js → chunk-UZPTGUDV.js}

@@ -4,7 +4,7 @@ import {
 import {
   DEFAULT_MAPS_PROXY_MODE,
   postToMemoryLibrary
-} from "./chunk-OWF2JJKN.js";
+} from "./chunk-GXBZXWXB.js";
 import {
   buildSerpEmailQuery,
   contactAttemptUrls,

package/dist/{chunk-BA6BPDSP.js → chunk-XIJI34RM.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.86.5";
+var PACKAGE_VERSION = "0.88.0";
 
 export {
   PACKAGE_VERSION

package/dist/{extract-bundle-FALNLKYY.js → extract-bundle-565SBZCR.js}

@@ -10,7 +10,7 @@ import {
   createSiteExtractContentReader,
   createSiteExtractImageArtifact,
   extractJobLimitInfo
-} from "./chunk-KQWVJOVR.js";
+} from "./chunk-IHXAXYIS.js";
 import "./chunk-OZJMVCDK.js";
 import {
   computeIssues,
@@ -19,7 +19,7 @@ import {
   renderLinkReport
 } from "./chunk-PGJQDMC2.js";
 import "./chunk-DNM65UCK.js";
-import "./chunk-NKRO2SUO.js";
+import "./chunk-4QMUF6XM.js";
 import "./chunk-GGZEC22A.js";
 import {
   getDb

package/dist/{gmail-service-BUSU4BVX.js → gmail-service-E6ALS7JG.js}

@@ -7,10 +7,10 @@ import {
   GmailServiceError,
   normalizeGmailMessage,
   parseGmailAddresses
-} from "./chunk-WFGAR4BZ.js";
+} from "./chunk-MZDNZQWT.js";
 import "./chunk-C5Z4OFKW.js";
 import "./chunk-T3MZISOF.js";
-import "./chunk-NKRO2SUO.js";
+import "./chunk-4QMUF6XM.js";
 import "./chunk-OPQIGAFB.js";
 import "./chunk-YXNDOQXN.js";
 export {