mcp-scraper 0.86.5 → 0.88.0

package/dist/index.cjs

@@ -395,6 +395,58 @@ var LocationMismatchError = class extends Error {
   }
 };
 
+// src/driver/IBrowserDriver.ts
+async function withManagedTemporaryPage(context, setup, operation, signal) {
+  let page;
+  let ended = false;
+  let timer;
+  let rejectStop = () => {
+  };
+  const aborted = () => signal?.reason instanceof DOMException && signal.reason.name === "TimeoutError" ? signal.reason : new RequestAbortedError();
+  const onAbort = () => rejectStop(aborted());
+  const stop = new Promise((_, reject) => {
+    rejectStop = reject;
+  });
+  const close = async (target) => {
+    let closeTimer;
+    try {
+      await Promise.race([
+        target.close().catch(() => {
+        }),
+        new Promise((resolve) => {
+          closeTimer = setTimeout(resolve, 2e3);
+        })
+      ]);
+    } finally {
+      if (closeTimer) clearTimeout(closeTimer);
+    }
+  };
+  if (signal?.aborted) throw aborted();
+  signal?.addEventListener("abort", onAbort, { once: true });
+  timer = setTimeout(() => rejectStop(new DOMException("Optional pagination timed out", "TimeoutError")), 3e4);
+  try {
+    return await Promise.race([
+      (async () => {
+        const created = await context.newPage();
+        if (ended) {
+          await close(created);
+          throw new RequestAbortedError();
+        }
+        page = created;
+        await setup(created);
+        if (ended) throw new RequestAbortedError();
+        return operation(created);
+      })(),
+      stop
+    ]);
+  } finally {
+    ended = true;
+    if (timer) clearTimeout(timer);
+    signal?.removeEventListener("abort", onAbort);
+    if (page) await close(page);
+  }
+}
+
 // src/api/cost-telemetry.ts
 var import_node_crypto2 = require("crypto");
 
@@ -1085,6 +1137,11 @@ var BrowserDriver = class {
       };
     });
   }
+  async withTemporaryPage(operation, signal) {
+    if (!this.context) throw new Error("Browser context is not available");
+    return withManagedTemporaryPage(this.context, async () => {
+    }, operation, signal);
+  }
   async navigateToSERP(query, uule, gl, hl, options) {
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
     if (options?.tbs) params.set("tbs", options.tbs);
@@ -1489,7 +1546,10 @@ var BrightDataSerpDriver = class {
     const customCdp = this.cdpSession;
     const sessionResponse = await customCdp.send("Browser.getSessionId");
     this.providerSessionId = validateProviderSessionId(sessionResponse?.sessionId);
-    await this.page.route("**/*", async (route) => {
+    await this.setupPage(this.page);
+  }
+  async setupPage(page) {
+    await page.route("**/*", async (route) => {
       const request = route.request();
       const resourceType = request.resourceType();
       let requestUrl;
@@ -1514,6 +1574,10 @@ var BrightDataSerpDriver = class {
       await route.continue();
     });
   }
+  async withTemporaryPage(operation, signal) {
+    if (!this.context || this.closing) throw new Error("Browser context is not available");
+    return withManagedTemporaryPage(this.context, (page) => this.setupPage(page), operation, signal);
+  }
   async navigateToSERP(query, uule, gl, hl, options) {
     if (!this.page) throw new Error("Browser page is not available");
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
@@ -2489,6 +2553,11 @@ function parsedGoogleGoto(value) {
   if (!url || url.protocol !== "https:" || !isGoogleHost(url.hostname) || url.pathname !== "/goto") return null;
   return url.searchParams.get("url") ? url : null;
 }
+function hasResolvedGotoDestination(value) {
+  if (value.linkType !== "google_goto_redirect" || value.resolutionStatus !== "resolved" || !parsedGoogleGoto(value.rawUrl) || !value.resolvedUrl) return false;
+  const destination = safeHttpUrl(value.resolvedUrl);
+  return Boolean(destination && !isGoogleHost(destination.hostname) && destination.href === value.resolvedUrl && value.url === destination.href);
+}
 function questionIdFor(question) {
   const normalized = question.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu, "").replace(/\s+/g, " ").trim();
   return `paa_${(0, import_node_crypto3.createHash)("sha256").update(normalized).digest("hex").slice(0, 16)}`;
@@ -2705,8 +2774,12 @@ async function resolveHarvestResultGoogleGotoLinks(result, options = {}) {
 }
 async function resolveGoogleOutboundLinks(page, values, options = {}) {
   const byRawUrl = /* @__PURE__ */ new Map();
-  for (const value of values) byRawUrl.set(value.rawUrl, value);
-  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect");
+  for (const value of values) {
+    const existing = byRawUrl.get(value.rawUrl);
+    if (existing && hasResolvedGotoDestination(existing)) continue;
+    byRawUrl.set(value.rawUrl, value.linkType === "google_goto_redirect" && !hasResolvedGotoDestination(value) ? { ...value, url: value.rawUrl, resolvedUrl: null, resolutionStatus: "unresolved" } : value);
+  }
+  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect" && !hasResolvedGotoDestination(value));
   if (inputs.length === 0) return byRawUrl;
   const deadlineMs = Date.now() + (options.totalBudgetMs ?? DEFAULT_TOTAL_BUDGET_MS);
   const direct = await resolveGoogleGotoUrls(inputs.map((value) => value.rawUrl), options);
@@ -2922,7 +2995,7 @@ var PAAExtractor = class {
     ];
     const unique = new Map(candidates.map((link) => [link.rawUrl, link]));
     let resolverNonGoogleRequestsObserved = 0;
-    const resolved = await resolveGoogleOutboundLinks(page, unique.values(), {
+    const resolved = await resolveGoogleOutboundLinks(page, candidates, {
       onNetworkAudit: (audit) => {
         resolverNonGoogleRequestsObserved = audit.nonGoogleRequestsObserved;
       }
@@ -3173,6 +3246,7 @@ var PAAExtractor = class {
       diagnostics: {
         completionStatus: "paa_found",
         problem: null,
+        ...material?.pagination ? { pagination: { ...material.pagination } } : {},
         resultQuality: "partial",
         degradedResult: false,
         retryRecommended: true,
@@ -4137,6 +4211,55 @@ var PAAExtractor = class {
       ...locationEvidence ? { locationEvidence } : {}
     };
   }
+  async captureSecondOrganicPage(page, options, signal) {
+    this.throwIfAborted(signal);
+    if (!this.driver.withTemporaryPage) return { organic: [], status: "failed", failureCode: "unsupported_driver" };
+    let guardedFailure;
+    try {
+      const initial = new URL(page.url());
+      const valid = (raw) => {
+        try {
+          const url = new URL(raw, initial);
+          return initial.origin === "https://www.google.com" && url.origin === initial.origin && !url.username && !url.password && url.pathname === "/search" && url.searchParams.getAll("q").length === 1 && url.searchParams.get("q") === options.query && url.searchParams.getAll("start").length === 1 && url.searchParams.get("start") === "10" && (!url.searchParams.has("num") || url.searchParams.get("num") === "10") && ["gl", "hl"].every((key) => !url.searchParams.has(key) || url.searchParams.getAll(key).length === 1 && url.searchParams.get(key) === options[key]) && ["uule", "tbs", "udm", "tbm", "pws"].every((key) => url.searchParams.getAll(key).length <= 1 && url.searchParams.get(key) === initial.searchParams.get(key));
+        } catch {
+          return false;
+        }
+      };
+      const href = await page.evaluate(() => {
+        const next = document.querySelector('a#pnnext, a[rel="next"]');
+        return next?.getAttribute("href") ?? null;
+      });
+      if (!href) return { organic: [], status: "unavailable", failureCode: "missing_next" };
+      if (!valid(href)) return { organic: [], status: "unavailable", failureCode: "invalid_next" };
+      const organic = await this.driver.withTemporaryPage(async (second) => {
+        await second.route("**/*", async (route) => {
+          const request = route.request();
+          if (request.isNavigationRequest() && request.frame() === second.mainFrame() && !valid(request.url())) {
+            guardedFailure = /^https:\/\/www\.google\.com\/sorry(?:\/|\?)/.test(request.url()) ? "captcha" : "invalid_next";
+            await route.abort("blockedbyclient");
+          } else await route.fallback();
+        });
+        await second.goto(new URL(href, initial).href, { waitUntil: "domcontentloaded", timeout: 2e4 });
+        this.throwIfAborted(signal);
+        await this.throwIfCaptcha(second, "Google SERP page 2");
+        if (!valid(second.url())) {
+          guardedFailure = "invalid_next";
+          throw new Error("Invalid pagination landing");
+        }
+        return this.extractOrganicResults(second);
+      }, signal);
+      this.throwIfAborted(signal);
+      return organic.length > 0 ? { organic: organic.map((row) => ({ ...row, position: row.position + 10 })), status: "captured" } : { organic: [], status: "unavailable", failureCode: "empty_page" };
+    } catch (err) {
+      this.throwIfAborted(signal);
+      if (err instanceof RequestAbortedError) throw err;
+      return {
+        organic: [],
+        status: "failed",
+        failureCode: guardedFailure ?? (err instanceof CaptchaError ? "captcha" : err instanceof Error && (err.name === "TimeoutError" || /timed? ?out/i.test(err.message)) ? "timeout" : "navigation_error")
+      };
+    }
+  }
   async extract(options, signal) {
     const startMs = Date.now();
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
@@ -4340,23 +4463,49 @@ var PAAExtractor = class {
       const initialLocationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults, localPack) : void 0;
       this.reporter.onVideos(videos);
       this.reporter.onForums(forums);
+      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
+      let pagination = {
+        requestedPages: (options.pages ?? 1) >= 2 ? 2 : 1,
+        capturedPages: 1,
+        page2Status: (options.pages ?? 1) >= 2 ? "not_attempted" : "not_requested",
+        page1OrganicCount: organicResults.length,
+        page2OrganicCount: 0
+      };
+      let allOrganic = organicResults;
+      this.checkpointMaterial = {
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        whatPeopleSaying,
+        videos,
+        forums,
+        organicResults: allOrganic,
+        localPack,
+        entityIds,
+        pagination
+      };
+      await this.emitProgress("serp_captured");
+      if (pagination.requestedPages === 2) {
+        const second = await this.captureSecondOrganicPage(page, executionOptions, signal);
+        allOrganic = [...organicResults, ...second.organic];
+        pagination = {
+          ...pagination,
+          capturedPages: second.status === "captured" ? 2 : 1,
+          page2Status: second.status,
+          page2OrganicCount: second.organic.length,
+          ...second.failureCode ? { failureCode: second.failureCode } : {}
+        };
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic, pagination };
+        await this.emitProgress("serp_captured");
+        await this.resolveMaterialLinks(page, [], allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic };
+        await this.emitProgress("serp_captured");
+      }
+      const locationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack) : initialLocationEvidence;
       if (!hasPaa) {
-        let noPaaOrganic = organicResults;
-        let locationEvidence2 = initialLocationEvidence;
-        if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
-          if (uule) p2params.set("uule", uule);
-          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
-          await this.throwIfCaptcha(page, "Google SERP page 2");
-          const p2organic = await this.extractOrganicResults(page);
-          noPaaOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
-          if (options.debug) {
-            locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
-          }
-        }
-        const aiSurfaces2 = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
-        await this.resolveMaterialLinks(page, [], noPaaOrganic, aiSurfaces2.aiOverview, aiSurfaces2.aiMode);
+        await this.resolveMaterialLinks(page, [], allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic };
+        await this.emitProgress("serp_captured");
         const stats2 = {
           seed: executionOptions.query,
           totalQuestions: 0,
@@ -4373,39 +4522,27 @@ var PAAExtractor = class {
             completionStatus: "no_paa",
             noPaaObserved: true,
             problem: null,
+            pagination,
             paaLifecycle: { ...this.paaLifecycle },
             completeness: { ...this.completeness },
             links: { ...this.linkDiagnostics },
-            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence, locationResolution) } : {}
           },
           totalQuestions: 0,
-          surface: aiSurfaces2.surface,
-          aiOverview: aiSurfaces2.aiOverview,
-          aiMode: aiSurfaces2.aiMode,
+          surface: aiSurfaces.surface,
+          aiOverview: aiSurfaces.aiOverview,
+          aiMode: aiSurfaces.aiMode,
           whatPeopleSaying,
           tree: [],
           flat: [],
           videos,
           forums,
-          organicResults: noPaaOrganic,
+          organicResults: allOrganic,
           localPack,
           entityIds,
           stats: stats2
         };
       }
-      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
-      this.checkpointMaterial = {
-        surface: aiSurfaces.surface,
-        aiOverview: aiSurfaces.aiOverview,
-        aiMode: aiSurfaces.aiMode,
-        whatPeopleSaying,
-        videos,
-        forums,
-        organicResults,
-        localPack,
-        entityIds
-      };
-      await this.emitProgress("serp_captured");
       const flat = await this.runBFS(page, executionOptions, signal);
       this.throwIfAborted(signal);
       const shortVidsParams = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
@@ -4426,20 +4563,6 @@ var PAAExtractor = class {
         }
       }
       this.reporter.onVideos(shortVideos);
-      let allOrganic = organicResults;
-      let locationEvidence = initialLocationEvidence;
-      if ((options.pages ?? 1) >= 2) {
-        const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-        if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
-        if (uule) p2params.set("uule", uule);
-        await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
-        await this.throwIfCaptcha(page, "Google SERP page 2");
-        const p2organic = await this.extractOrganicResults(page);
-        allOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
-        if (options.debug) {
-          locationEvidence = inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack);
-        }
-      }
       await this.resolveMaterialLinks(page, flat, allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
       this.checkpointMaterial = {
         ...this.checkpointMaterial,
@@ -4449,6 +4572,7 @@ var PAAExtractor = class {
         videos: [...videos, ...shortVideos],
         organicResults: allOrganic
       };
+      await this.emitProgress("expansion_finished");
       const allVideos = [...videos, ...shortVideos];
       const tree = this.buildTree(flat, executionOptions.query);
       const stats = {
@@ -4466,6 +4590,7 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          pagination,
           paaLifecycle: { ...this.paaLifecycle },
           completeness: { ...this.completeness },
           links: { ...this.linkDiagnostics },
@@ -4510,18 +4635,18 @@ var import_papaparse = __toESM(require("papaparse"), 1);
 
 // src/output/memory-library-sink.ts
 async function postToMemoryLibrary(args) {
-  const url = process.env.MCP_MEMORY_INGEST_URL;
-  const key = process.env.MCP_MEMORY_KEY;
-  if (!url || !key) return;
+  const key = process.env.MCP_SCRAPER_API_KEY?.trim();
+  const baseUrl = (process.env.MCP_SCRAPER_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
+  if (!key) return;
   try {
-    const res = await fetch(url, {
+    const res = await fetch(`${baseUrl}/memory/mcp-call`, {
       method: "POST",
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify({ data: { apiKey: key, title: args.title, content: args.content, source: args.source } })
+      headers: { "content-type": "application/json", "x-api-key": key },
+      body: JSON.stringify({ toolName: "libraryIngestTool", args })
     });
     const body = await res.json().catch(() => null);
-    if (!res.ok || body?.result?.error) {
-      console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.result?.message ?? "");
+    if (!res.ok || body?.ok === false) {
+      console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.error ?? "");
     }
   } catch (err) {
     console.warn("[memory-library-sink] ingest failed:", err?.message);

package/dist/index.d.cts

@@ -177,9 +177,19 @@ interface PaaInteractionObservation {
     errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
 }
 type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
+/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
+interface HarvestPaginationDiagnostics {
+    requestedPages: 1 | 2;
+    capturedPages: 1 | 2;
+    page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
+    page1OrganicCount: number;
+    page2OrganicCount: number;
+    failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
+}
 /** Material already observed before PAA expansion completes. It is checkpointed
  * so a worker interruption cannot discard the SERP that led to the PAA graph. */
 interface PaaProgressMaterial {
+    pagination?: HarvestPaginationDiagnostics;
     surface: GoogleSurface;
     aiOverview: AIOverviewResult;
     aiMode: AIModeResult;
@@ -201,6 +211,7 @@ interface PaaProgressSnapshot {
     interactions?: PaaInteractionObservation[];
 }
 interface HarvestDiagnostics {
+    pagination?: HarvestPaginationDiagnostics;
     completionStatus: HarvestCompletionStatus;
     /** Set only when the extractor directly observed a readable SERP without a PAA block. */
     noPaaObserved?: boolean;

package/dist/index.d.ts

@@ -177,9 +177,19 @@ interface PaaInteractionObservation {
     errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
 }
 type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
+/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
+interface HarvestPaginationDiagnostics {
+    requestedPages: 1 | 2;
+    capturedPages: 1 | 2;
+    page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
+    page1OrganicCount: number;
+    page2OrganicCount: number;
+    failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
+}
 /** Material already observed before PAA expansion completes. It is checkpointed
  * so a worker interruption cannot discard the SERP that led to the PAA graph. */
 interface PaaProgressMaterial {
+    pagination?: HarvestPaginationDiagnostics;
     surface: GoogleSurface;
     aiOverview: AIOverviewResult;
     aiMode: AIModeResult;
@@ -201,6 +211,7 @@ interface PaaProgressSnapshot {
     interactions?: PaaInteractionObservation[];
 }
 interface HarvestDiagnostics {
+    pagination?: HarvestPaginationDiagnostics;
     completionStatus: HarvestCompletionStatus;
     /** Set only when the extractor directly observed a readable SERP without a PAA block. */
     noPaaObserved?: boolean;

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-NH4QAH3X.js";
+} from "./chunk-S24LFPL7.js";
 import "./chunk-OM7HVEJ3.js";
 import {
   recordVendorUsage
 } from "./chunk-OPQIGAFB.js";
-import "./chunk-OWF2JJKN.js";
+import "./chunk-GXBZXWXB.js";
 import "./chunk-YXNDOQXN.js";
 
 // src/video/VideoGenerator.ts