mcp-scraper 0.86.5 → 0.87.0

package/dist/{chunk-QJJQ4OAZ.js → chunk-XX6XUZQM.js}

@@ -50,7 +50,7 @@ import {
 } from "./chunk-P7FWOMU7.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-BA6BPDSP.js";
+} from "./chunk-Y3AIFO7C.js";
 import {
   PUBLIC_ERROR_CODES,
   buildPublicErrorEnvelope,
@@ -1126,6 +1126,28 @@ function errorAttemptsSection(body) {
 Attempts:
 ${lines.join("\n")}`;
 }
+function publicHarvestPagination(value) {
+  const row = structuredRecord(value);
+  const statuses = ["not_requested", "not_attempted", "captured", "unavailable", "failed"];
+  const codes = ["missing_next", "invalid_next", "empty_page", "captcha", "timeout", "navigation_error", "unsupported_driver"];
+  if (row.requestedPages !== 1 && row.requestedPages !== 2 || row.capturedPages !== 1 && row.capturedPages !== 2 || !statuses.includes(row.page2Status) || !Number.isInteger(row.page1OrganicCount) || Number(row.page1OrganicCount) < 0 || !Number.isInteger(row.page2OrganicCount) || Number(row.page2OrganicCount) < 0) return null;
+  const captured = row.page2Status === "captured";
+  if (captured !== (row.capturedPages === 2) || captured && (row.requestedPages !== 2 || Number(row.page2OrganicCount) === 0) || !captured && row.page2OrganicCount !== 0 || row.requestedPages === 1 && row.page2Status !== "not_requested" || row.requestedPages === 2 && row.page2Status === "not_requested" || row.page2Status === "not_attempted" && row.failureCode !== void 0) return null;
+  return {
+    requestedPages: row.requestedPages,
+    capturedPages: row.capturedPages,
+    page2Status: row.page2Status,
+    page1OrganicCount: Number(row.page1OrganicCount),
+    page2OrganicCount: Number(row.page2OrganicCount),
+    ...codes.includes(row.failureCode) ? { failureCode: row.failureCode } : {}
+  };
+}
+function harvestPaginationText(pagination) {
+  if (!pagination) return "";
+  const outcome = pagination.page2Status === "captured" ? "Page 2 organic results captured; PAA harvested on the original first page." : pagination.requestedPages === 2 ? `Page 2 ${pagination.page2Status.replaceAll("_", " ")}${pagination.failureCode ? ` (${pagination.failureCode})` : ""}; first-page evidence retained.` : "PAA harvested on the first page.";
+  return `
+**Pagination:** ${pagination.capturedPages} of ${pagination.requestedPages} requested organic pages captured. ${outcome}`;
+}
 function publicPaaLifecycle(value) {
   if (!value || typeof value !== "object" || Array.isArray(value)) return null;
   const row = value;
@@ -1285,6 +1307,7 @@ function formatHarvestPaa(raw, input) {
   const aiOvw = d.aiOverview;
   const diagnostics = d.diagnostics;
   const durationMs = d.stats?.durationMs;
+  const pagination = publicHarvestPagination(diagnostics?.pagination);
   const paaRows = flat.map(
     (r, i) => `| ${i + 1} | ${cell(r.question)} | ${cell(truncate(r.answer, 120))} | ${cell(r.source_title || r.source_site || "")} |`
   ).join("\n");
@@ -1317,7 +1340,7 @@ ${serpRows}` : "";
 - Dig into a result: use \`extract_url\` on any organic URL`;
   const full = `# PAA Report: "${input.query}"${input.location ? ` \xB7 ${input.location}` : ""}
 
-${paaTable}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${debugSection(diagnostics?.debug)}${tips}`;
+${paaTable}${harvestPaginationText(pagination)}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${debugSection(diagnostics?.debug)}${tips}`;
   return {
     ...oneBlock(full),
     structuredContent: {
@@ -1330,6 +1353,7 @@ ${paaTable}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${d
       degradationReasons: diagnostics?.degradationReasons ?? [],
       retryRecommended: diagnostics?.retryRecommended ?? null,
       paaLifecycle: publicPaaLifecycle(diagnostics?.paaLifecycle),
+      pagination,
       questions: flat.map((r) => publicPaaQuestion(r)),
       organicResults: organic.map((r) => publicOrganicResult(r)),
       aiOverview: publicAiOverview(aiOvw),
@@ -1390,6 +1414,7 @@ function durablePaaProgress(result, options) {
     );
   }
   return {
+    pagination: publicHarvestPagination(diagnostics.pagination ?? structuredRecord(rawProgress.material).pagination),
     requestedQuestions: finiteNonNegative(completeness.requestedQuestions ?? options.maxQuestions),
     capturedQuestions: captured,
     answeredQuestions: answered,
@@ -1410,6 +1435,7 @@ function durablePaaResult(result) {
     resultQuality: nullableBoundedString(diagnostics.resultQuality ?? result.resultQuality),
     retryRecommended: typeof diagnostics.retryRecommended === "boolean" ? diagnostics.retryRecommended : typeof result.retryRecommended === "boolean" ? result.retryRecommended : null,
     paaLifecycle: publicPaaLifecycle(diagnostics.paaLifecycle ?? structuredRecord(result.progress).lifecycle),
+    pagination: publicHarvestPagination(diagnostics.pagination ?? structuredRecord(structuredRecord(result.progress).material).pagination),
     questionCount: rows.length,
     questions: rows.map(publicPaaQuestion),
     organicResults: Array.isArray(result.organicResults) ? result.organicResults.filter((row) => row && typeof row === "object").map((row) => publicOrganicResult(row)) : [],
@@ -1521,7 +1547,7 @@ function formatHarvestPaaStatus(raw) {
   return {
     content: [{
       type: "text",
-      text: recovering ? `PAA job ${jobId}: recovering automatically after an interrupted worker. ${progress.capturedQuestions ?? 0} questions are preserved; keep polling this same job.` : `PAA job ${jobId}: ${state}. ${progress.capturedQuestions ?? 0} questions are preserved.`
+      text: (recovering ? `PAA job ${jobId}: recovering automatically after an interrupted worker. ${progress.capturedQuestions ?? 0} questions are preserved; keep polling this same job.` : `PAA job ${jobId}: ${state}. ${progress.capturedQuestions ?? 0} questions are preserved.`) + harvestPaginationText(publicHarvestPagination(structuredRecord(result.diagnostics).pagination ?? structuredRecord(structuredRecord(result.progress).material).pagination))
     }],
     structuredContent
   };
@@ -8725,6 +8751,7 @@ var WebsiteUrlOrDomainSchema = z7.string().trim().min(1).transform((raw, ctx) =>
   }
 });
 var HarvestPaaInputSchema = {
+  pages: z7.number().int().min(1).max(2).default(1).describe("Organic result pages to capture. Default 1, maximum 2. Page 2 is captured when available before harvesting PAA on the original first page; it does not add a second PAA graph. Pagination output reports the pages actually captured."),
   query: z7.string().min(1).describe('The search topic, exactly as it should be searched, e.g. "best hvac company in Denver". Include the place here when you want it in the search terms \u2014 the server sends your query to Google unchanged and never adds or removes a location.'),
   location: z7.string().optional().describe('Where Google should think the searcher is, e.g. "Denver, CO". Sets the Google UULE parameter only \u2014 it never changes your query text and never selects a proxy. To put the place in the search terms too, write it into query.'),
   maxQuestions: z7.number().int().min(1).max(200).default(30).describe("PAA questions to extract. Default 30, maximum 200. Use 10 for quick probes, 100-200 for deep research. Billed per extracted question; unused hold refunded."),
@@ -10019,6 +10046,14 @@ var PaaInteractionOutput = z7.object({
   sourceCount: z7.number().int().min(0),
   errorCode: z7.enum(["click_failed", "click_ack_timeout", "control_missing", "confirmation_timeout"]).nullable()
 });
+var HarvestPaginationOutput = z7.object({
+  requestedPages: z7.union([z7.literal(1), z7.literal(2)]),
+  capturedPages: z7.union([z7.literal(1), z7.literal(2)]),
+  page2Status: z7.enum(["not_requested", "not_attempted", "captured", "unavailable", "failed"]),
+  page1OrganicCount: z7.number().int().min(0),
+  page2OrganicCount: z7.number().int().min(0),
+  failureCode: z7.enum(["missing_next", "invalid_next", "empty_page", "captcha", "timeout", "navigation_error", "unsupported_driver"]).optional()
+}).nullable();
 var HarvestPaaOutputSchema = {
   query: z7.string(),
   location: NullableString,
@@ -10029,6 +10064,7 @@ var HarvestPaaOutputSchema = {
   degradationReasons: z7.array(z7.string()),
   retryRecommended: z7.boolean().nullable(),
   paaLifecycle: PaaLifecycleOutput,
+  pagination: HarvestPaginationOutput,
   questions: z7.array(PaaQuestionOutput),
   organicResults: z7.array(OrganicResultOutput),
   aiOverview: AiOverviewOutput,
@@ -10036,6 +10072,7 @@ var HarvestPaaOutputSchema = {
   durationMs: z7.number().min(0).nullable()
 };
 var HarvestPaaDurableProgressOutput = z7.object({
+  pagination: HarvestPaginationOutput,
   requestedQuestions: z7.number().int().min(0).nullable(),
   capturedQuestions: z7.number().int().min(0),
   answeredQuestions: z7.number().int().min(0),
@@ -10077,6 +10114,7 @@ var HarvestPaaDurableBillingOutput = z7.object({
   refundMc: z7.number().int().min(0).nullable()
 });
 var HarvestPaaDurableResultOutput = z7.object({
+  pagination: HarvestPaginationOutput,
   completionStatus: NullableString,
   resultQuality: NullableString,
   retryRecommended: z7.boolean().nullable(),
@@ -17286,7 +17324,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   registerPersonalAssistantMcpSurface(server, executor);
   server.registerTool("harvest_paa", {
     title: "Google PAA + SERP Harvest",
-    description: `Expand one Google People Also Ask SERP into questions, answers, every preserved source, AI Overview evidence, ranking URLs, and entity IDs. maxQuestions is a target count, not traversal depth. Results distinguish target_reached, proven frontier_exhausted, interruption, and recovery_exhausted; a failed click or browser timeout is never reported as exhaustion. This compatibility tool waits; use harvest_paa_start plus harvest_paa_status for long runs. Optional SERP modules require their include flags. Use gl and location for regional context. Costs ${PAA_BASE_CREDITS} Credits per harvest plus ${PAA_QUESTION_CREDITS} Credits per question actually returned; unused hold is refunded. After a timeout or unknown response, reuse the same idempotencyKey. Call credits_info for current pricing and balance.`,
+    description: `Expand one Google People Also Ask SERP into questions, answers, every preserved source, AI Overview evidence, ranking URLs, and entity IDs. Set pages to 2 to add the second organic-results page when available; PAA is still expanded once on the preserved first page, and pagination reports what was captured. maxQuestions is a target count, not traversal depth. Results distinguish target_reached, proven frontier_exhausted, interruption, and recovery_exhausted; a failed click or browser timeout is never reported as exhaustion. This compatibility tool waits; use harvest_paa_start plus harvest_paa_status for long runs. Optional SERP modules require their include flags. Use gl and location for regional context. Costs ${PAA_BASE_CREDITS} Credits per harvest plus ${PAA_QUESTION_CREDITS} Credits per question actually returned; unused hold is refunded. After a timeout or unknown response, reuse the same idempotencyKey. Call credits_info for current pricing and balance.`,
     inputSchema: harvestPaaInputSchema,
     outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
     annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
@@ -17299,7 +17337,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   });
   server.registerTool("harvest_paa_start", {
     title: "Start Durable Google PAA Harvest",
-    description: `Start a durable Google People Also Ask harvest and return its job receipt. maxQuestions is the requested target count. The job automatically resumes an interrupted serverless worker under the same jobId, idempotency key, and billing hold while preserving checkpoints. Keep one idempotencyKey after a timeout, unknown response, or in-progress reply; replaying it recovers the existing job without a duplicate charge. Poll jobId with harvest_paa_status. Costs ${PAA_BASE_CREDITS} Credits plus ${PAA_QUESTION_CREDITS} per retained question; unused hold is refunded.`,
+    description: `Start a durable Google People Also Ask harvest and return its job receipt. Set pages to 2 to add the second organic-results page when available; PAA is still expanded once on the preserved first page, and pagination survives checkpoint recovery. maxQuestions is the requested target count. The job automatically resumes an interrupted serverless worker under the same jobId, idempotency key, and billing hold while preserving checkpoints. Keep one idempotencyKey after a timeout, unknown response, or in-progress reply; replaying it recovers the existing job without a duplicate charge. Poll jobId with harvest_paa_status. Costs ${PAA_BASE_CREDITS} Credits plus ${PAA_QUESTION_CREDITS} per retained question; unused hold is refunded.`,
     inputSchema: harvestPaaStartInputSchema,
     outputSchema: recordOutputSchema("harvest_paa_start", HarvestPaaStartOutputSchema),
     annotations: { ...liveWebToolAnnotations("Start Durable Google PAA Harvest"), readOnlyHint: false, idempotentHint: true }
@@ -17309,7 +17347,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   });
   server.registerTool("harvest_paa_status", {
     title: "Check Durable Google PAA Harvest",
-    description: "Poll an owner-scoped harvest_paa_start job. Returns the job/operation/task correlation ID, saved progress, automatic-recovery count, target and discovery status, material completeness, recent per-control interaction outcomes with 0.7/1.0/1.4-second confirmation telemetry, provider-session-correlated attempts, terminal rows, and billing. frontier_exhausted means every observed eligible PAA control was processed plus three healthy no-growth confirmations; interruption never means exhaustion. Polling never starts or bills another run.",
+    description: "Poll an owner-scoped harvest_paa_start job. Returns the job/operation/task correlation ID, saved progress, automatic-recovery count, target and discovery status, material completeness, bounded organic-page pagination, recent per-control interaction outcomes with 0.7/1.0/1.4-second confirmation telemetry, provider-session-correlated attempts, terminal rows, and billing. Page-two outcomes describe organic results only; the PAA graph always comes from the preserved first page. frontier_exhausted means every observed eligible PAA control was processed plus three healthy no-growth confirmations; interruption never means exhaustion. Polling never starts or bills another run.",
     inputSchema: HarvestPaaStatusInputSchema,
     outputSchema: recordOutputSchema("harvest_paa_status", HarvestPaaStatusOutputSchema),
     annotations: { ...liveWebToolAnnotations("Check Durable Google PAA Harvest"), idempotentHint: true, openWorldHint: false }

package/dist/{chunk-BA6BPDSP.js → chunk-Y3AIFO7C.js}

@@ -1,5 +1,5 @@
 // src/version.ts
-var PACKAGE_VERSION = "0.86.5";
+var PACKAGE_VERSION = "0.87.0";
 
 export {
   PACKAGE_VERSION

package/dist/index.cjs

@@ -395,6 +395,58 @@ var LocationMismatchError = class extends Error {
   }
 };
 
+// src/driver/IBrowserDriver.ts
+async function withManagedTemporaryPage(context, setup, operation, signal) {
+  let page;
+  let ended = false;
+  let timer;
+  let rejectStop = () => {
+  };
+  const aborted = () => signal?.reason instanceof DOMException && signal.reason.name === "TimeoutError" ? signal.reason : new RequestAbortedError();
+  const onAbort = () => rejectStop(aborted());
+  const stop = new Promise((_, reject) => {
+    rejectStop = reject;
+  });
+  const close = async (target) => {
+    let closeTimer;
+    try {
+      await Promise.race([
+        target.close().catch(() => {
+        }),
+        new Promise((resolve) => {
+          closeTimer = setTimeout(resolve, 2e3);
+        })
+      ]);
+    } finally {
+      if (closeTimer) clearTimeout(closeTimer);
+    }
+  };
+  if (signal?.aborted) throw aborted();
+  signal?.addEventListener("abort", onAbort, { once: true });
+  timer = setTimeout(() => rejectStop(new DOMException("Optional pagination timed out", "TimeoutError")), 3e4);
+  try {
+    return await Promise.race([
+      (async () => {
+        const created = await context.newPage();
+        if (ended) {
+          await close(created);
+          throw new RequestAbortedError();
+        }
+        page = created;
+        await setup(created);
+        if (ended) throw new RequestAbortedError();
+        return operation(created);
+      })(),
+      stop
+    ]);
+  } finally {
+    ended = true;
+    if (timer) clearTimeout(timer);
+    signal?.removeEventListener("abort", onAbort);
+    if (page) await close(page);
+  }
+}
+
 // src/api/cost-telemetry.ts
 var import_node_crypto2 = require("crypto");
 
@@ -1085,6 +1137,11 @@ var BrowserDriver = class {
       };
     });
   }
+  async withTemporaryPage(operation, signal) {
+    if (!this.context) throw new Error("Browser context is not available");
+    return withManagedTemporaryPage(this.context, async () => {
+    }, operation, signal);
+  }
   async navigateToSERP(query, uule, gl, hl, options) {
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
     if (options?.tbs) params.set("tbs", options.tbs);
@@ -1489,7 +1546,10 @@ var BrightDataSerpDriver = class {
     const customCdp = this.cdpSession;
     const sessionResponse = await customCdp.send("Browser.getSessionId");
     this.providerSessionId = validateProviderSessionId(sessionResponse?.sessionId);
-    await this.page.route("**/*", async (route) => {
+    await this.setupPage(this.page);
+  }
+  async setupPage(page) {
+    await page.route("**/*", async (route) => {
       const request = route.request();
       const resourceType = request.resourceType();
       let requestUrl;
@@ -1514,6 +1574,10 @@ var BrightDataSerpDriver = class {
       await route.continue();
     });
   }
+  async withTemporaryPage(operation, signal) {
+    if (!this.context || this.closing) throw new Error("Browser context is not available");
+    return withManagedTemporaryPage(this.context, (page) => this.setupPage(page), operation, signal);
+  }
   async navigateToSERP(query, uule, gl, hl, options) {
     if (!this.page) throw new Error("Browser page is not available");
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
@@ -2489,6 +2553,11 @@ function parsedGoogleGoto(value) {
   if (!url || url.protocol !== "https:" || !isGoogleHost(url.hostname) || url.pathname !== "/goto") return null;
   return url.searchParams.get("url") ? url : null;
 }
+function hasResolvedGotoDestination(value) {
+  if (value.linkType !== "google_goto_redirect" || value.resolutionStatus !== "resolved" || !parsedGoogleGoto(value.rawUrl) || !value.resolvedUrl) return false;
+  const destination = safeHttpUrl(value.resolvedUrl);
+  return Boolean(destination && !isGoogleHost(destination.hostname) && destination.href === value.resolvedUrl && value.url === destination.href);
+}
 function questionIdFor(question) {
   const normalized = question.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu, "").replace(/\s+/g, " ").trim();
   return `paa_${(0, import_node_crypto3.createHash)("sha256").update(normalized).digest("hex").slice(0, 16)}`;
@@ -2705,8 +2774,12 @@ async function resolveHarvestResultGoogleGotoLinks(result, options = {}) {
 }
 async function resolveGoogleOutboundLinks(page, values, options = {}) {
   const byRawUrl = /* @__PURE__ */ new Map();
-  for (const value of values) byRawUrl.set(value.rawUrl, value);
-  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect");
+  for (const value of values) {
+    const existing = byRawUrl.get(value.rawUrl);
+    if (existing && hasResolvedGotoDestination(existing)) continue;
+    byRawUrl.set(value.rawUrl, value.linkType === "google_goto_redirect" && !hasResolvedGotoDestination(value) ? { ...value, url: value.rawUrl, resolvedUrl: null, resolutionStatus: "unresolved" } : value);
+  }
+  const inputs = [...byRawUrl.values()].filter((value) => value.linkType === "google_goto_redirect" && !hasResolvedGotoDestination(value));
   if (inputs.length === 0) return byRawUrl;
   const deadlineMs = Date.now() + (options.totalBudgetMs ?? DEFAULT_TOTAL_BUDGET_MS);
   const direct = await resolveGoogleGotoUrls(inputs.map((value) => value.rawUrl), options);
@@ -2922,7 +2995,7 @@ var PAAExtractor = class {
     ];
     const unique = new Map(candidates.map((link) => [link.rawUrl, link]));
     let resolverNonGoogleRequestsObserved = 0;
-    const resolved = await resolveGoogleOutboundLinks(page, unique.values(), {
+    const resolved = await resolveGoogleOutboundLinks(page, candidates, {
       onNetworkAudit: (audit) => {
         resolverNonGoogleRequestsObserved = audit.nonGoogleRequestsObserved;
       }
@@ -3173,6 +3246,7 @@ var PAAExtractor = class {
       diagnostics: {
         completionStatus: "paa_found",
         problem: null,
+        ...material?.pagination ? { pagination: { ...material.pagination } } : {},
         resultQuality: "partial",
         degradedResult: false,
         retryRecommended: true,
@@ -4137,6 +4211,55 @@ var PAAExtractor = class {
       ...locationEvidence ? { locationEvidence } : {}
     };
   }
+  async captureSecondOrganicPage(page, options, signal) {
+    this.throwIfAborted(signal);
+    if (!this.driver.withTemporaryPage) return { organic: [], status: "failed", failureCode: "unsupported_driver" };
+    let guardedFailure;
+    try {
+      const initial = new URL(page.url());
+      const valid = (raw) => {
+        try {
+          const url = new URL(raw, initial);
+          return initial.origin === "https://www.google.com" && url.origin === initial.origin && !url.username && !url.password && url.pathname === "/search" && url.searchParams.getAll("q").length === 1 && url.searchParams.get("q") === options.query && url.searchParams.getAll("start").length === 1 && url.searchParams.get("start") === "10" && (!url.searchParams.has("num") || url.searchParams.get("num") === "10") && ["gl", "hl"].every((key) => !url.searchParams.has(key) || url.searchParams.getAll(key).length === 1 && url.searchParams.get(key) === options[key]) && ["uule", "tbs", "udm", "tbm", "pws"].every((key) => url.searchParams.getAll(key).length <= 1 && url.searchParams.get(key) === initial.searchParams.get(key));
+        } catch {
+          return false;
+        }
+      };
+      const href = await page.evaluate(() => {
+        const next = document.querySelector('a#pnnext, a[rel="next"]');
+        return next?.getAttribute("href") ?? null;
+      });
+      if (!href) return { organic: [], status: "unavailable", failureCode: "missing_next" };
+      if (!valid(href)) return { organic: [], status: "unavailable", failureCode: "invalid_next" };
+      const organic = await this.driver.withTemporaryPage(async (second) => {
+        await second.route("**/*", async (route) => {
+          const request = route.request();
+          if (request.isNavigationRequest() && request.frame() === second.mainFrame() && !valid(request.url())) {
+            guardedFailure = /^https:\/\/www\.google\.com\/sorry(?:\/|\?)/.test(request.url()) ? "captcha" : "invalid_next";
+            await route.abort("blockedbyclient");
+          } else await route.fallback();
+        });
+        await second.goto(new URL(href, initial).href, { waitUntil: "domcontentloaded", timeout: 2e4 });
+        this.throwIfAborted(signal);
+        await this.throwIfCaptcha(second, "Google SERP page 2");
+        if (!valid(second.url())) {
+          guardedFailure = "invalid_next";
+          throw new Error("Invalid pagination landing");
+        }
+        return this.extractOrganicResults(second);
+      }, signal);
+      this.throwIfAborted(signal);
+      return organic.length > 0 ? { organic: organic.map((row) => ({ ...row, position: row.position + 10 })), status: "captured" } : { organic: [], status: "unavailable", failureCode: "empty_page" };
+    } catch (err) {
+      this.throwIfAborted(signal);
+      if (err instanceof RequestAbortedError) throw err;
+      return {
+        organic: [],
+        status: "failed",
+        failureCode: guardedFailure ?? (err instanceof CaptchaError ? "captcha" : err instanceof Error && (err.name === "TimeoutError" || /timed? ?out/i.test(err.message)) ? "timeout" : "navigation_error")
+      };
+    }
+  }
   async extract(options, signal) {
     const startMs = Date.now();
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
@@ -4340,23 +4463,49 @@ var PAAExtractor = class {
       const initialLocationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults, localPack) : void 0;
       this.reporter.onVideos(videos);
       this.reporter.onForums(forums);
+      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
+      let pagination = {
+        requestedPages: (options.pages ?? 1) >= 2 ? 2 : 1,
+        capturedPages: 1,
+        page2Status: (options.pages ?? 1) >= 2 ? "not_attempted" : "not_requested",
+        page1OrganicCount: organicResults.length,
+        page2OrganicCount: 0
+      };
+      let allOrganic = organicResults;
+      this.checkpointMaterial = {
+        surface: aiSurfaces.surface,
+        aiOverview: aiSurfaces.aiOverview,
+        aiMode: aiSurfaces.aiMode,
+        whatPeopleSaying,
+        videos,
+        forums,
+        organicResults: allOrganic,
+        localPack,
+        entityIds,
+        pagination
+      };
+      await this.emitProgress("serp_captured");
+      if (pagination.requestedPages === 2) {
+        const second = await this.captureSecondOrganicPage(page, executionOptions, signal);
+        allOrganic = [...organicResults, ...second.organic];
+        pagination = {
+          ...pagination,
+          capturedPages: second.status === "captured" ? 2 : 1,
+          page2Status: second.status,
+          page2OrganicCount: second.organic.length,
+          ...second.failureCode ? { failureCode: second.failureCode } : {}
+        };
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic, pagination };
+        await this.emitProgress("serp_captured");
+        await this.resolveMaterialLinks(page, [], allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic };
+        await this.emitProgress("serp_captured");
+      }
+      const locationEvidence = options.debug ? inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack) : initialLocationEvidence;
       if (!hasPaa) {
-        let noPaaOrganic = organicResults;
-        let locationEvidence2 = initialLocationEvidence;
-        if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
-          if (uule) p2params.set("uule", uule);
-          await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
-          await this.throwIfCaptcha(page, "Google SERP page 2");
-          const p2organic = await this.extractOrganicResults(page);
-          noPaaOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
-          if (options.debug) {
-            locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
-          }
-        }
-        const aiSurfaces2 = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
-        await this.resolveMaterialLinks(page, [], noPaaOrganic, aiSurfaces2.aiOverview, aiSurfaces2.aiMode);
+        await this.resolveMaterialLinks(page, [], allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
+        this.checkpointMaterial = { ...this.checkpointMaterial, organicResults: allOrganic };
+        await this.emitProgress("serp_captured");
         const stats2 = {
           seed: executionOptions.query,
           totalQuestions: 0,
@@ -4373,39 +4522,27 @@ var PAAExtractor = class {
             completionStatus: "no_paa",
             noPaaObserved: true,
             problem: null,
+            pagination,
             paaLifecycle: { ...this.paaLifecycle },
             completeness: { ...this.completeness },
             links: { ...this.linkDiagnostics },
-            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2, locationResolution) } : {}
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence, locationResolution) } : {}
           },
           totalQuestions: 0,
-          surface: aiSurfaces2.surface,
-          aiOverview: aiSurfaces2.aiOverview,
-          aiMode: aiSurfaces2.aiMode,
+          surface: aiSurfaces.surface,
+          aiOverview: aiSurfaces.aiOverview,
+          aiMode: aiSurfaces.aiMode,
           whatPeopleSaying,
           tree: [],
           flat: [],
           videos,
           forums,
-          organicResults: noPaaOrganic,
+          organicResults: allOrganic,
           localPack,
           entityIds,
           stats: stats2
         };
       }
-      const aiSurfaces = includeAiOverview ? await this.extractAISurfaces(page, options) : emptyAiSurfaces;
-      this.checkpointMaterial = {
-        surface: aiSurfaces.surface,
-        aiOverview: aiSurfaces.aiOverview,
-        aiMode: aiSurfaces.aiMode,
-        whatPeopleSaying,
-        videos,
-        forums,
-        organicResults,
-        localPack,
-        entityIds
-      };
-      await this.emitProgress("serp_captured");
       const flat = await this.runBFS(page, executionOptions, signal);
       this.throwIfAborted(signal);
       const shortVidsParams = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
@@ -4426,20 +4563,6 @@ var PAAExtractor = class {
         }
       }
       this.reporter.onVideos(shortVideos);
-      let allOrganic = organicResults;
-      let locationEvidence = initialLocationEvidence;
-      if ((options.pages ?? 1) >= 2) {
-        const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-        if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
-        if (uule) p2params.set("uule", uule);
-        await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
-        await this.throwIfCaptcha(page, "Google SERP page 2");
-        const p2organic = await this.extractOrganicResults(page);
-        allOrganic = [...organicResults, ...p2organic.map((r) => ({ ...r, position: r.position + 10 }))];
-        if (options.debug) {
-          locationEvidence = inferSerpLocationEvidence(canonicalLocation, allOrganic, localPack);
-        }
-      }
       await this.resolveMaterialLinks(page, flat, allOrganic, aiSurfaces.aiOverview, aiSurfaces.aiMode);
       this.checkpointMaterial = {
         ...this.checkpointMaterial,
@@ -4449,6 +4572,7 @@ var PAAExtractor = class {
         videos: [...videos, ...shortVideos],
         organicResults: allOrganic
       };
+      await this.emitProgress("expansion_finished");
       const allVideos = [...videos, ...shortVideos];
       const tree = this.buildTree(flat, executionOptions.query);
       const stats = {
@@ -4466,6 +4590,7 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          pagination,
           paaLifecycle: { ...this.paaLifecycle },
           completeness: { ...this.completeness },
           links: { ...this.linkDiagnostics },

package/dist/index.d.cts

@@ -177,9 +177,19 @@ interface PaaInteractionObservation {
     errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
 }
 type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
+/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
+interface HarvestPaginationDiagnostics {
+    requestedPages: 1 | 2;
+    capturedPages: 1 | 2;
+    page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
+    page1OrganicCount: number;
+    page2OrganicCount: number;
+    failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
+}
 /** Material already observed before PAA expansion completes. It is checkpointed
  * so a worker interruption cannot discard the SERP that led to the PAA graph. */
 interface PaaProgressMaterial {
+    pagination?: HarvestPaginationDiagnostics;
     surface: GoogleSurface;
     aiOverview: AIOverviewResult;
     aiMode: AIModeResult;
@@ -201,6 +211,7 @@ interface PaaProgressSnapshot {
     interactions?: PaaInteractionObservation[];
 }
 interface HarvestDiagnostics {
+    pagination?: HarvestPaginationDiagnostics;
     completionStatus: HarvestCompletionStatus;
     /** Set only when the extractor directly observed a readable SERP without a PAA block. */
     noPaaObserved?: boolean;

package/dist/index.d.ts

@@ -177,9 +177,19 @@ interface PaaInteractionObservation {
     errorCode: 'click_failed' | 'click_ack_timeout' | 'control_missing' | 'confirmation_timeout' | null;
 }
 type PaaProgressPhase = 'serp_captured' | 'discovered' | 'click_observed' | 'answer_captured' | 'fill_incomplete' | 'expansion_finished' | 'attempt_ending';
+/** Observed organic pagination outcome; page two never expands a separate PAA graph. */
+interface HarvestPaginationDiagnostics {
+    requestedPages: 1 | 2;
+    capturedPages: 1 | 2;
+    page2Status: 'not_requested' | 'not_attempted' | 'captured' | 'unavailable' | 'failed';
+    page1OrganicCount: number;
+    page2OrganicCount: number;
+    failureCode?: 'missing_next' | 'invalid_next' | 'empty_page' | 'captcha' | 'timeout' | 'navigation_error' | 'unsupported_driver';
+}
 /** Material already observed before PAA expansion completes. It is checkpointed
  * so a worker interruption cannot discard the SERP that led to the PAA graph. */
 interface PaaProgressMaterial {
+    pagination?: HarvestPaginationDiagnostics;
     surface: GoogleSurface;
     aiOverview: AIOverviewResult;
     aiMode: AIModeResult;
@@ -201,6 +211,7 @@ interface PaaProgressSnapshot {
     interactions?: PaaInteractionObservation[];
 }
 interface HarvestDiagnostics {
+    pagination?: HarvestPaginationDiagnostics;
     completionStatus: HarvestCompletionStatus;
     /** Set only when the extractor directly observed a readable SERP without a PAA block. */
     noPaaObserved?: boolean;

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   harvest
-} from "./chunk-NH4QAH3X.js";
+} from "./chunk-R2HRN2FW.js";
 import "./chunk-OM7HVEJ3.js";
 import {
   recordVendorUsage