mcp-scraper 0.86.5 → 0.87.0

package/dist/{server-ARMF2IRI.js → server-4QWARJET.js}

@@ -240,7 +240,7 @@ import {
   serpArrivalUrl,
   validateProviderSessionId,
   withRuntimeUserAgent
-} from "./chunk-NH4QAH3X.js";
+} from "./chunk-R2HRN2FW.js";
 import {
   ANALYTICS_REST_GENERIC_EXEMPTIONS,
   ASSISTANT_COMMAND_SCHEMA_VERSION,
@@ -388,7 +388,7 @@ import {
   transcribeMediaUrl,
   transitionAnalyticsExperiment,
   validateAnalyticsRestResponseForRequest
-} from "./chunk-QJJQ4OAZ.js";
+} from "./chunk-XX6XUZQM.js";
 import {
   auditImageUrls,
   auditImages,
@@ -509,7 +509,7 @@ import {
 } from "./chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-BA6BPDSP.js";
+} from "./chunk-Y3AIFO7C.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -26992,6 +26992,40 @@ function flatFromRecord(record7, seedQuery) {
     extracted_at: record7.observedAt
   };
 }
+function mergePaaProgressMaterial(existing, incoming) {
+  if (!existing) return structuredClone(incoming);
+  const merged = structuredClone(incoming);
+  const keepOrganic = existing.pagination?.page2Status === "captured" && incoming.pagination?.page2Status !== "captured" || existing.pagination?.page2Status === incoming.pagination?.page2Status && existing.organicResults.length > incoming.organicResults.length || existing.organicResults.length > 0 && incoming.organicResults.length === 0;
+  if (keepOrganic) {
+    merged.organicResults = structuredClone(existing.organicResults);
+    merged.pagination = existing.pagination ? structuredClone(existing.pagination) : incoming.pagination;
+  }
+  const resolvedLinks = /* @__PURE__ */ new Map();
+  for (const row of [...existing.organicResults, ...incoming.organicResults]) {
+    if (row.resolutionStatus !== "resolved" || !row.resolvedUrl) continue;
+    for (const identity of [row.rawUrl, row.url, row.resolvedUrl]) {
+      if (identity) resolvedLinks.set(identity, row);
+    }
+  }
+  merged.organicResults = merged.organicResults.map((row) => {
+    if (row.resolutionStatus !== "unresolved" && row.resolutionStatus !== void 0) return row;
+    const resolved = [row.rawUrl, row.url].filter(Boolean).map((identity) => resolvedLinks.get(identity)).find(Boolean);
+    if (!resolved) return row;
+    return {
+      ...row,
+      url: resolved.url,
+      resolvedUrl: resolved.resolvedUrl,
+      resolutionStatus: resolved.resolutionStatus,
+      domain: resolved.domain
+    };
+  });
+  for (const key3 of ["videos", "forums", "localPack", "whatPeopleSaying"]) {
+    if (!merged[key3].length && existing[key3].length) Object.assign(merged, { [key3]: structuredClone(existing[key3]) });
+  }
+  if (!merged.aiOverview.detected && existing.aiOverview.detected) merged.aiOverview = structuredClone(existing.aiOverview);
+  if (!merged.aiMode.detected && existing.aiMode.detected) merged.aiMode = structuredClone(existing.aiMode);
+  return merged;
+}
 var ProgressivePaaCheckpoint = class {
   constructor(jobId2, durableMetadata, persist, now = Date.now, initialResult) {
     this.jobId = jobId2;
@@ -27061,7 +27095,7 @@ var ProgressivePaaCheckpoint = class {
     this.sequence += 1;
     if (snapshot.lifecycle) this.lifecycle = { ...snapshot.lifecycle };
     if (snapshot.material) {
-      this.material = structuredClone(snapshot.material);
+      this.material = mergePaaProgressMaterial(this.material, snapshot.material);
       this.improvementsSinceWrite += 1;
     }
     for (const question of snapshot.clickedQuestions) if (clean2(question)) this.clickedQuestions.add(clean2(question));
@@ -27166,6 +27200,7 @@ function partialTerminalResult(job, options, maxQuestions, progress, recoveryCou
       resultQuality: "partial",
       degradedResult: false,
       retryRecommended: true,
+      pagination: material?.pagination,
       paaLifecycle: {
         ...progress.lifecycle ?? {
           requestedQuestions: maxQuestions,
@@ -27267,7 +27302,7 @@ async function runPaaHarvestJob(jobId2) {
     };
     const onPaaProgress = async (snapshot) => {
       progress.accept(snapshot);
-      const force = snapshot.phase === "attempt_ending" || snapshot.phase === "click_observed" || snapshot.phase === "expansion_finished";
+      const force = snapshot.phase === "serp_captured" || snapshot.phase === "attempt_ending" || snapshot.phase === "click_observed" || snapshot.phase === "expansion_finished";
       await progress.flush(snapshot.phase === "attempt_ending" ? "failed" : "expanding", { force });
     };
     const harvestCtx = currentCostContext();
@@ -27279,6 +27314,7 @@ async function runPaaHarvestJob(jobId2) {
         gl: options.gl ?? "us",
         hl: options.hl ?? "en",
         maxQuestions,
+        pages: options.pages ?? 1,
         maxAttempts: DIRECT_PAA_MAX_ATTEMPTS,
         proxyMode: "none",
         forceManagedSerp: true,
@@ -27297,9 +27333,31 @@ async function runPaaHarvestJob(jobId2) {
       })
     );
     const harvestResult = harvested;
+    if (Array.isArray(harvested.organicResults)) {
+      progress.accept({
+        sequence: 0,
+        observedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        clickedQuestions: [],
+        records: [],
+        phase: "attempt_ending",
+        material: {
+          surface: harvested.surface ?? "unknown",
+          organicResults: harvested.organicResults,
+          videos: harvested.videos ?? [],
+          forums: harvested.forums ?? [],
+          localPack: harvested.localPack ?? [],
+          whatPeopleSaying: harvested.whatPeopleSaying ?? [],
+          aiOverview: harvested.aiOverview ?? { detected: false, text: null, citations: [] },
+          aiMode: harvested.aiMode ?? { detected: false, text: null, citations: [] },
+          entityIds: harvested.entityIds ?? { entities: [], kgIds: [], cids: [], gcids: [] },
+          pagination: harvested.diagnostics?.pagination
+        }
+      });
+    }
     progress.mergeTerminalRows(harvestResult.flat, lastAttempt, (/* @__PURE__ */ new Date()).toISOString());
     await progress.flush("captured", { force: true });
     const terminalProgress = progress.checkpoint("captured");
+    const { pagination: terminalPagination, ...terminalMaterial } = terminalProgress.material ?? {};
     const mergedFlat = terminalProgress.records.map((record7) => flatFromRecord(record7, options.query ?? job.query));
     const rawDiagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : {};
     const rawCompleteness = rawDiagnostics.completeness && typeof rawDiagnostics.completeness === "object" ? rawDiagnostics.completeness : {};
@@ -27312,11 +27370,13 @@ async function runPaaHarvestJob(jobId2) {
     const automaticRetries = recoveryCount + terminalProgress.interruptions.filter((value) => value.willRetry).length;
     const result3 = {
       ...harvestResult,
+      ...terminalMaterial,
       totalQuestions: mergedFlat.length,
       flat: mergedFlat,
       stats: harvestResult.stats && typeof harvestResult.stats === "object" ? { ...harvestResult.stats, totalQuestions: mergedFlat.length } : harvestResult.stats,
       diagnostics: {
         ...rawDiagnostics,
+        pagination: terminalPagination ?? rawDiagnostics.pagination,
         resultQuality: incomplete ? "partial" : "complete",
         degradedResult: false,
         retryRecommended: discoveryStatus === "interrupted" || discoveryStatus === "recovery_exhausted",
@@ -27446,6 +27506,34 @@ function stalePartialResult(result3, records, seedQuery, recoveryCount) {
   const progress = current.progress && typeof current.progress === "object" ? current.progress : {};
   const material = progress.material && typeof progress.material === "object" ? progress.material : {};
   const diagnostics = current.diagnostics && typeof current.diagnostics === "object" ? current.diagnostics : {};
+  const currentPagination = diagnostics.pagination && typeof diagnostics.pagination === "object" ? diagnostics.pagination : {};
+  const materialPagination = material.pagination && typeof material.pagination === "object" ? material.pagination : {};
+  const currentOrganic = Array.isArray(current.organicResults) ? current.organicResults : [];
+  const materialOrganic = Array.isArray(material.organicResults) ? material.organicResults : [];
+  const capturesPageTwo = (pagination, rows) => pagination.requestedPages === 2 && pagination.capturedPages === 2 && pagination.page2Status === "captured" && typeof pagination.page1OrganicCount === "number" && typeof pagination.page2OrganicCount === "number" && pagination.page2OrganicCount > 0 && rows.length >= pagination.page1OrganicCount + pagination.page2OrganicCount;
+  const currentCaptured = capturesPageTwo(currentPagination, currentOrganic);
+  const materialCaptured = capturesPageTwo(materialPagination, materialOrganic);
+  const keepCurrentOrganic = currentCaptured && !materialCaptured || currentCaptured === materialCaptured && currentOrganic.length > materialOrganic.length;
+  const selectedOrganic = keepCurrentOrganic ? currentOrganic : materialOrganic.length ? materialOrganic : currentOrganic;
+  const fallbackOrganic = keepCurrentOrganic ? materialOrganic : currentOrganic;
+  const resolvedByIdentity = /* @__PURE__ */ new Map();
+  for (const row of [...fallbackOrganic, ...selectedOrganic]) {
+    if (row.resolutionStatus !== "resolved" || typeof row.resolvedUrl !== "string" || !row.resolvedUrl) continue;
+    for (const identity of [row.rawUrl, row.url, row.resolvedUrl]) {
+      if (typeof identity === "string" && identity) resolvedByIdentity.set(identity, row);
+    }
+  }
+  const reconciledOrganic = selectedOrganic.map((row) => {
+    if (row.resolutionStatus !== "unresolved" && row.resolutionStatus !== void 0) return row;
+    const resolved = [row.rawUrl, row.url].filter((identity) => typeof identity === "string" && Boolean(identity)).map((identity) => resolvedByIdentity.get(identity)).find(Boolean);
+    return resolved ? {
+      ...row,
+      url: resolved.url,
+      resolvedUrl: resolved.resolvedUrl,
+      resolutionStatus: resolved.resolutionStatus,
+      domain: resolved.domain
+    } : row;
+  });
   const completeness = diagnostics.completeness && typeof diagnostics.completeness === "object" ? diagnostics.completeness : {};
   const flat = records.map((record7) => ({
     seed_query: typeof record7.seed_query === "string" ? record7.seed_query : seedQuery,
@@ -27470,7 +27558,7 @@ function stalePartialResult(result3, records, seedQuery, recoveryCount) {
     tree: Array.isArray(current.tree) ? current.tree : [],
     videos: Array.isArray(material.videos) ? material.videos : current.videos ?? [],
     forums: Array.isArray(material.forums) ? material.forums : current.forums ?? [],
-    organicResults: Array.isArray(material.organicResults) ? material.organicResults : current.organicResults ?? [],
+    organicResults: reconciledOrganic,
     localPack: Array.isArray(material.localPack) ? material.localPack : current.localPack ?? [],
     whatPeopleSaying: Array.isArray(material.whatPeopleSaying) ? material.whatPeopleSaying : current.whatPeopleSaying ?? [],
     aiOverview: material.aiOverview ?? current.aiOverview,
@@ -27480,6 +27568,7 @@ function stalePartialResult(result3, records, seedQuery, recoveryCount) {
     partial: true,
     diagnostics: {
       ...diagnostics,
+      pagination: keepCurrentOrganic ? diagnostics.pagination : material.pagination ?? diagnostics.pagination,
       completionStatus: "paa_found",
       problem: null,
       resultQuality: "partial",
@@ -86145,7 +86234,7 @@ app.get("/cron/tick", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const startedAt = Date.now();
-  const { drainQueue } = await import("./worker-OQSFSLNG.js");
+  const { drainQueue } = await import("./worker-DAVCZ3FP.js");
   const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
   const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
   const unfinished = [];

package/dist/{worker-OQSFSLNG.js → worker-DAVCZ3FP.js}

@@ -9,7 +9,7 @@ import {
 import "./chunk-OZJMVCDK.js";
 import {
   harvest
-} from "./chunk-NH4QAH3X.js";
+} from "./chunk-R2HRN2FW.js";
 import {
   sanitizeAttempts,
   sanitizeHarvestResult

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.86.5",
+  "version": "0.87.0",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",