mcp-scraper 0.86.4 → 0.87.0

package/dist/{server-4FREXPPR.js → server-4QWARJET.js}

@@ -240,7 +240,7 @@ import {
   serpArrivalUrl,
   validateProviderSessionId,
   withRuntimeUserAgent
-} from "./chunk-NH4QAH3X.js";
+} from "./chunk-R2HRN2FW.js";
 import {
   ANALYTICS_REST_GENERIC_EXEMPTIONS,
   ASSISTANT_COMMAND_SCHEMA_VERSION,
@@ -388,7 +388,7 @@ import {
   transcribeMediaUrl,
   transitionAnalyticsExperiment,
   validateAnalyticsRestResponseForRequest
-} from "./chunk-FHLNW4L6.js";
+} from "./chunk-XX6XUZQM.js";
 import {
   auditImageUrls,
   auditImages,
@@ -509,7 +509,7 @@ import {
 } from "./chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-CM62EJ3S.js";
+} from "./chunk-Y3AIFO7C.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -21650,8 +21650,8 @@ var BLOCKED_RESOURCE_TYPES = /* @__PURE__ */ new Set(["image", "media", "font",
 var BLOCKED_HOSTS = /* @__PURE__ */ new Set(["styles.redditmedia.com", "preview.redd.it"]);
 var REQUIRED_HOSTS = /* @__PURE__ */ new Set(["external-preview.redd.it", "googletagmanager.com"]);
 var MAX_COMMENTS = 25;
-var TELEMETRY_RETRY_DELAYS_MS = [0, 1500];
-var BRIGHT_DATA_REDDIT_ATTEMPT_MS = 14e3;
+var INLINE_TELEMETRY_TIMEOUT_MS = 500;
+var BRIGHT_DATA_REDDIT_ATTEMPT_MS = 17e3;
 var CLEANUP_TIMEOUT_MS2 = 1e3;
 function remainingTimeout2(deadlineAt, maximumMs) {
   const remaining = deadlineAt - Date.now();
@@ -21693,18 +21693,9 @@ function isAcceptedRedditThread(thread, requestedComments) {
 function hashSessionId(sessionId) {
   return sessionId ? createHash6("sha256").update(sessionId).digest("hex").slice(0, 16) : null;
 }
-async function sleep(ms) {
-  if (ms > 0) await new Promise((resolve3) => setTimeout(resolve3, ms));
-}
 async function reconcileSession(sessionId) {
   if (!sessionId) return null;
-  let latest = null;
-  for (const delayMs of TELEMETRY_RETRY_DELAYS_MS) {
-    await sleep(delayMs);
-    latest = await lookupBrightDataSession(sessionId);
-    if (latest.lookupStatus === "complete") break;
-    if (latest.lookupStatus === "unavailable_credentials" || latest.lookupStatus === "invalid_payload") break;
-  }
+  const latest = await lookupBrightDataSession(sessionId, { timeoutMs: INLINE_TELEMETRY_TIMEOUT_MS });
   await recordPendingBrightDataRedditUsage(sessionId, latest);
   return latest;
 }
@@ -27001,6 +26992,40 @@ function flatFromRecord(record7, seedQuery) {
     extracted_at: record7.observedAt
   };
 }
+function mergePaaProgressMaterial(existing, incoming) {
+  if (!existing) return structuredClone(incoming);
+  const merged = structuredClone(incoming);
+  const keepOrganic = existing.pagination?.page2Status === "captured" && incoming.pagination?.page2Status !== "captured" || existing.pagination?.page2Status === incoming.pagination?.page2Status && existing.organicResults.length > incoming.organicResults.length || existing.organicResults.length > 0 && incoming.organicResults.length === 0;
+  if (keepOrganic) {
+    merged.organicResults = structuredClone(existing.organicResults);
+    merged.pagination = existing.pagination ? structuredClone(existing.pagination) : incoming.pagination;
+  }
+  const resolvedLinks = /* @__PURE__ */ new Map();
+  for (const row of [...existing.organicResults, ...incoming.organicResults]) {
+    if (row.resolutionStatus !== "resolved" || !row.resolvedUrl) continue;
+    for (const identity of [row.rawUrl, row.url, row.resolvedUrl]) {
+      if (identity) resolvedLinks.set(identity, row);
+    }
+  }
+  merged.organicResults = merged.organicResults.map((row) => {
+    if (row.resolutionStatus !== "unresolved" && row.resolutionStatus !== void 0) return row;
+    const resolved = [row.rawUrl, row.url].filter(Boolean).map((identity) => resolvedLinks.get(identity)).find(Boolean);
+    if (!resolved) return row;
+    return {
+      ...row,
+      url: resolved.url,
+      resolvedUrl: resolved.resolvedUrl,
+      resolutionStatus: resolved.resolutionStatus,
+      domain: resolved.domain
+    };
+  });
+  for (const key3 of ["videos", "forums", "localPack", "whatPeopleSaying"]) {
+    if (!merged[key3].length && existing[key3].length) Object.assign(merged, { [key3]: structuredClone(existing[key3]) });
+  }
+  if (!merged.aiOverview.detected && existing.aiOverview.detected) merged.aiOverview = structuredClone(existing.aiOverview);
+  if (!merged.aiMode.detected && existing.aiMode.detected) merged.aiMode = structuredClone(existing.aiMode);
+  return merged;
+}
 var ProgressivePaaCheckpoint = class {
   constructor(jobId2, durableMetadata, persist, now = Date.now, initialResult) {
     this.jobId = jobId2;
@@ -27070,7 +27095,7 @@ var ProgressivePaaCheckpoint = class {
     this.sequence += 1;
     if (snapshot.lifecycle) this.lifecycle = { ...snapshot.lifecycle };
     if (snapshot.material) {
-      this.material = structuredClone(snapshot.material);
+      this.material = mergePaaProgressMaterial(this.material, snapshot.material);
       this.improvementsSinceWrite += 1;
     }
     for (const question of snapshot.clickedQuestions) if (clean2(question)) this.clickedQuestions.add(clean2(question));
@@ -27175,6 +27200,7 @@ function partialTerminalResult(job, options, maxQuestions, progress, recoveryCou
       resultQuality: "partial",
       degradedResult: false,
       retryRecommended: true,
+      pagination: material?.pagination,
       paaLifecycle: {
         ...progress.lifecycle ?? {
           requestedQuestions: maxQuestions,
@@ -27276,7 +27302,7 @@ async function runPaaHarvestJob(jobId2) {
     };
     const onPaaProgress = async (snapshot) => {
       progress.accept(snapshot);
-      const force = snapshot.phase === "attempt_ending" || snapshot.phase === "click_observed" || snapshot.phase === "expansion_finished";
+      const force = snapshot.phase === "serp_captured" || snapshot.phase === "attempt_ending" || snapshot.phase === "click_observed" || snapshot.phase === "expansion_finished";
       await progress.flush(snapshot.phase === "attempt_ending" ? "failed" : "expanding", { force });
     };
     const harvestCtx = currentCostContext();
@@ -27288,6 +27314,7 @@ async function runPaaHarvestJob(jobId2) {
         gl: options.gl ?? "us",
         hl: options.hl ?? "en",
         maxQuestions,
+        pages: options.pages ?? 1,
         maxAttempts: DIRECT_PAA_MAX_ATTEMPTS,
         proxyMode: "none",
         forceManagedSerp: true,
@@ -27306,9 +27333,31 @@ async function runPaaHarvestJob(jobId2) {
       })
     );
     const harvestResult = harvested;
+    if (Array.isArray(harvested.organicResults)) {
+      progress.accept({
+        sequence: 0,
+        observedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        clickedQuestions: [],
+        records: [],
+        phase: "attempt_ending",
+        material: {
+          surface: harvested.surface ?? "unknown",
+          organicResults: harvested.organicResults,
+          videos: harvested.videos ?? [],
+          forums: harvested.forums ?? [],
+          localPack: harvested.localPack ?? [],
+          whatPeopleSaying: harvested.whatPeopleSaying ?? [],
+          aiOverview: harvested.aiOverview ?? { detected: false, text: null, citations: [] },
+          aiMode: harvested.aiMode ?? { detected: false, text: null, citations: [] },
+          entityIds: harvested.entityIds ?? { entities: [], kgIds: [], cids: [], gcids: [] },
+          pagination: harvested.diagnostics?.pagination
+        }
+      });
+    }
     progress.mergeTerminalRows(harvestResult.flat, lastAttempt, (/* @__PURE__ */ new Date()).toISOString());
     await progress.flush("captured", { force: true });
     const terminalProgress = progress.checkpoint("captured");
+    const { pagination: terminalPagination, ...terminalMaterial } = terminalProgress.material ?? {};
     const mergedFlat = terminalProgress.records.map((record7) => flatFromRecord(record7, options.query ?? job.query));
     const rawDiagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : {};
     const rawCompleteness = rawDiagnostics.completeness && typeof rawDiagnostics.completeness === "object" ? rawDiagnostics.completeness : {};
@@ -27321,11 +27370,13 @@ async function runPaaHarvestJob(jobId2) {
     const automaticRetries = recoveryCount + terminalProgress.interruptions.filter((value) => value.willRetry).length;
     const result3 = {
       ...harvestResult,
+      ...terminalMaterial,
       totalQuestions: mergedFlat.length,
       flat: mergedFlat,
       stats: harvestResult.stats && typeof harvestResult.stats === "object" ? { ...harvestResult.stats, totalQuestions: mergedFlat.length } : harvestResult.stats,
       diagnostics: {
         ...rawDiagnostics,
+        pagination: terminalPagination ?? rawDiagnostics.pagination,
         resultQuality: incomplete ? "partial" : "complete",
         degradedResult: false,
         retryRecommended: discoveryStatus === "interrupted" || discoveryStatus === "recovery_exhausted",
@@ -27455,6 +27506,34 @@ function stalePartialResult(result3, records, seedQuery, recoveryCount) {
   const progress = current.progress && typeof current.progress === "object" ? current.progress : {};
   const material = progress.material && typeof progress.material === "object" ? progress.material : {};
   const diagnostics = current.diagnostics && typeof current.diagnostics === "object" ? current.diagnostics : {};
+  const currentPagination = diagnostics.pagination && typeof diagnostics.pagination === "object" ? diagnostics.pagination : {};
+  const materialPagination = material.pagination && typeof material.pagination === "object" ? material.pagination : {};
+  const currentOrganic = Array.isArray(current.organicResults) ? current.organicResults : [];
+  const materialOrganic = Array.isArray(material.organicResults) ? material.organicResults : [];
+  const capturesPageTwo = (pagination, rows) => pagination.requestedPages === 2 && pagination.capturedPages === 2 && pagination.page2Status === "captured" && typeof pagination.page1OrganicCount === "number" && typeof pagination.page2OrganicCount === "number" && pagination.page2OrganicCount > 0 && rows.length >= pagination.page1OrganicCount + pagination.page2OrganicCount;
+  const currentCaptured = capturesPageTwo(currentPagination, currentOrganic);
+  const materialCaptured = capturesPageTwo(materialPagination, materialOrganic);
+  const keepCurrentOrganic = currentCaptured && !materialCaptured || currentCaptured === materialCaptured && currentOrganic.length > materialOrganic.length;
+  const selectedOrganic = keepCurrentOrganic ? currentOrganic : materialOrganic.length ? materialOrganic : currentOrganic;
+  const fallbackOrganic = keepCurrentOrganic ? materialOrganic : currentOrganic;
+  const resolvedByIdentity = /* @__PURE__ */ new Map();
+  for (const row of [...fallbackOrganic, ...selectedOrganic]) {
+    if (row.resolutionStatus !== "resolved" || typeof row.resolvedUrl !== "string" || !row.resolvedUrl) continue;
+    for (const identity of [row.rawUrl, row.url, row.resolvedUrl]) {
+      if (typeof identity === "string" && identity) resolvedByIdentity.set(identity, row);
+    }
+  }
+  const reconciledOrganic = selectedOrganic.map((row) => {
+    if (row.resolutionStatus !== "unresolved" && row.resolutionStatus !== void 0) return row;
+    const resolved = [row.rawUrl, row.url].filter((identity) => typeof identity === "string" && Boolean(identity)).map((identity) => resolvedByIdentity.get(identity)).find(Boolean);
+    return resolved ? {
+      ...row,
+      url: resolved.url,
+      resolvedUrl: resolved.resolvedUrl,
+      resolutionStatus: resolved.resolutionStatus,
+      domain: resolved.domain
+    } : row;
+  });
   const completeness = diagnostics.completeness && typeof diagnostics.completeness === "object" ? diagnostics.completeness : {};
   const flat = records.map((record7) => ({
     seed_query: typeof record7.seed_query === "string" ? record7.seed_query : seedQuery,
@@ -27479,7 +27558,7 @@ function stalePartialResult(result3, records, seedQuery, recoveryCount) {
     tree: Array.isArray(current.tree) ? current.tree : [],
     videos: Array.isArray(material.videos) ? material.videos : current.videos ?? [],
     forums: Array.isArray(material.forums) ? material.forums : current.forums ?? [],
-    organicResults: Array.isArray(material.organicResults) ? material.organicResults : current.organicResults ?? [],
+    organicResults: reconciledOrganic,
     localPack: Array.isArray(material.localPack) ? material.localPack : current.localPack ?? [],
     whatPeopleSaying: Array.isArray(material.whatPeopleSaying) ? material.whatPeopleSaying : current.whatPeopleSaying ?? [],
     aiOverview: material.aiOverview ?? current.aiOverview,
@@ -27489,6 +27568,7 @@ function stalePartialResult(result3, records, seedQuery, recoveryCount) {
     partial: true,
     diagnostics: {
       ...diagnostics,
+      pagination: keepCurrentOrganic ? diagnostics.pagination : material.pagination ?? diagnostics.pagination,
       completionStatus: "paa_found",
       problem: null,
       resultQuality: "partial",
@@ -40026,13 +40106,13 @@ function validateNewRecord(vault, props) {
 function shouldRetryMemoryRead(error) {
   return /deadlock|timeout|timed out|temporar|econnreset|fetch failed/i.test(error ?? "");
 }
-function sleep2(ms) {
+function sleep(ms) {
   return new Promise((resolve3) => setTimeout(resolve3, ms));
 }
 async function readMemory(tool, args, token7) {
   let result3 = await memoryCall(tool, args, token7);
   for (let attempt = 1; !result3.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result3.error); attempt += 1) {
-    await sleep2(attempt * 150);
+    await sleep(attempt * 150);
     result3 = await memoryCall(tool, args, token7);
   }
   return result3;
@@ -86154,7 +86234,7 @@ app.get("/cron/tick", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const startedAt = Date.now();
-  const { drainQueue } = await import("./worker-OQSFSLNG.js");
+  const { drainQueue } = await import("./worker-DAVCZ3FP.js");
   const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
   const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
   const unfinished = [];

package/dist/{worker-OQSFSLNG.js → worker-DAVCZ3FP.js}

@@ -9,7 +9,7 @@ import {
 import "./chunk-OZJMVCDK.js";
 import {
   harvest
-} from "./chunk-NH4QAH3X.js";
+} from "./chunk-R2HRN2FW.js";
 import {
   sanitizeAttempts,
   sanitizeHarvestResult

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.86.4",
+  "version": "0.87.0",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",