mcp-scraper 0.38.0 → 0.38.2

package/dist/bin/mcp-stdio-server.js

@@ -7,19 +7,19 @@ import {
   registerMemoryMcpTools,
   registerPaaExtractorMcpTools,
   registerSerpIntelligenceCaptureTools
-} from "../chunk-P64PZUCA.js";
+} from "../chunk-NKCCGADE.js";
 import "../chunk-345BQXZH.js";
-import "../chunk-MA5JBAUZ.js";
+import "../chunk-EGJKUB4Q.js";
 import {
   renderInstallTerminal
 } from "../chunk-NNW3O6ZD.js";
 import "../chunk-CB5C3BPB.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-H3M3O457.js";
-import "../chunk-PWPUKR5U.js";
+} from "../chunk-YODBNTTN.js";
+import "../chunk-AZRPG43B.js";
 import "../chunk-44HZLHDV.js";
-import "../chunk-CSCD2HNS.js";
+import "../chunk-N7KUTTCC.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";

package/dist/bin/paa-harvest.cjs

@@ -327,7 +327,7 @@ function currentCostContext() {
 // src/api/cost-rates.ts
 var KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
 var KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
-var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel", "yt_transcription", "fb_search", "fb_ad", "instagram"]);
+var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "fb_search", "fb_ad", "instagram"]);
 function envRate(name, fallback) {
   const raw = process.env[name]?.trim();
   if (!raw) return fallback;
@@ -965,8 +965,10 @@ var BrowserDriver = class {
     return this.kernelClient ? "Google returned a CAPTCHA on this session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
   }
   async navigateTo(url) {
+    const arrivalBase = process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim();
+    const navUrl = arrivalBase ? serpArrivalUrl(url, arrivalBase) : url;
     try {
-      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
       if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
         throw new CaptchaError(this.captchaMessage());
       }
@@ -1681,6 +1683,8 @@ var PAAExtractor = class {
     aioShareCaptured: null
   };
   aioShareUrlEarly = null;
+  collectedRows = [];
+  partialContext = null;
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -1785,11 +1789,50 @@ var PAAExtractor = class {
       extracted_at: (/* @__PURE__ */ new Date()).toISOString()
     };
   }
+  getPartialResult() {
+    const ctx = this.partialContext;
+    if (!ctx || this.collectedRows.length === 0) return null;
+    const flat = [...this.collectedRows];
+    const stats = {
+      seed: ctx.seed,
+      totalQuestions: flat.length,
+      maxDepthReached: flat.reduce((m, r) => Math.max(m, r.depth), 0),
+      durationMs: Date.now() - ctx.startMs,
+      errorCount: 0
+    };
+    return {
+      seed: ctx.seed,
+      location: ctx.location,
+      extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      diagnostics: {
+        completionStatus: "paa_found",
+        problem: null,
+        completeness: { ...this.completeness },
+        warnings: [{
+          code: "paa_partial",
+          surface: "paa",
+          message: `Harvest stopped before finishing; returning the ${flat.length} question(s) collected up to that point.`,
+          retryable: true
+        }]
+      },
+      totalQuestions: flat.length,
+      whatPeopleSaying: [],
+      tree: this.buildTree(flat, ctx.seed),
+      flat,
+      videos: [],
+      forums: [],
+      organicResults: [],
+      localPack: [],
+      entityIds: {},
+      stats
+    };
+  }
   async runBFS(page, options, signal) {
     const seenKeys = /* @__PURE__ */ new Set();
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
-    const results = [];
+    this.collectedRows = [];
+    const results = this.collectedRows;
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickedOnceEver = /* @__PURE__ */ new Set();
     const itemArgs = {
@@ -2401,6 +2444,8 @@ var PAAExtractor = class {
     const startMs = Date.now();
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
     this.aioShareUrlEarly = null;
+    this.collectedRows = [];
+    this.partialContext = { seed: options.query, location: options.location ?? null, startMs };
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -3462,7 +3507,23 @@ async function extractOnce(options, signal) {
     } : null);
     cleanup = await driver.close();
   }
-  return error ? { result: null, error, cleanup, debug } : { result, error: null, cleanup, debug };
+  if (error) {
+    const outcome = classifyAttemptError(error);
+    const budgetExhausted = outcome === "timeout" || outcome === "request_aborted";
+    let salvaged = null;
+    if (budgetExhausted) {
+      try {
+        salvaged = typeof extractor.getPartialResult === "function" ? extractor.getPartialResult() : null;
+      } catch {
+        salvaged = null;
+      }
+    }
+    if (salvaged) {
+      return { result: salvaged, error: null, cleanup, debug, salvagedFrom: error };
+    }
+    return { result: null, error, cleanup, debug };
+  }
+  return { result, error: null, cleanup, debug };
 }
 async function harvest(rawOptions) {
   const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};