mcp-scraper 0.35.1 → 0.37.0

package/dist/index.cjs

@@ -341,6 +341,16 @@ var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel",
 var FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
 var FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
 var DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
+function envRate(name, fallback) {
+  const raw = process.env[name]?.trim();
+  if (!raw) return fallback;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+}
+var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1);
+var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
+var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
+var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
 function kernelCostUsd(ms, headful) {
   const sec = Math.max(0, ms) / 1e3;
   return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
@@ -354,6 +364,10 @@ function vendorCostUsd(vendor, units) {
   if (vendor === "openrouter") return Math.max(0, units);
   if (vendor === "mcp_memory_video") return Math.max(0, units);
   if (vendor === "mcp_memory_ai") return Math.max(0, units);
+  if (vendor === "nango_connection") return Math.max(0, units) * NANGO_USD_PER_CONNECTION_MONTH;
+  if (vendor === "nango_function_run") return Math.max(0, units) * NANGO_USD_PER_FUNCTION_RUN;
+  if (vendor === "nango_proxy_request") return Math.max(0, units) * NANGO_USD_PER_PROXY_REQUEST;
+  if (vendor === "nango_compute") return Math.max(0, units) * NANGO_USD_PER_COMPUTE_SEC;
   return 0;
 }
 
@@ -553,8 +567,6 @@ var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X)
 var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
 var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
 var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
-var DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS = 2500;
-var SERP_CHALLENGE_SOLVE_POLL_MS = 500;
 function positiveIntFromEnv(name, fallback) {
   const raw = process.env[name];
   if (!raw) return fallback;
@@ -926,8 +938,7 @@ var BrowserDriver = class {
     }
     if (await this.isSerpChallengeVisible()) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
-      const solved = await this.waitForSerpChallengeToClear(navDebug, url);
-      if (!solved) throw new CaptchaError(this.captchaMessage());
+      throw new CaptchaError(this.captchaMessage());
     }
     const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
     if (fastFound) {
@@ -936,8 +947,7 @@ var BrowserDriver = class {
     }
     if (await this.isSerpChallengeVisible()) {
       await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
-      const solved = await this.waitForSerpChallengeToClear(navDebug, url);
-      if (!solved) throw new CaptchaError(this.captchaMessage());
+      throw new CaptchaError(this.captchaMessage());
     }
     for (let i = 1; i <= 6; i++) {
       await this.page.evaluate((f) => {
@@ -965,23 +975,6 @@ var BrowserDriver = class {
     }).catch(() => "");
     return /recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(bodySnippet);
   }
-  async waitForSerpChallengeToClear(navDebug, requestedUrl) {
-    if (!this.kernelClient || !this.page) return false;
-    const waitMs = positiveIntFromEnv("SERP_CHALLENGE_SOLVE_WAIT_MS", DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS);
-    const deadline = Date.now() + waitMs;
-    while (Date.now() < deadline) {
-      await this.page.waitForTimeout(Math.min(SERP_CHALLENGE_SOLVE_POLL_MS, Math.max(250, deadline - Date.now())));
-      if (!await this.isSerpChallengeVisible()) {
-        await this.page.waitForTimeout(1e3);
-        if (!await this.isSerpChallengeVisible()) {
-          await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: null, captchaDetected: false });
-          return true;
-        }
-      }
-    }
-    await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: false, captchaDetected: true });
-    return false;
-  }
   async updateSerpNavigationDebug(navDebug, requestedUrl, state) {
     if (!navDebug || !this.page) return;
     try {
@@ -1030,8 +1023,7 @@ var BrowserDriver = class {
     try {
       await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
       if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
-        const solved = await this.waitForSerpChallengeToClear(null, url);
-        if (!solved) throw new CaptchaError(this.captchaMessage());
+        throw new CaptchaError(this.captchaMessage());
       }
     } catch (err) {
       if (err instanceof CaptchaError) throw err;
@@ -1261,6 +1253,27 @@ function normalizeLocation(input) {
   return input;
 }
 
+// src/serp-location-query.ts
+function normalizedTerms(value) {
+  return value.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
+}
+function includesTermSequence(query, phrase) {
+  const queryTerms = normalizedTerms(query);
+  const phraseTerms = normalizedTerms(phrase);
+  if (phraseTerms.length === 0 || phraseTerms.length > queryTerms.length) return false;
+  return queryTerms.some((_, start) => phraseTerms.every((term, index) => queryTerms[start + index] === term));
+}
+function cityFromCanonicalLocation(location, fallback) {
+  return location.split(",", 1)[0]?.trim() || fallback.split(",", 1)[0]?.trim() || fallback.trim();
+}
+function buildLocalizedSerpQuery(query, location, canonicalLocation) {
+  const normalizedQuery = query.trim();
+  const requestedLocation = location?.trim();
+  if (!requestedLocation || !canonicalLocation) return normalizedQuery;
+  const city = cityFromCanonicalLocation(canonicalLocation, requestedLocation);
+  return includesTermSequence(normalizedQuery, city) ? normalizedQuery : `${normalizedQuery} ${requestedLocation}`;
+}
+
 // src/serp-location-debug.ts
 var STATE_TO_CODE = {
   alabama: "AL",
@@ -1514,6 +1527,11 @@ async function extractAISurfacesFromDocument(config) {
     if (!el) return "";
     return (el.innerText ?? el.textContent ?? "").trim();
   }
+  function hasSubstantiveContent(text) {
+    if (!text) return false;
+    const normalized = text.replace(/\s+/g, " ").trim();
+    return normalized.length >= 16 && normalized.split(" ").filter(Boolean).length >= 2;
+  }
   function hasAIOverviewLabel(el) {
     const heading = el.querySelector(selectors.aio.heading);
     if (textOf(heading) === "AI Overview") return true;
@@ -1676,19 +1694,22 @@ async function extractAISurfacesFromDocument(config) {
     aioCitations = extractCitations(aioRoot);
   }
   const aimRoot = document.querySelector(selectors.aim.root);
-  const aimDetected = surface === "aim" && !!aimRoot;
+  const aimRootDetected = surface === "aim" && !!aimRoot;
   const aimContainer = aimRoot?.closest(selectors.aim.wrapper) ?? aimRoot;
-  const aimText = cleanText(aimContainer);
+  const rawAimText = aimRootDetected ? cleanText(aimContainer) : null;
+  const aimDetected = aimRootDetected && hasSubstantiveContent(rawAimText);
+  const aimText = aimDetected ? rawAimText : null;
   const aimCitations = aimDetected ? extractCitations(aimContainer) : [];
+  const aiOverviewDetected = !!aioRoot && hasSubstantiveContent(aioText);
   return {
     surface,
     aiOverview: {
-      detected: !!aioRoot && aioText !== null,
-      text: aioText,
-      citations: aioCitations,
-      expanded: aioExpanded,
-      fullyExpanded: aioFullyExpanded,
-      sections: aioSections
+      detected: aiOverviewDetected,
+      text: aiOverviewDetected ? aioText : null,
+      citations: aiOverviewDetected ? aioCitations : [],
+      expanded: aiOverviewDetected ? aioExpanded : false,
+      fullyExpanded: aiOverviewDetected ? aioFullyExpanded : false,
+      sections: aiOverviewDetected ? aioSections : []
     },
     aiMode: {
       detected: aimDetected && aimText !== null,
@@ -2462,8 +2483,10 @@ var PAAExtractor = class {
       this.throwIfAborted(signal);
       const canonicalLocation = options.location ? normalizeLocation(options.location) : null;
       const uule = canonicalLocation ? encodeUule(canonicalLocation) : null;
+      const executedQuery = buildLocalizedSerpQuery(options.query, options.location, canonicalLocation);
+      const executionOptions = executedQuery === options.query ? options : { ...options, query: executedQuery };
       const { hasPaa } = await this.driver.navigateToSERP(
-        options.query,
+        executionOptions.query,
         uule,
         options.gl,
         options.hl,
@@ -2488,8 +2511,7 @@ var PAAExtractor = class {
         let locationEvidence2 = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults2, localPack2) : void 0;
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
+          const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
           if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
@@ -2501,7 +2523,7 @@ var PAAExtractor = class {
           }
         }
         const stats2 = {
-          seed: options.query,
+          seed: executionOptions.query,
           totalQuestions: 0,
           maxDepthReached: 0,
           durationMs: Date.now() - startMs,
@@ -2509,14 +2531,14 @@ var PAAExtractor = class {
         };
         this.reporter.onComplete(stats2);
         return {
-          seed: options.query,
+          seed: executionOptions.query,
           location: options.location ?? null,
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
           diagnostics: {
             completionStatus: "serp_only",
             problem: null,
             completeness: { ...this.completeness },
-            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
           surface: aiSurfaces2.surface,
@@ -2549,8 +2571,7 @@ var PAAExtractor = class {
         let noPaaOrganic = organicResults;
         let locationEvidence2 = initialLocationEvidence;
         if ((options.pages ?? 1) >= 2) {
-          const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
-          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
+          const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
           if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
@@ -2563,7 +2584,7 @@ var PAAExtractor = class {
         }
         const aiSurfaces2 = await this.extractAISurfaces(page, options);
         const stats2 = {
-          seed: options.query,
+          seed: executionOptions.query,
           totalQuestions: 0,
           maxDepthReached: 0,
           durationMs: Date.now() - startMs,
@@ -2571,14 +2592,14 @@ var PAAExtractor = class {
         };
         this.reporter.onComplete(stats2);
         return {
-          seed: options.query,
+          seed: executionOptions.query,
           location: options.location ?? null,
           extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
           diagnostics: {
             completionStatus: "no_paa",
             problem: null,
             completeness: { ...this.completeness },
-            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
+            ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
           surface: aiSurfaces2.surface,
@@ -2595,10 +2616,10 @@ var PAAExtractor = class {
           stats: stats2
         };
       }
-      const flat = await this.runBFS(page, options, signal);
+      const flat = await this.runBFS(page, executionOptions, signal);
       this.throwIfAborted(signal);
       const aiSurfaces = await this.extractAISurfaces(page, options);
-      const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
+      const shortVidsParams = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
       if (uule) shortVidsParams.set("uule", uule);
       let shortVideos = [];
       try {
@@ -2617,7 +2638,7 @@ var PAAExtractor = class {
       let allOrganic = organicResults;
       let locationEvidence = initialLocationEvidence;
       if ((options.pages ?? 1) >= 2) {
-        const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+        const p2params = new URLSearchParams({ q: executionOptions.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
         if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
         if (uule) p2params.set("uule", uule);
         await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
@@ -2629,9 +2650,9 @@ var PAAExtractor = class {
         }
       }
       const allVideos = [...videos, ...shortVideos];
-      const tree = this.buildTree(flat, options.query);
+      const tree = this.buildTree(flat, executionOptions.query);
       const stats = {
-        seed: options.query,
+        seed: executionOptions.query,
         totalQuestions: flat.length,
         maxDepthReached: flat.reduce((m, r) => Math.max(m, r.depth), 0),
         durationMs: Date.now() - startMs,
@@ -2639,7 +2660,7 @@ var PAAExtractor = class {
       };
       this.reporter.onComplete(stats);
       return {
-        seed: options.query,
+        seed: executionOptions.query,
         location: options.location ?? null,
         extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
         diagnostics: {
@@ -2647,7 +2668,7 @@ var PAAExtractor = class {
           problem: null,
           completeness: { ...this.completeness },
           ...diagnosticWarnings.length > 0 ? { warnings: diagnosticWarnings } : {},
-          ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence) } : {}
+          ...options.debug ? { debug: this.buildHarvestDebugSnapshot(executionOptions, canonicalLocation, uule, locationEvidence) } : {}
         },
         totalQuestions: flat.length,
         surface: aiSurfaces.surface,
@@ -3374,8 +3395,33 @@ function classifyAttemptError(err) {
   return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message) ? "timeout" : "error";
 }
 function classifyAttemptResult(result) {
+  if (result.diagnostics.degradedResult) return "degraded_result";
   return result.diagnostics?.completionStatus ?? (result.totalQuestions > 0 ? "paa_found" : "no_paa");
 }
+function resultHasUsableContent(result) {
+  return result.flat.length > 0 || result.organicResults.length > 0 || result.localPack.length > 0 || result.videos.length > 0 || result.forums.length > 0 || result.whatPeopleSaying.length > 0 || result.aiOverview.detected || result.aiMode.detected;
+}
+function withResultQuality(result) {
+  const hasUsableContent = resultHasUsableContent(result);
+  const hasWarnings = (result.diagnostics.warnings?.length ?? 0) > 0;
+  const resultQuality = hasUsableContent ? hasWarnings ? "partial" : "complete" : "degraded";
+  const degradedResult = resultQuality === "degraded";
+  result.diagnostics = {
+    ...result.diagnostics,
+    resultQuality,
+    degradedResult,
+    degradationReasons: degradedResult ? ["empty_primary_serp"] : [],
+    retryRecommended: degradedResult
+  };
+  return result;
+}
+function degradedResultMessage(result) {
+  const reasons = result.diagnostics.degradationReasons ?? [];
+  if (reasons.includes("empty_primary_serp")) {
+    return "No usable primary SERP surface was captured; the result may be incomplete.";
+  }
+  return "The harvest result was marked degraded.";
+}
 function errorMessage(err) {
   return err instanceof Error ? err.message : String(err);
 }
@@ -3545,60 +3591,60 @@ async function harvest(rawOptions) {
       const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
-        const outcome = classifyAttemptError(err);
-        if ((outcome === "captcha" || outcome === "proxy_tunnel_failed") && effectiveProxyMode === "location") {
+        const outcome2 = classifyAttemptError(err);
+        if ((outcome2 === "captcha" || outcome2 === "proxy_tunnel_failed") && effectiveProxyMode === "location") {
           proxyFailures++;
           if (proxyFailures >= 2) effectiveProxyMode = "none";
         }
-        const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
-        if (outcome === "captcha") {
+        const willRetry2 = i < maxAttempts - 1 && (outcome2 === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome2));
+        if (outcome2 === "captcha") {
           console.warn(JSON.stringify({
             event: "harvest_attempt_captcha",
             attempt_number: attemptNumber,
             max_attempts: maxAttempts,
             message: errorMessage(err),
-            will_retry: willRetry
+            will_retry: willRetry2
           }));
-        } else if (willRetry) {
+        } else if (willRetry2) {
           console.warn(JSON.stringify({
             event: "harvest_attempt_proxy_retry",
             attempt_number: attemptNumber,
             max_attempts: maxAttempts,
-            outcome,
+            outcome: outcome2,
             message: errorMessage(err),
-            will_retry: willRetry
+            will_retry: willRetry2
           }));
         }
         await emitAttemptEvent(onAttemptEvent, {
           type: "finished",
           attemptNumber,
           maxAttempts,
-          outcome,
+          outcome: outcome2,
           kernelSessionId: attempt.cleanup.kernelSessionId,
           questionCount: 0,
           durationMs: Date.now() - startedAtMs,
           error: errorMessage(err),
-          willRetry,
+          willRetry: willRetry2,
           cleanup: attempt.cleanup,
           debug: attempt.debug,
           completedAt: (/* @__PURE__ */ new Date()).toISOString()
         });
         await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
         lastError = err;
-        if (willRetry) continue;
+        if (willRetry2) continue;
         break;
       }
       const result = attempt.result;
       if (!result) throw new Error("Harvest attempt completed without a result");
       if (shouldRetryLocationMismatch(result, proxyMode)) {
         const err = new LocationMismatchError(locationMismatchMessage(result));
-        const willRetry = i < maxAttempts - 1;
+        const willRetry2 = i < maxAttempts - 1;
         console.warn(JSON.stringify({
           event: "harvest_attempt_location_mismatch",
           attempt_number: attemptNumber,
           max_attempts: maxAttempts,
           message: err.message,
-          will_retry: willRetry
+          will_retry: willRetry2
         }));
         await emitAttemptEvent(onAttemptEvent, {
           type: "finished",
@@ -3609,32 +3655,45 @@ async function harvest(rawOptions) {
           questionCount: result.totalQuestions,
           durationMs: Date.now() - startedAtMs,
           error: err.message,
-          willRetry,
+          willRetry: willRetry2,
           cleanup: attempt.cleanup,
           debug: attempt.debug,
           completedAt: (/* @__PURE__ */ new Date()).toISOString()
         });
         await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
         lastError = err;
-        if (willRetry) continue;
+        if (willRetry2) continue;
         break;
       }
-      const finalResult = stripInternalDebug(result, requestedDebug);
+      const finalResult = withResultQuality(stripInternalDebug(result, requestedDebug));
+      const outcome = classifyAttemptResult(finalResult);
+      const willRetry = outcome === "degraded_result" && i < maxAttempts - 1;
+      const resultError = outcome === "degraded_result" ? degradedResultMessage(finalResult) : null;
+      if (willRetry) {
+        console.warn(JSON.stringify({
+          event: "harvest_attempt_degraded_result",
+          attempt_number: attemptNumber,
+          max_attempts: maxAttempts,
+          degradation_reasons: finalResult.diagnostics.degradationReasons,
+          will_retry: true
+        }));
+      }
       await emitAttemptEvent(onAttemptEvent, {
         type: "finished",
         attemptNumber,
         maxAttempts,
-        outcome: classifyAttemptResult(finalResult),
+        outcome,
         kernelSessionId: attempt.cleanup.kernelSessionId,
         questionCount: finalResult.totalQuestions,
         durationMs: Date.now() - startedAtMs,
-        error: null,
-        willRetry: false,
+        error: resultError,
+        willRetry,
         cleanup: attempt.cleanup,
         debug: attempt.debug,
         completedAt: (/* @__PURE__ */ new Date()).toISOString()
       });
       await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
+      if (willRetry) continue;
       if (attemptOptions.format === "json" || attemptOptions.format === "both") {
         await serializer.writeJSON(finalResult, attemptOptions.outputDir);
       }