mcp-scraper 0.2.0 → 0.2.2

package/dist/index.cjs

@@ -77,8 +77,12 @@ var MapsSearchOptionsSchema = import_zod.z.object({
   gl: import_zod.z.string().length(2).default("us"),
   hl: import_zod.z.string().length(2).default("en"),
   maxResults: import_zod.z.number().int().min(1).max(50).default(10),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location"),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional(),
+  debug: import_zod.z.boolean().default(false),
   kernelApiKey: import_zod.z.string().optional(),
   kernelProxyId: import_zod.z.string().optional(),
+  kernelProxyResolution: import_zod.z.unknown().optional(),
   headless: import_zod.z.boolean().default(true)
 });
 var RawPAAItemSchema = import_zod.z.object({
@@ -254,6 +258,12 @@ var RequestAbortedError = class extends Error {
     super(message);
   }
 };
+var LocationMismatchError = class extends Error {
+  name = "LocationMismatchError";
+  constructor(message = "Google returned results for a different location than requested") {
+    super(message);
+  }
+};
 
 // src/driver/BrowserDriver.ts
 import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
@@ -2274,16 +2284,18 @@ var US_CITY_CENTER_ZIPS = {
 function proxyIdSuffix2(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
-function resolution(source, proxyMode, proxyId, target, error) {
+function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
   return {
     kernelProxyId: proxyId,
+    ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
     resolution: {
       source,
       proxyMode,
       proxyIdPresent: Boolean(proxyId),
       proxyIdSuffix: proxyIdSuffix2(proxyId),
       target,
-      error
+      error,
+      disposable
     }
   };
 }
@@ -2313,6 +2325,10 @@ function kernelCityIdentifierCandidates(city) {
 function proxyName(country, state, city) {
   return city ? `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}-${city}` : `mcp-serp-residential-${country.toLowerCase()}-${state.toLowerCase()}`;
 }
+function freshProxyName(baseName, attemptIndex) {
+  const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
+  return `${baseName}-fresh-${stamp}`;
+}
 function zipProxyName(zip) {
   return `mcp-serp-residential-us-zip-${zip}`;
 }
@@ -2382,6 +2398,12 @@ function zipTarget(target, zip) {
     }
   };
 }
+function withProxyName(target, name) {
+  return {
+    ...target,
+    proxyName: name
+  };
+}
 function configMatches(config, target, city) {
   if (target.level === "zip") {
     return config?.country?.toUpperCase() === target.country && config?.zip === target.zip;
@@ -2420,6 +2442,55 @@ function escalatedTargetLevel(target, attemptIndex) {
 function errorText2(err) {
   return err instanceof Error ? err.message : String(err);
 }
+function freshTargetCandidates(target, explicitZip, attemptIndex) {
+  const out = [];
+  const zip = knownZipFor(target, explicitZip);
+  if (zip) {
+    const targetZip = zipTarget(target, zip);
+    out.push(withProxyName(targetZip, freshProxyName(targetZip.proxyName, attemptIndex)));
+  }
+  for (const city of target.cityCandidates) {
+    const cityTarget = {
+      ...target,
+      level: "city",
+      city,
+      proxyName: proxyName(target.country, target.state, city),
+      config: {
+        country: target.country,
+        state: target.state,
+        city
+      }
+    };
+    out.push(withProxyName(cityTarget, freshProxyName(cityTarget.proxyName, attemptIndex)));
+  }
+  const fallbackTarget = stateTarget(target);
+  out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
+  return out;
+}
+async function createFreshLocationProxy(kernel, options, target) {
+  const createErrors = [];
+  for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
+    try {
+      const created = await kernel.proxies.create({
+        type: "residential",
+        name: candidate.proxyName,
+        config: candidate.level === "zip" ? { country: candidate.country, zip: candidate.zip } : candidate.config
+      });
+      if (created.id) {
+        return resolution("location_created", options.proxyMode, created.id, candidate, null, true);
+      }
+      createErrors.push(`${candidate.proxyName}: Kernel did not return a proxy id`);
+    } catch (err) {
+      createErrors.push(`${candidate.proxyName}: ${errorText2(err)}`);
+    }
+  }
+  return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+}
+async function deleteKernelProxyId(kernelApiKey, proxyId) {
+  if (!kernelApiKey || !proxyId) return;
+  const kernel = new import_sdk2.default({ apiKey: kernelApiKey });
+  await kernel.proxies.delete(proxyId);
+}
 async function resolveKernelProxyId(options) {
   if (options.proxyMode === "none") {
     return resolution("disabled", options.proxyMode, void 0, null, null);
@@ -2434,6 +2505,9 @@ async function resolveKernelProxyId(options) {
   const kernel = new import_sdk2.default({ apiKey: options.kernelApiKey });
   try {
     const attemptIndex = options.attemptIndex ?? 0;
+    if (options.fresh) {
+      return await createFreshLocationProxy(kernel, options, target);
+    }
     if (attemptIndex >= 1) {
       const escalatedTarget = escalatedTargetLevel(target, attemptIndex);
       const createErrors2 = [];
@@ -2537,6 +2611,7 @@ async function resolveKernelProxyId(options) {
 
 // src/harvest.ts
 var MAX_ATTEMPTS = 3;
+var LOCATION_PROXY_MAX_ATTEMPTS = 5;
 function abortReason(signal) {
   if (signal.reason instanceof DOMException && signal.reason.name === "TimeoutError") return signal.reason;
   return new RequestAbortedError();
@@ -2566,9 +2641,12 @@ async function emitAttemptEvent(sink, event) {
 }
 function classifyAttemptError(err) {
   if (err instanceof CaptchaError) return "captcha";
+  if (err instanceof LocationMismatchError) return "location_mismatch";
   if (err instanceof RequestAbortedError) return "request_aborted";
   if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return "timeout";
   const message = err instanceof Error ? err.message : String(err);
+  if (looksLikeProxyTunnelFailure(message)) return "proxy_tunnel_failed";
+  if (looksLikeProxyUnavailable(message)) return "proxy_unavailable";
   return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message) ? "timeout" : "error";
 }
 function classifyAttemptResult(result) {
@@ -2577,6 +2655,49 @@ function classifyAttemptResult(result) {
 function errorMessage(err) {
   return err instanceof Error ? err.message : String(err);
 }
+function maxAttemptsForProxyMode(proxyMode) {
+  return proxyMode === "location" ? LOCATION_PROXY_MAX_ATTEMPTS : MAX_ATTEMPTS;
+}
+function looksLikeProxyTunnelFailure(message) {
+  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
+}
+function looksLikeProxyUnavailable(message) {
+  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
+}
+function retryableLocationProxyError(outcome) {
+  return outcome === "captcha" || outcome === "proxy_tunnel_failed" || outcome === "proxy_unavailable";
+}
+function locationMismatchMessage(result) {
+  const evidence = result.diagnostics.debug?.locationEvidence;
+  const expected = evidence?.expected?.canonicalLocation ?? result.location ?? "requested location";
+  const candidates = evidence?.candidates.slice(0, 3).map((candidate) => `${candidate.city}, ${candidate.regionCode}`).join("; ");
+  return candidates ? `Google returned results for ${candidates}, not ${expected}` : `Google returned results for a different location than ${expected}`;
+}
+function shouldRetryLocationMismatch(result, proxyMode) {
+  return proxyMode === "location" && result.diagnostics.debug?.locationEvidence?.status === "mismatch";
+}
+function stripInternalDebug(result, keepDebug) {
+  if (keepDebug || !result.diagnostics.debug) return result;
+  const diagnostics = { ...result.diagnostics };
+  delete diagnostics.debug;
+  return { ...result, diagnostics };
+}
+async function cleanupDisposableProxy(kernelApiKey, proxyId) {
+  if (!kernelApiKey || !proxyId) return;
+  try {
+    await deleteKernelProxyId(kernelApiKey, proxyId);
+    console.info(JSON.stringify({
+      event: "kernel_proxy_deleted",
+      proxy_id_suffix: proxyId.slice(-6)
+    }));
+  } catch (err) {
+    console.warn(JSON.stringify({
+      event: "kernel_proxy_delete_failed",
+      proxy_id_suffix: proxyId.slice(-6),
+      message: errorMessage(err)
+    }));
+  }
+}
 async function extractOnce(options, signal) {
   const driver = new BrowserDriver();
   const reporter = new ProgressReporter();
@@ -2644,26 +2765,35 @@ async function harvest(rawOptions) {
     proxyZip: typeof raw.proxyZip === "string" ? raw.proxyZip : void 0,
     gl: typeof raw.gl === "string" ? raw.gl : "us"
   };
+  const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
+  const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
+  const maxAttempts = maxAttemptsForProxyMode(proxyMode);
   const serializer = new OutputSerializer();
-  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+  let lastError = null;
+  for (let i = 0; i < maxAttempts; i++) {
     const attemptNumber = i + 1;
     const startedAtMs = Date.now();
     try {
       if (signal?.aborted) throw abortReason(signal);
-      const resolution2 = await resolveKernelProxyId({ ...proxyOpts, attemptIndex: i });
+      const resolution2 = await resolveKernelProxyId({
+        ...proxyOpts,
+        attemptIndex: i,
+        fresh: proxyMode === "location"
+      });
       const mergedAttempt = {
         ...raw,
         kernelApiKey,
         kernelProxyId: resolution2.kernelProxyId,
         kernelProxyResolution: resolution2.resolution,
-        proxyMode
+        proxyMode,
+        debug: requestedDebug || needsLocationEvidence
       };
       if (proxyMode === "none") mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",
         attemptNumber,
-        maxAttempts: MAX_ATTEMPTS,
+        maxAttempts,
         query: attemptOptions.query,
         location: attemptOptions.location ?? null,
         maxQuestions: attemptOptions.maxQuestions,
@@ -2672,7 +2802,7 @@ async function harvest(rawOptions) {
       console.info(JSON.stringify({
         event: "harvest_attempt_started",
         attempt_number: attemptNumber,
-        max_attempts: MAX_ATTEMPTS,
+        max_attempts: maxAttempts,
         query: attemptOptions.query,
         location: attemptOptions.location ?? null,
         max_questions: attemptOptions.maxQuestions
@@ -2680,57 +2810,84 @@ async function harvest(rawOptions) {
       const attempt = await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
-        if (err instanceof CaptchaError) {
-          const willRetry = i < MAX_ATTEMPTS - 1;
+        const outcome = classifyAttemptError(err);
+        const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
+        if (outcome === "captcha") {
           console.warn(JSON.stringify({
             event: "harvest_attempt_captcha",
             attempt_number: attemptNumber,
-            max_attempts: MAX_ATTEMPTS,
-            message: err.message,
+            max_attempts: maxAttempts,
+            message: errorMessage(err),
+            will_retry: willRetry
+          }));
+        } else if (willRetry) {
+          console.warn(JSON.stringify({
+            event: "harvest_attempt_proxy_retry",
+            attempt_number: attemptNumber,
+            max_attempts: maxAttempts,
+            outcome,
+            message: errorMessage(err),
             will_retry: willRetry
           }));
-          await emitAttemptEvent(onAttemptEvent, {
-            type: "finished",
-            attemptNumber,
-            maxAttempts: MAX_ATTEMPTS,
-            outcome: "captcha",
-            kernelSessionId: attempt.cleanup.kernelSessionId,
-            questionCount: 0,
-            durationMs: Date.now() - startedAtMs,
-            error: err.message,
-            willRetry,
-            cleanup: attempt.cleanup,
-            debug: attempt.debug,
-            completedAt: (/* @__PURE__ */ new Date()).toISOString()
-          });
-          if (willRetry) continue;
-          break;
         }
         await emitAttemptEvent(onAttemptEvent, {
           type: "finished",
           attemptNumber,
-          maxAttempts: MAX_ATTEMPTS,
-          outcome: classifyAttemptError(err),
+          maxAttempts,
+          outcome,
           kernelSessionId: attempt.cleanup.kernelSessionId,
           questionCount: 0,
           durationMs: Date.now() - startedAtMs,
           error: errorMessage(err),
-          willRetry: false,
+          willRetry,
           cleanup: attempt.cleanup,
           debug: attempt.debug,
           completedAt: (/* @__PURE__ */ new Date()).toISOString()
         });
-        throw err;
+        await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
+        lastError = err;
+        if (willRetry) continue;
+        break;
       }
       const result = attempt.result;
       if (!result) throw new Error("Harvest attempt completed without a result");
+      if (shouldRetryLocationMismatch(result, proxyMode)) {
+        const err = new LocationMismatchError(locationMismatchMessage(result));
+        const willRetry = i < maxAttempts - 1;
+        console.warn(JSON.stringify({
+          event: "harvest_attempt_location_mismatch",
+          attempt_number: attemptNumber,
+          max_attempts: maxAttempts,
+          message: err.message,
+          will_retry: willRetry
+        }));
+        await emitAttemptEvent(onAttemptEvent, {
+          type: "finished",
+          attemptNumber,
+          maxAttempts,
+          outcome: "location_mismatch",
+          kernelSessionId: attempt.cleanup.kernelSessionId,
+          questionCount: result.totalQuestions,
+          durationMs: Date.now() - startedAtMs,
+          error: err.message,
+          willRetry,
+          cleanup: attempt.cleanup,
+          debug: attempt.debug,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
+        lastError = err;
+        if (willRetry) continue;
+        break;
+      }
+      const finalResult = stripInternalDebug(result, requestedDebug);
       await emitAttemptEvent(onAttemptEvent, {
         type: "finished",
         attemptNumber,
-        maxAttempts: MAX_ATTEMPTS,
-        outcome: classifyAttemptResult(result),
+        maxAttempts,
+        outcome: classifyAttemptResult(finalResult),
         kernelSessionId: attempt.cleanup.kernelSessionId,
-        questionCount: result.totalQuestions,
+        questionCount: finalResult.totalQuestions,
         durationMs: Date.now() - startedAtMs,
         error: null,
         willRetry: false,
@@ -2738,64 +2895,52 @@ async function harvest(rawOptions) {
         debug: attempt.debug,
         completedAt: (/* @__PURE__ */ new Date()).toISOString()
       });
+      await cleanupDisposableProxy(kernelApiKey, resolution2.disposableProxyId);
       if (attemptOptions.format === "json" || attemptOptions.format === "both") {
-        await serializer.writeJSON(result, attemptOptions.outputDir);
+        await serializer.writeJSON(finalResult, attemptOptions.outputDir);
       }
       if (attemptOptions.format === "csv" || attemptOptions.format === "both") {
         await Promise.all([
-          serializer.writeCSV(result.flat, attemptOptions.outputDir),
-          result.videos.length > 0 ? serializer.writeVideoCSV(result.videos, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
-          result.forums.length > 0 ? serializer.writeForumCSV(result.forums, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
-          result.aiOverview.detected ? serializer.writeAIOverviewCSV(result.aiOverview.citations, result.aiOverview.text, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
-          result.aiMode.detected ? serializer.writeAIModeCSV(result.aiMode.citations, result.aiMode.text, result.seed, attemptOptions.outputDir) : Promise.resolve(""),
-          result.whatPeopleSaying.length > 0 ? serializer.writeWhatPeopleSayingCSV(result.whatPeopleSaying, result.seed, attemptOptions.outputDir) : Promise.resolve("")
+          serializer.writeCSV(finalResult.flat, attemptOptions.outputDir),
+          finalResult.videos.length > 0 ? serializer.writeVideoCSV(finalResult.videos, finalResult.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          finalResult.forums.length > 0 ? serializer.writeForumCSV(finalResult.forums, finalResult.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          finalResult.aiOverview.detected ? serializer.writeAIOverviewCSV(finalResult.aiOverview.citations, finalResult.aiOverview.text, finalResult.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          finalResult.aiMode.detected ? serializer.writeAIModeCSV(finalResult.aiMode.citations, finalResult.aiMode.text, finalResult.seed, attemptOptions.outputDir) : Promise.resolve(""),
+          finalResult.whatPeopleSaying.length > 0 ? serializer.writeWhatPeopleSayingCSV(finalResult.whatPeopleSaying, finalResult.seed, attemptOptions.outputDir) : Promise.resolve("")
         ]);
       }
-      return result;
+      return finalResult;
     } catch (err) {
-      if (err instanceof CaptchaError) {
-        const willRetry = i < MAX_ATTEMPTS - 1;
+      const outcome = classifyAttemptError(err);
+      const willRetry = i < maxAttempts - 1 && (outcome === "captcha" || proxyMode === "location" && retryableLocationProxyError(outcome));
+      if (outcome === "captcha") {
         console.warn(JSON.stringify({
           event: "harvest_attempt_captcha",
           attempt_number: attemptNumber,
-          max_attempts: MAX_ATTEMPTS,
-          message: err.message,
+          max_attempts: maxAttempts,
+          message: errorMessage(err),
+          will_retry: willRetry
+        }));
+      } else if (willRetry) {
+        console.warn(JSON.stringify({
+          event: "harvest_attempt_proxy_retry",
+          attempt_number: attemptNumber,
+          max_attempts: maxAttempts,
+          outcome,
+          message: errorMessage(err),
           will_retry: willRetry
         }));
-        await emitAttemptEvent(onAttemptEvent, {
-          type: "finished",
-          attemptNumber,
-          maxAttempts: MAX_ATTEMPTS,
-          outcome: "captcha",
-          kernelSessionId: null,
-          questionCount: 0,
-          durationMs: Date.now() - startedAtMs,
-          error: err.message,
-          willRetry,
-          cleanup: {
-            kernelSessionId: null,
-            kernelDeleteStarted: false,
-            kernelDeleteSucceeded: null,
-            kernelDeleteError: null,
-            browserCloseSucceeded: null,
-            browserCloseError: null
-          },
-          debug: null,
-          completedAt: (/* @__PURE__ */ new Date()).toISOString()
-        });
-        if (willRetry) continue;
-        break;
       }
       await emitAttemptEvent(onAttemptEvent, {
         type: "finished",
         attemptNumber,
-        maxAttempts: MAX_ATTEMPTS,
-        outcome: classifyAttemptError(err),
+        maxAttempts,
+        outcome,
         kernelSessionId: null,
         questionCount: 0,
         durationMs: Date.now() - startedAtMs,
         error: errorMessage(err),
-        willRetry: false,
+        willRetry,
         cleanup: {
           kernelSessionId: null,
           kernelDeleteStarted: false,
@@ -2807,15 +2952,19 @@ async function harvest(rawOptions) {
         debug: null,
         completedAt: (/* @__PURE__ */ new Date()).toISOString()
       });
+      lastError = err;
+      if (willRetry) continue;
+      if (outcome === "captcha") break;
       throw err;
     }
   }
+  if (lastError && !(lastError instanceof CaptchaError)) throw lastError;
   console.warn(JSON.stringify({
     event: "harvest_captcha_exhausted",
-    max_attempts: MAX_ATTEMPTS,
+    max_attempts: maxAttempts,
     session_kind: kernelApiKey ? "kernel" : "local"
   }));
-  throw new CaptchaError(sanitizeVendorName(`CAPTCHA on all ${MAX_ATTEMPTS} fresh sessions. Try again in a few minutes.`));
+  throw new CaptchaError(sanitizeVendorName(`CAPTCHA on all ${maxAttempts} fresh sessions. Try again in a few minutes.`));
 }
 
 // src/video/VideoGenerator.ts