mcp-scraper 0.2.7 → 0.2.8

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   harvest
-} from "./chunk-MY3S7EX7.js";
+} from "./chunk-CQTAKXBN.js";
 import "./chunk-M2S27J6Z.js";
 
 // src/video/VideoGenerator.ts

package/dist/{server-RIKRBDOI.js → server-BTTDFPSQ.js}

@@ -31,7 +31,7 @@ import {
   deleteKernelProxyId,
   harvest,
   resolveKernelProxyId
-} from "./chunk-MY3S7EX7.js";
+} from "./chunk-CQTAKXBN.js";
 import {
   CaptureSerpPageSnapshotsInputSchema,
   CaptureSerpSnapshotInputSchema,
@@ -41,13 +41,13 @@ import {
   harvestTimeoutBudget,
   liveWebToolAnnotations,
   outputBaseDir
-} from "./chunk-7SI6XIR3.js";
+} from "./chunk-6NEXSNSA.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
   sanitizeVendorName
 } from "./chunk-M2S27J6Z.js";
-import "./chunk-IZE5UW7Y.js";
+import "./chunk-I26QN7WQ.js";
 import {
   SiteAuditJobRowSchema,
   cancelJob,
@@ -9893,24 +9893,205 @@ var MapsSearchExtractor = class {
   }
 };
 
-// src/api/maps-routes.ts
-function mapsErrorResponse(c, msg, errorCode) {
-  const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
-  return c.json({
-    error: sanitizeVendorName(msg),
-    error_code: blocked ? "captcha_or_blocked" : errorCode,
-    retryable: blocked
-  }, blocked ? 503 : 500);
+// src/maps-search-rotation.ts
+var MAPS_SEARCH_MAX_ATTEMPTS = 3;
+var MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS = 5;
+var MapsSearchRotationError = class extends Error {
+  constructor(message, attempts, retryable, errorCode = "maps_search_failed") {
+    super(message);
+    this.attempts = attempts;
+    this.retryable = retryable;
+    this.errorCode = errorCode;
+  }
+  attempts;
+  retryable;
+  errorCode;
+  name = "MapsSearchRotationError";
+};
+function mapsSearchMaxAttemptsForProxyMode(proxyMode) {
+  return proxyMode === "location" ? MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS : MAPS_SEARCH_MAX_ATTEMPTS;
+}
+function mapsSearchErrorMessage(err) {
+  return sanitizeVendorName(err instanceof Error ? err.message : String(err));
+}
+function looksLikeProxyTunnelFailure(message) {
+  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
+}
+function looksLikeProxyUnavailable(message) {
+  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
+}
+function looksLikeBrowserClosed(message) {
+  return /Target page, context or browser has been closed|browser has been closed|context has been closed|page has been closed|browser not found|CDP session closed|Connection closed|WebSocket.*closed|Protocol error.*closed|Browser closed|disconnected/i.test(message);
+}
+function mapsSearchFailureOutcome(err) {
+  if (err instanceof CaptchaError) return "captcha";
+  const message = mapsSearchErrorMessage(err);
+  if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return "timeout";
+  if (looksLikeProxyTunnelFailure(message)) return "proxy_tunnel_failed";
+  if (looksLikeProxyUnavailable(message)) return "proxy_unavailable";
+  if (looksLikeBrowserClosed(message)) return "browser_closed";
+  return "unknown_error";
 }
-async function cleanupDisposableProxy(kernelApiKey, proxyId) {
+function retryableMapsSearchError(err, proxyMode) {
+  if (err instanceof CaptchaError) return true;
+  const message = mapsSearchErrorMessage(err);
+  if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
+  if (looksLikeBrowserClosed(message)) return true;
+  return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
+}
+function suffix(value) {
+  return value ? value.slice(-8) : null;
+}
+function attemptFromDebug(input) {
+  const network = input.debug.networkLocation;
+  const browserSessionId = input.debug.kernel?.sessionId ?? null;
+  const target = input.resolution?.target ?? null;
+  return {
+    attemptNumber: input.attemptIndex + 1,
+    maxAttempts: input.maxAttempts,
+    status: input.status,
+    outcome: input.outcome,
+    willRetry: input.willRetry,
+    durationMs: input.durationMs,
+    resultCount: input.resultCount,
+    error: input.error,
+    proxyMode: input.proxyMode,
+    proxyResolutionSource: input.resolution?.source ?? null,
+    proxyIdSuffix: input.resolution?.proxyIdSuffix ?? null,
+    proxyTargetLevel: target?.level ?? null,
+    proxyTargetLocation: target?.canonicalLocation ?? null,
+    proxyTargetZip: target?.zip ?? null,
+    browserSessionIdSuffix: suffix(browserSessionId),
+    observedIp: network?.ip ?? null,
+    observedCity: network?.city ?? null,
+    observedRegion: network?.region ?? null
+  };
+}
+async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
   if (!kernelApiKey || !proxyId) return;
-  await deleteKernelProxyId(kernelApiKey, proxyId).catch((err) => {
+  try {
+    await deleteKernelProxyId(kernelApiKey, proxyId);
+  } catch (err) {
     console.warn(JSON.stringify({
-      event: "maps_search_proxy_delete_failed",
-      proxy_id_suffix: proxyId.slice(-6),
-      message: err instanceof Error ? err.message : String(err)
+      event: eventName,
+      proxy_id_suffix: suffix(proxyId),
+      message: mapsSearchErrorMessage(err)
     }));
-  });
+  }
+}
+async function runMapsSearchWithRotation(options) {
+  const attempts = [];
+  const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
+  const started = Date.now();
+  let lastError = null;
+  for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
+    const attemptStarted = Date.now();
+    const driver = new BrowserDriver();
+    const extractor = new MapsSearchExtractor(driver);
+    const proxyZip = options.proxyZipForAttempt?.(attemptIndex) ?? options.proxyZip;
+    let resolution = null;
+    let disposableProxyId;
+    try {
+      resolution = await resolveKernelProxyId({
+        kernelApiKey: options.kernelApiKey,
+        proxyMode: options.proxyMode,
+        configuredKernelProxyId: options.kernelProxyId ?? options.configuredKernelProxyId,
+        location: options.location,
+        proxyZip,
+        gl: options.gl,
+        attemptIndex,
+        fresh: options.proxyMode === "location"
+      });
+      disposableProxyId = resolution.disposableProxyId;
+      const result = await extractor.extract({
+        ...options,
+        proxyZip,
+        kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
+        kernelProxyResolution: resolution.resolution
+      });
+      const attempt = attemptFromDebug({
+        attemptIndex,
+        maxAttempts,
+        status: "ok",
+        outcome: "success",
+        willRetry: false,
+        durationMs: Date.now() - attemptStarted,
+        resultCount: result.resultCount,
+        error: null,
+        proxyMode: options.proxyMode,
+        resolution: resolution.resolution,
+        debug: driver.getDebugSnapshot()
+      });
+      attempts.push(attempt);
+      return {
+        ...result,
+        attempts: attempts.slice(),
+        durationMs: Date.now() - started
+      };
+    } catch (err) {
+      lastError = err;
+      const retryable = retryableMapsSearchError(err, options.proxyMode);
+      const willRetry = attemptIndex < maxAttempts - 1 && retryable;
+      const attempt = attemptFromDebug({
+        attemptIndex,
+        maxAttempts,
+        status: "failed",
+        outcome: mapsSearchFailureOutcome(err),
+        willRetry,
+        durationMs: Date.now() - attemptStarted,
+        resultCount: 0,
+        error: mapsSearchErrorMessage(err),
+        proxyMode: options.proxyMode,
+        resolution: resolution?.resolution ?? null,
+        debug: driver.getDebugSnapshot()
+      });
+      attempts.push(attempt);
+      console.warn(JSON.stringify({
+        event: options.failureEventName ?? "maps_search_attempt_failed",
+        ...options.logContext,
+        attempt_number: attempt.attemptNumber,
+        max_attempts: attempt.maxAttempts,
+        outcome: attempt.outcome,
+        will_retry: attempt.willRetry,
+        proxy_resolution_source: attempt.proxyResolutionSource,
+        proxy_id_suffix: attempt.proxyIdSuffix,
+        browser_session_id_suffix: attempt.browserSessionIdSuffix,
+        observed_city: attempt.observedCity,
+        observed_region: attempt.observedRegion,
+        message: attempt.error
+      }));
+      if (!willRetry) {
+        throw new MapsSearchRotationError(
+          mapsSearchErrorMessage(err),
+          attempts,
+          retryable,
+          err instanceof CaptchaError ? "captcha_or_blocked" : "maps_search_failed"
+        );
+      }
+    } finally {
+      await cleanupDisposableProxy(options.kernelApiKey, disposableProxyId, "maps_search_proxy_delete_failed");
+      await driver.close();
+    }
+  }
+  throw new MapsSearchRotationError(
+    lastError ? mapsSearchErrorMessage(lastError) : "Maps search failed",
+    attempts,
+    true
+  );
+}
+
+// src/api/maps-routes.ts
+function mapsErrorResponse(c, err, errorCode) {
+  const msg = err instanceof Error ? err.message : String(err);
+  const rotationError = err instanceof MapsSearchRotationError ? err : null;
+  const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
+  const retryable = rotationError?.retryable ?? blocked;
+  return c.json({
+    error: sanitizeVendorName(msg),
+    error_code: rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : errorCode),
+    retryable,
+    attempts: rotationError?.attempts ?? void 0
+  }, retryable ? 503 : 500);
 }
 var mapsApp = new Hono5();
 mapsApp.post("/search", createApiKeyAuth(), async (c) => {
@@ -9930,24 +10111,10 @@ mapsApp.post("/search", createApiKeyAuth(), async (c) => {
     [parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
   );
   if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
-  const driver = new BrowserDriver();
-  const extractor = new MapsSearchExtractor(driver);
-  let disposableProxyId;
   try {
-    const resolution = await resolveKernelProxyId({
-      kernelApiKey: parsed.data.kernelApiKey,
-      proxyMode: parsed.data.proxyMode,
-      configuredKernelProxyId: browserServiceProxyId(),
-      location: parsed.data.location,
-      proxyZip: parsed.data.proxyZip,
-      gl: parsed.data.gl,
-      fresh: parsed.data.proxyMode === "location"
-    });
-    disposableProxyId = resolution.disposableProxyId;
-    const result = await extractor.extract({
+    const result = await runMapsSearchWithRotation({
       ...parsed.data,
-      kernelProxyId: parsed.data.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
-      kernelProxyResolution: resolution.resolution
+      configuredKernelProxyId: browserServiceProxyId()
     });
     await logRequestEvent({
       userId: user.id,
@@ -9968,12 +10135,10 @@ mapsApp.post("/search", createApiKeyAuth(), async (c) => {
       status: "failed",
       query: [parsed.data.query, parsed.data.location].filter(Boolean).join(" "),
       location: parsed.data.location,
-      error: msg
+      error: msg,
+      result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
     });
-    return mapsErrorResponse(c, msg, "maps_search_failed");
-  } finally {
-    await cleanupDisposableProxy(parsed.data.kernelApiKey, disposableProxyId);
-    await driver.close();
+    return mapsErrorResponse(c, err, "maps_search_failed");
   }
 });
 mapsApp.post("/place", createApiKeyAuth(), async (c) => {
@@ -10278,8 +10443,6 @@ async function resolveDirectoryMarkets(options) {
 }
 
 // src/directory/directory-workflow.ts
-var DIRECTORY_MAX_ATTEMPTS = 3;
-var DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS = 5;
 var DirectoryWorkflowOptionsSchema = z15.object({
   query: z15.string().min(1),
   state: z15.string().min(2).default("TN"),
@@ -10299,36 +10462,9 @@ var DirectoryWorkflowOptionsSchema = z15.object({
   headless: z15.boolean().default(true),
   kernelApiKey: z15.string().optional()
 });
-async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
-  if (!kernelApiKey || !proxyId) return;
-  try {
-    await deleteKernelProxyId(kernelApiKey, proxyId);
-  } catch (err) {
-    console.warn(JSON.stringify({
-      event: "directory_workflow_proxy_delete_failed",
-      proxy_id_suffix: proxyId.slice(-6),
-      message: err instanceof Error ? err.message : String(err)
-    }));
-  }
-}
-function maxAttemptsForProxyMode(proxyMode) {
-  return proxyMode === "location" ? DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS : DIRECTORY_MAX_ATTEMPTS;
-}
 function errorMessage(err) {
   return err instanceof Error ? err.message : String(err);
 }
-function looksLikeProxyTunnelFailure(message) {
-  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
-}
-function looksLikeProxyUnavailable(message) {
-  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
-}
-function retryableCitySearchError(err, proxyMode) {
-  if (err instanceof CaptchaError) return true;
-  const message = errorMessage(err);
-  if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
-  return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
-}
 function proxyZipForAttempt(options, market, attemptIndex) {
   if (options.proxyZip) return options.proxyZip;
   if (!market.zips.length) return void 0;
@@ -10347,25 +10483,10 @@ async function mapLimit(items, limit, fn) {
   await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
   return out;
 }
-async function searchCityAttempt(options, market, attemptIndex) {
-  const driver = new BrowserDriver();
-  const extractor = new MapsSearchExtractor(driver);
-  const start = Date.now();
-  let disposableProxyId;
+async function searchCity(options, market) {
+  const started = Date.now();
   try {
-    const proxyZip = proxyZipForAttempt(options, market, attemptIndex);
-    const resolution = await resolveKernelProxyId({
-      kernelApiKey: options.kernelApiKey,
-      proxyMode: options.proxyMode,
-      configuredKernelProxyId: browserServiceProxyId(),
-      location: market.location,
-      proxyZip,
-      gl: options.gl,
-      attemptIndex,
-      fresh: options.proxyMode === "location"
-    });
-    disposableProxyId = resolution.disposableProxyId;
-    const result = await extractor.extract({
+    const result = await runMapsSearchWithRotation({
       query: options.query,
       location: market.location,
       gl: options.gl,
@@ -10373,11 +10494,16 @@ async function searchCityAttempt(options, market, attemptIndex) {
       maxResults: options.maxResultsPerCity,
       headless: options.headless,
       kernelApiKey: options.kernelApiKey,
-      kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
-      kernelProxyResolution: resolution.resolution,
       proxyMode: options.proxyMode,
-      proxyZip,
-      debug: options.debug
+      proxyZip: options.proxyZip,
+      debug: options.debug,
+      configuredKernelProxyId: browserServiceProxyId(),
+      proxyZipForAttempt: (attemptIndex) => proxyZipForAttempt(options, market, attemptIndex),
+      failureEventName: "directory_workflow_city_attempt_failed",
+      logContext: {
+        city: market.city,
+        state: market.state
+      }
     });
     return {
       city: market.city,
@@ -10393,50 +10519,29 @@ async function searchCityAttempt(options, market, attemptIndex) {
       error: null,
       resultCount: result.resultCount,
       durationMs: result.durationMs,
+      attempts: result.attempts ?? [],
       results: result.results
     };
-  } finally {
-    await cleanupDisposableProxy2(options.kernelApiKey, disposableProxyId);
-  }
-}
-async function searchCity(options, market) {
-  const started = Date.now();
-  const maxAttempts = maxAttemptsForProxyMode(options.proxyMode);
-  let lastError = null;
-  for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
-    try {
-      return await searchCityAttempt(options, market, attemptIndex);
-    } catch (err) {
-      lastError = err;
-      const willRetry = attemptIndex < maxAttempts - 1 && retryableCitySearchError(err, options.proxyMode);
-      console.warn(JSON.stringify({
-        event: "directory_workflow_city_attempt_failed",
-        city: market.city,
-        state: market.state,
-        attempt_number: attemptIndex + 1,
-        max_attempts: maxAttempts,
-        will_retry: willRetry,
-        message: errorMessage(err)
-      }));
-      if (!willRetry) break;
-    }
+  } catch (err) {
+    const attempts = err instanceof MapsSearchRotationError ? err.attempts : [];
+    return {
+      city: market.city,
+      state: market.state,
+      location: market.location,
+      cityKey: market.cityKey,
+      censusName: market.censusName,
+      population: market.population,
+      populationYear: market.populationYear,
+      zips: market.zips,
+      counties: market.counties,
+      status: "failed",
+      error: errorMessage(err),
+      resultCount: 0,
+      durationMs: Date.now() - started,
+      attempts,
+      results: []
+    };
   }
-  return {
-    city: market.city,
-    state: market.state,
-    location: market.location,
-    cityKey: market.cityKey,
-    censusName: market.censusName,
-    population: market.population,
-    populationYear: market.populationYear,
-    zips: market.zips,
-    counties: market.counties,
-    status: "failed",
-    error: lastError ? errorMessage(lastError) : "City Maps search failed",
-    resultCount: 0,
-    durationMs: Date.now() - started,
-    results: []
-  };
 }
 function csvRowsFor(result) {
   const rows = [];
@@ -11552,7 +11657,7 @@ var mcpApp = new Hono8();
 function registerSerpIntelligenceCaptureTools(server, executor) {
   server.registerTool("capture_serp_snapshot", {
     title: "SERP Intelligence Snapshot",
-    description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries and rejects wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
+    description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or location evidence is wrong. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
     inputSchema: CaptureSerpSnapshotInputSchema,
     annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
   }, async (input) => executor.captureSerpSnapshot(input));
@@ -13594,7 +13699,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-NAKGTIF5.js");
+  const { drainQueue } = await import("./worker-5O44YBF4.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const [results, sweepResult] = await Promise.all([
     drainQueue(budget),
@@ -13720,4 +13825,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-RIKRBDOI.js.map
+//# sourceMappingURL=server-BTTDFPSQ.js.map