mcp-scraper 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +13 -6
  2. package/dist/bin/api-server.cjs +582 -171
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/browser-agent-stdio-server.cjs +442 -9
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -2
  8. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +519 -31
  9. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
  11. package/dist/bin/mcp-scraper-install.cjs +2 -1
  12. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  13. package/dist/bin/mcp-scraper-install.js +2 -1
  14. package/dist/bin/mcp-scraper-install.js.map +1 -1
  15. package/dist/bin/mcp-stdio-server.cjs +68 -13
  16. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  17. package/dist/bin/mcp-stdio-server.js +2 -2
  18. package/dist/bin/paa-harvest.cjs +42 -6
  19. package/dist/bin/paa-harvest.cjs.map +1 -1
  20. package/dist/bin/paa-harvest.js +1 -1
  21. package/dist/chunk-5HMOPP76.js +793 -0
  22. package/dist/chunk-5HMOPP76.js.map +1 -0
  23. package/dist/{chunk-7WB2W6FO.js → chunk-6NEXSNSA.js} +69 -14
  24. package/dist/chunk-6NEXSNSA.js.map +1 -0
  25. package/dist/{chunk-MY3S7EX7.js → chunk-CQTAKXBN.js} +43 -7
  26. package/dist/chunk-CQTAKXBN.js.map +1 -0
  27. package/dist/chunk-I26QN7WQ.js +7 -0
  28. package/dist/chunk-I26QN7WQ.js.map +1 -0
  29. package/dist/index.cjs +42 -6
  30. package/dist/index.cjs.map +1 -1
  31. package/dist/index.js +1 -1
  32. package/dist/{server-MKCU3M7Y.js → server-BTTDFPSQ.js} +456 -142
  33. package/dist/server-BTTDFPSQ.js.map +1 -0
  34. package/dist/{worker-NAKGTIF5.js → worker-5O44YBF4.js} +2 -2
  35. package/docs/mcp-tool-quality-spec.md +3 -1
  36. package/package.json +1 -1
  37. package/dist/chunk-7WB2W6FO.js.map +0 -1
  38. package/dist/chunk-MY3S7EX7.js.map +0 -1
  39. package/dist/chunk-T6SIPPOQ.js +0 -7
  40. package/dist/chunk-T6SIPPOQ.js.map +0 -1
  41. package/dist/chunk-XL4V7VKD.js +0 -360
  42. package/dist/chunk-XL4V7VKD.js.map +0 -1
  43. package/dist/server-MKCU3M7Y.js.map +0 -1
  44. /package/dist/{worker-NAKGTIF5.js.map → worker-5O44YBF4.js.map} +0 -0
@@ -31,7 +31,7 @@ import {
31
31
  deleteKernelProxyId,
32
32
  harvest,
33
33
  resolveKernelProxyId
34
- } from "./chunk-MY3S7EX7.js";
34
+ } from "./chunk-CQTAKXBN.js";
35
35
  import {
36
36
  CaptureSerpPageSnapshotsInputSchema,
37
37
  CaptureSerpSnapshotInputSchema,
@@ -41,13 +41,13 @@ import {
41
41
  harvestTimeoutBudget,
42
42
  liveWebToolAnnotations,
43
43
  outputBaseDir
44
- } from "./chunk-7WB2W6FO.js";
44
+ } from "./chunk-6NEXSNSA.js";
45
45
  import {
46
46
  CaptchaError,
47
47
  RECAPTCHA_INSTRUCTIONS,
48
48
  sanitizeVendorName
49
49
  } from "./chunk-M2S27J6Z.js";
50
- import "./chunk-T6SIPPOQ.js";
50
+ import "./chunk-I26QN7WQ.js";
51
51
  import {
52
52
  SiteAuditJobRowSchema,
53
53
  cancelJob,
@@ -9893,24 +9893,205 @@ var MapsSearchExtractor = class {
9893
9893
  }
9894
9894
  };
9895
9895
 
9896
- // src/api/maps-routes.ts
9897
- function mapsErrorResponse(c, msg, errorCode) {
9898
- const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
9899
- return c.json({
9900
- error: sanitizeVendorName(msg),
9901
- error_code: blocked ? "captcha_or_blocked" : errorCode,
9902
- retryable: blocked
9903
- }, blocked ? 503 : 500);
9896
+ // src/maps-search-rotation.ts
9897
+ var MAPS_SEARCH_MAX_ATTEMPTS = 3;
9898
+ var MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS = 5;
9899
+ var MapsSearchRotationError = class extends Error {
9900
+ constructor(message, attempts, retryable, errorCode = "maps_search_failed") {
9901
+ super(message);
9902
+ this.attempts = attempts;
9903
+ this.retryable = retryable;
9904
+ this.errorCode = errorCode;
9905
+ }
9906
+ attempts;
9907
+ retryable;
9908
+ errorCode;
9909
+ name = "MapsSearchRotationError";
9910
+ };
9911
+ function mapsSearchMaxAttemptsForProxyMode(proxyMode) {
9912
+ return proxyMode === "location" ? MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS : MAPS_SEARCH_MAX_ATTEMPTS;
9913
+ }
9914
+ function mapsSearchErrorMessage(err) {
9915
+ return sanitizeVendorName(err instanceof Error ? err.message : String(err));
9916
+ }
9917
+ function looksLikeProxyTunnelFailure(message) {
9918
+ return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
9919
+ }
9920
+ function looksLikeProxyUnavailable(message) {
9921
+ return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
9922
+ }
9923
+ function looksLikeBrowserClosed(message) {
9924
+ return /Target page, context or browser has been closed|browser has been closed|context has been closed|page has been closed|browser not found|CDP session closed|Connection closed|WebSocket.*closed|Protocol error.*closed|Browser closed|disconnected/i.test(message);
9925
+ }
9926
+ function mapsSearchFailureOutcome(err) {
9927
+ if (err instanceof CaptchaError) return "captcha";
9928
+ const message = mapsSearchErrorMessage(err);
9929
+ if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return "timeout";
9930
+ if (looksLikeProxyTunnelFailure(message)) return "proxy_tunnel_failed";
9931
+ if (looksLikeProxyUnavailable(message)) return "proxy_unavailable";
9932
+ if (looksLikeBrowserClosed(message)) return "browser_closed";
9933
+ return "unknown_error";
9934
+ }
9935
+ function retryableMapsSearchError(err, proxyMode) {
9936
+ if (err instanceof CaptchaError) return true;
9937
+ const message = mapsSearchErrorMessage(err);
9938
+ if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
9939
+ if (looksLikeBrowserClosed(message)) return true;
9940
+ return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
9941
+ }
9942
+ function suffix(value) {
9943
+ return value ? value.slice(-8) : null;
9944
+ }
9945
+ function attemptFromDebug(input) {
9946
+ const network = input.debug.networkLocation;
9947
+ const browserSessionId = input.debug.kernel?.sessionId ?? null;
9948
+ const target = input.resolution?.target ?? null;
9949
+ return {
9950
+ attemptNumber: input.attemptIndex + 1,
9951
+ maxAttempts: input.maxAttempts,
9952
+ status: input.status,
9953
+ outcome: input.outcome,
9954
+ willRetry: input.willRetry,
9955
+ durationMs: input.durationMs,
9956
+ resultCount: input.resultCount,
9957
+ error: input.error,
9958
+ proxyMode: input.proxyMode,
9959
+ proxyResolutionSource: input.resolution?.source ?? null,
9960
+ proxyIdSuffix: input.resolution?.proxyIdSuffix ?? null,
9961
+ proxyTargetLevel: target?.level ?? null,
9962
+ proxyTargetLocation: target?.canonicalLocation ?? null,
9963
+ proxyTargetZip: target?.zip ?? null,
9964
+ browserSessionIdSuffix: suffix(browserSessionId),
9965
+ observedIp: network?.ip ?? null,
9966
+ observedCity: network?.city ?? null,
9967
+ observedRegion: network?.region ?? null
9968
+ };
9904
9969
  }
9905
- async function cleanupDisposableProxy(kernelApiKey, proxyId) {
9970
+ async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
9906
9971
  if (!kernelApiKey || !proxyId) return;
9907
- await deleteKernelProxyId(kernelApiKey, proxyId).catch((err) => {
9972
+ try {
9973
+ await deleteKernelProxyId(kernelApiKey, proxyId);
9974
+ } catch (err) {
9908
9975
  console.warn(JSON.stringify({
9909
- event: "maps_search_proxy_delete_failed",
9910
- proxy_id_suffix: proxyId.slice(-6),
9911
- message: err instanceof Error ? err.message : String(err)
9976
+ event: eventName,
9977
+ proxy_id_suffix: suffix(proxyId),
9978
+ message: mapsSearchErrorMessage(err)
9912
9979
  }));
9913
- });
9980
+ }
9981
+ }
9982
+ async function runMapsSearchWithRotation(options) {
9983
+ const attempts = [];
9984
+ const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
9985
+ const started = Date.now();
9986
+ let lastError = null;
9987
+ for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
9988
+ const attemptStarted = Date.now();
9989
+ const driver = new BrowserDriver();
9990
+ const extractor = new MapsSearchExtractor(driver);
9991
+ const proxyZip = options.proxyZipForAttempt?.(attemptIndex) ?? options.proxyZip;
9992
+ let resolution = null;
9993
+ let disposableProxyId;
9994
+ try {
9995
+ resolution = await resolveKernelProxyId({
9996
+ kernelApiKey: options.kernelApiKey,
9997
+ proxyMode: options.proxyMode,
9998
+ configuredKernelProxyId: options.kernelProxyId ?? options.configuredKernelProxyId,
9999
+ location: options.location,
10000
+ proxyZip,
10001
+ gl: options.gl,
10002
+ attemptIndex,
10003
+ fresh: options.proxyMode === "location"
10004
+ });
10005
+ disposableProxyId = resolution.disposableProxyId;
10006
+ const result = await extractor.extract({
10007
+ ...options,
10008
+ proxyZip,
10009
+ kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
10010
+ kernelProxyResolution: resolution.resolution
10011
+ });
10012
+ const attempt = attemptFromDebug({
10013
+ attemptIndex,
10014
+ maxAttempts,
10015
+ status: "ok",
10016
+ outcome: "success",
10017
+ willRetry: false,
10018
+ durationMs: Date.now() - attemptStarted,
10019
+ resultCount: result.resultCount,
10020
+ error: null,
10021
+ proxyMode: options.proxyMode,
10022
+ resolution: resolution.resolution,
10023
+ debug: driver.getDebugSnapshot()
10024
+ });
10025
+ attempts.push(attempt);
10026
+ return {
10027
+ ...result,
10028
+ attempts: attempts.slice(),
10029
+ durationMs: Date.now() - started
10030
+ };
10031
+ } catch (err) {
10032
+ lastError = err;
10033
+ const retryable = retryableMapsSearchError(err, options.proxyMode);
10034
+ const willRetry = attemptIndex < maxAttempts - 1 && retryable;
10035
+ const attempt = attemptFromDebug({
10036
+ attemptIndex,
10037
+ maxAttempts,
10038
+ status: "failed",
10039
+ outcome: mapsSearchFailureOutcome(err),
10040
+ willRetry,
10041
+ durationMs: Date.now() - attemptStarted,
10042
+ resultCount: 0,
10043
+ error: mapsSearchErrorMessage(err),
10044
+ proxyMode: options.proxyMode,
10045
+ resolution: resolution?.resolution ?? null,
10046
+ debug: driver.getDebugSnapshot()
10047
+ });
10048
+ attempts.push(attempt);
10049
+ console.warn(JSON.stringify({
10050
+ event: options.failureEventName ?? "maps_search_attempt_failed",
10051
+ ...options.logContext,
10052
+ attempt_number: attempt.attemptNumber,
10053
+ max_attempts: attempt.maxAttempts,
10054
+ outcome: attempt.outcome,
10055
+ will_retry: attempt.willRetry,
10056
+ proxy_resolution_source: attempt.proxyResolutionSource,
10057
+ proxy_id_suffix: attempt.proxyIdSuffix,
10058
+ browser_session_id_suffix: attempt.browserSessionIdSuffix,
10059
+ observed_city: attempt.observedCity,
10060
+ observed_region: attempt.observedRegion,
10061
+ message: attempt.error
10062
+ }));
10063
+ if (!willRetry) {
10064
+ throw new MapsSearchRotationError(
10065
+ mapsSearchErrorMessage(err),
10066
+ attempts,
10067
+ retryable,
10068
+ err instanceof CaptchaError ? "captcha_or_blocked" : "maps_search_failed"
10069
+ );
10070
+ }
10071
+ } finally {
10072
+ await cleanupDisposableProxy(options.kernelApiKey, disposableProxyId, "maps_search_proxy_delete_failed");
10073
+ await driver.close();
10074
+ }
10075
+ }
10076
+ throw new MapsSearchRotationError(
10077
+ lastError ? mapsSearchErrorMessage(lastError) : "Maps search failed",
10078
+ attempts,
10079
+ true
10080
+ );
10081
+ }
10082
+
10083
+ // src/api/maps-routes.ts
10084
+ function mapsErrorResponse(c, err, errorCode) {
10085
+ const msg = err instanceof Error ? err.message : String(err);
10086
+ const rotationError = err instanceof MapsSearchRotationError ? err : null;
10087
+ const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
10088
+ const retryable = rotationError?.retryable ?? blocked;
10089
+ return c.json({
10090
+ error: sanitizeVendorName(msg),
10091
+ error_code: rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : errorCode),
10092
+ retryable,
10093
+ attempts: rotationError?.attempts ?? void 0
10094
+ }, retryable ? 503 : 500);
9914
10095
  }
9915
10096
  var mapsApp = new Hono5();
9916
10097
  mapsApp.post("/search", createApiKeyAuth(), async (c) => {
@@ -9930,24 +10111,10 @@ mapsApp.post("/search", createApiKeyAuth(), async (c) => {
9930
10111
  [parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
9931
10112
  );
9932
10113
  if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
9933
- const driver = new BrowserDriver();
9934
- const extractor = new MapsSearchExtractor(driver);
9935
- let disposableProxyId;
9936
10114
  try {
9937
- const resolution = await resolveKernelProxyId({
9938
- kernelApiKey: parsed.data.kernelApiKey,
9939
- proxyMode: parsed.data.proxyMode,
9940
- configuredKernelProxyId: browserServiceProxyId(),
9941
- location: parsed.data.location,
9942
- proxyZip: parsed.data.proxyZip,
9943
- gl: parsed.data.gl,
9944
- fresh: parsed.data.proxyMode === "location"
9945
- });
9946
- disposableProxyId = resolution.disposableProxyId;
9947
- const result = await extractor.extract({
10115
+ const result = await runMapsSearchWithRotation({
9948
10116
  ...parsed.data,
9949
- kernelProxyId: parsed.data.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
9950
- kernelProxyResolution: resolution.resolution
10117
+ configuredKernelProxyId: browserServiceProxyId()
9951
10118
  });
9952
10119
  await logRequestEvent({
9953
10120
  userId: user.id,
@@ -9968,12 +10135,10 @@ mapsApp.post("/search", createApiKeyAuth(), async (c) => {
9968
10135
  status: "failed",
9969
10136
  query: [parsed.data.query, parsed.data.location].filter(Boolean).join(" "),
9970
10137
  location: parsed.data.location,
9971
- error: msg
10138
+ error: msg,
10139
+ result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
9972
10140
  });
9973
- return mapsErrorResponse(c, msg, "maps_search_failed");
9974
- } finally {
9975
- await cleanupDisposableProxy(parsed.data.kernelApiKey, disposableProxyId);
9976
- await driver.close();
10141
+ return mapsErrorResponse(c, err, "maps_search_failed");
9977
10142
  }
9978
10143
  });
9979
10144
  mapsApp.post("/place", createApiKeyAuth(), async (c) => {
@@ -10278,8 +10443,6 @@ async function resolveDirectoryMarkets(options) {
10278
10443
  }
10279
10444
 
10280
10445
  // src/directory/directory-workflow.ts
10281
- var DIRECTORY_MAX_ATTEMPTS = 3;
10282
- var DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS = 5;
10283
10446
  var DirectoryWorkflowOptionsSchema = z15.object({
10284
10447
  query: z15.string().min(1),
10285
10448
  state: z15.string().min(2).default("TN"),
@@ -10299,36 +10462,9 @@ var DirectoryWorkflowOptionsSchema = z15.object({
10299
10462
  headless: z15.boolean().default(true),
10300
10463
  kernelApiKey: z15.string().optional()
10301
10464
  });
10302
- async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
10303
- if (!kernelApiKey || !proxyId) return;
10304
- try {
10305
- await deleteKernelProxyId(kernelApiKey, proxyId);
10306
- } catch (err) {
10307
- console.warn(JSON.stringify({
10308
- event: "directory_workflow_proxy_delete_failed",
10309
- proxy_id_suffix: proxyId.slice(-6),
10310
- message: err instanceof Error ? err.message : String(err)
10311
- }));
10312
- }
10313
- }
10314
- function maxAttemptsForProxyMode(proxyMode) {
10315
- return proxyMode === "location" ? DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS : DIRECTORY_MAX_ATTEMPTS;
10316
- }
10317
10465
  function errorMessage(err) {
10318
10466
  return err instanceof Error ? err.message : String(err);
10319
10467
  }
10320
- function looksLikeProxyTunnelFailure(message) {
10321
- return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
10322
- }
10323
- function looksLikeProxyUnavailable(message) {
10324
- return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
10325
- }
10326
- function retryableCitySearchError(err, proxyMode) {
10327
- if (err instanceof CaptchaError) return true;
10328
- const message = errorMessage(err);
10329
- if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
10330
- return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
10331
- }
10332
10468
  function proxyZipForAttempt(options, market, attemptIndex) {
10333
10469
  if (options.proxyZip) return options.proxyZip;
10334
10470
  if (!market.zips.length) return void 0;
@@ -10347,25 +10483,10 @@ async function mapLimit(items, limit, fn) {
10347
10483
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
10348
10484
  return out;
10349
10485
  }
10350
- async function searchCityAttempt(options, market, attemptIndex) {
10351
- const driver = new BrowserDriver();
10352
- const extractor = new MapsSearchExtractor(driver);
10353
- const start = Date.now();
10354
- let disposableProxyId;
10486
+ async function searchCity(options, market) {
10487
+ const started = Date.now();
10355
10488
  try {
10356
- const proxyZip = proxyZipForAttempt(options, market, attemptIndex);
10357
- const resolution = await resolveKernelProxyId({
10358
- kernelApiKey: options.kernelApiKey,
10359
- proxyMode: options.proxyMode,
10360
- configuredKernelProxyId: browserServiceProxyId(),
10361
- location: market.location,
10362
- proxyZip,
10363
- gl: options.gl,
10364
- attemptIndex,
10365
- fresh: options.proxyMode === "location"
10366
- });
10367
- disposableProxyId = resolution.disposableProxyId;
10368
- const result = await extractor.extract({
10489
+ const result = await runMapsSearchWithRotation({
10369
10490
  query: options.query,
10370
10491
  location: market.location,
10371
10492
  gl: options.gl,
@@ -10373,11 +10494,16 @@ async function searchCityAttempt(options, market, attemptIndex) {
10373
10494
  maxResults: options.maxResultsPerCity,
10374
10495
  headless: options.headless,
10375
10496
  kernelApiKey: options.kernelApiKey,
10376
- kernelProxyId: options.proxyMode === "none" ? void 0 : resolution.kernelProxyId,
10377
- kernelProxyResolution: resolution.resolution,
10378
10497
  proxyMode: options.proxyMode,
10379
- proxyZip,
10380
- debug: options.debug
10498
+ proxyZip: options.proxyZip,
10499
+ debug: options.debug,
10500
+ configuredKernelProxyId: browserServiceProxyId(),
10501
+ proxyZipForAttempt: (attemptIndex) => proxyZipForAttempt(options, market, attemptIndex),
10502
+ failureEventName: "directory_workflow_city_attempt_failed",
10503
+ logContext: {
10504
+ city: market.city,
10505
+ state: market.state
10506
+ }
10381
10507
  });
10382
10508
  return {
10383
10509
  city: market.city,
@@ -10393,50 +10519,29 @@ async function searchCityAttempt(options, market, attemptIndex) {
10393
10519
  error: null,
10394
10520
  resultCount: result.resultCount,
10395
10521
  durationMs: result.durationMs,
10522
+ attempts: result.attempts ?? [],
10396
10523
  results: result.results
10397
10524
  };
10398
- } finally {
10399
- await cleanupDisposableProxy2(options.kernelApiKey, disposableProxyId);
10400
- }
10401
- }
10402
- async function searchCity(options, market) {
10403
- const started = Date.now();
10404
- const maxAttempts = maxAttemptsForProxyMode(options.proxyMode);
10405
- let lastError = null;
10406
- for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
10407
- try {
10408
- return await searchCityAttempt(options, market, attemptIndex);
10409
- } catch (err) {
10410
- lastError = err;
10411
- const willRetry = attemptIndex < maxAttempts - 1 && retryableCitySearchError(err, options.proxyMode);
10412
- console.warn(JSON.stringify({
10413
- event: "directory_workflow_city_attempt_failed",
10414
- city: market.city,
10415
- state: market.state,
10416
- attempt_number: attemptIndex + 1,
10417
- max_attempts: maxAttempts,
10418
- will_retry: willRetry,
10419
- message: errorMessage(err)
10420
- }));
10421
- if (!willRetry) break;
10422
- }
10525
+ } catch (err) {
10526
+ const attempts = err instanceof MapsSearchRotationError ? err.attempts : [];
10527
+ return {
10528
+ city: market.city,
10529
+ state: market.state,
10530
+ location: market.location,
10531
+ cityKey: market.cityKey,
10532
+ censusName: market.censusName,
10533
+ population: market.population,
10534
+ populationYear: market.populationYear,
10535
+ zips: market.zips,
10536
+ counties: market.counties,
10537
+ status: "failed",
10538
+ error: errorMessage(err),
10539
+ resultCount: 0,
10540
+ durationMs: Date.now() - started,
10541
+ attempts,
10542
+ results: []
10543
+ };
10423
10544
  }
10424
- return {
10425
- city: market.city,
10426
- state: market.state,
10427
- location: market.location,
10428
- cityKey: market.cityKey,
10429
- censusName: market.censusName,
10430
- population: market.population,
10431
- populationYear: market.populationYear,
10432
- zips: market.zips,
10433
- counties: market.counties,
10434
- status: "failed",
10435
- error: lastError ? errorMessage(lastError) : "City Maps search failed",
10436
- resultCount: 0,
10437
- durationMs: Date.now() - started,
10438
- results: []
10439
- };
10440
10545
  }
10441
10546
  function csvRowsFor(result) {
10442
10547
  const rows = [];
@@ -11552,7 +11657,7 @@ var mcpApp = new Hono8();
11552
11657
  function registerSerpIntelligenceCaptureTools(server, executor) {
11553
11658
  server.registerTool("capture_serp_snapshot", {
11554
11659
  title: "SERP Intelligence Snapshot",
11555
- description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries and rejects wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
11660
+ description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or location evidence is wrong. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
11556
11661
  inputSchema: CaptureSerpSnapshotInputSchema,
11557
11662
  annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
11558
11663
  }, async (input) => executor.captureSerpSnapshot(input));
@@ -11710,18 +11815,26 @@ async function recordAction(input) {
11710
11815
  async function recordReplayStart(input) {
11711
11816
  const db = getDb();
11712
11817
  await db.execute({
11713
- sql: `INSERT INTO browser_agent_replays (replay_id, session_id, view_url, label)
11714
- VALUES (?, ?, ?, ?)`,
11715
- args: [input.replayId, input.sessionId, input.viewUrl, input.label]
11818
+ sql: `INSERT INTO browser_agent_replays (replay_id, session_id, view_url, label, started_at)
11819
+ VALUES (?, ?, ?, ?, ?)`,
11820
+ args: [input.replayId, input.sessionId, input.viewUrl, input.label, (/* @__PURE__ */ new Date()).toISOString()]
11716
11821
  });
11717
11822
  }
11718
11823
  async function recordReplayStop(replayId, viewUrl) {
11719
11824
  const db = getDb();
11720
11825
  await db.execute({
11721
- sql: `UPDATE browser_agent_replays SET stopped_at = datetime('now'), view_url = COALESCE(?, view_url) WHERE replay_id = ?`,
11722
- args: [viewUrl, replayId]
11826
+ sql: `UPDATE browser_agent_replays SET stopped_at = ?, view_url = COALESCE(?, view_url) WHERE replay_id = ?`,
11827
+ args: [(/* @__PURE__ */ new Date()).toISOString(), viewUrl, replayId]
11723
11828
  });
11724
11829
  }
11830
+ async function getActiveReplayRow(sessionId) {
11831
+ const db = getDb();
11832
+ const res = await db.execute({
11833
+ sql: `SELECT * FROM browser_agent_replays WHERE session_id = ? AND stopped_at IS NULL ORDER BY started_at DESC LIMIT 1`,
11834
+ args: [sessionId]
11835
+ });
11836
+ return res.rows[0] ?? null;
11837
+ }
11725
11838
  async function listReplayRows(sessionId) {
11726
11839
  const db = getDb();
11727
11840
  const res = await db.execute({
@@ -11835,7 +11948,13 @@ async function readPage(cdpWsUrl) {
11835
11948
  role: el2.getAttribute("role") || el2.tagName.toLowerCase(),
11836
11949
  name,
11837
11950
  x: Math.round(r.left + r.width / 2),
11838
- y: Math.round(r.top + r.height / 2)
11951
+ y: Math.round(r.top + r.height / 2),
11952
+ left: Math.round(r.left),
11953
+ top: Math.round(r.top),
11954
+ width: Math.round(r.width),
11955
+ height: Math.round(r.height),
11956
+ right: Math.round(r.right),
11957
+ bottom: Math.round(r.bottom)
11839
11958
  });
11840
11959
  }
11841
11960
  const text = (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").trim().slice(0, 6e3);
@@ -11845,13 +11964,168 @@ async function readPage(cdpWsUrl) {
11845
11964
  url,
11846
11965
  title,
11847
11966
  text: data.text,
11848
- elements: data.els.map((e, i) => ({ ref: i + 1, role: e.role, name: e.name, x: e.x, y: e.y }))
11967
+ elements: data.els.map((e, i) => ({
11968
+ ref: i + 1,
11969
+ role: e.role,
11970
+ name: e.name,
11971
+ x: e.x,
11972
+ y: e.y,
11973
+ left: e.left,
11974
+ top: e.top,
11975
+ width: e.width,
11976
+ height: e.height,
11977
+ right: e.right,
11978
+ bottom: e.bottom
11979
+ }))
11849
11980
  };
11850
11981
  } finally {
11851
11982
  await browser.close().catch(() => {
11852
11983
  });
11853
11984
  }
11854
11985
  }
11986
+ async function locatePageTargets(cdpWsUrl, targets) {
11987
+ const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
11988
+ try {
11989
+ const context = browser.contexts()[0] ?? await browser.newContext();
11990
+ const page = context.pages()[0] ?? await context.newPage();
11991
+ const url = page.url();
11992
+ const title = await page.title().catch(() => "");
11993
+ await page.evaluate(() => {
11994
+ ;
11995
+ globalThis.__name = (target) => target;
11996
+ }).catch(() => {
11997
+ });
11998
+ const data = await page.evaluate((rawTargets) => {
11999
+ const normalize = (value) => value.replace(/\s+/g, " ").trim();
12000
+ const isVisibleRect = (r) => r.width >= 1 && r.height >= 1 && r.bottom >= 0 && r.right >= 0 && r.top <= window.innerHeight && r.left <= window.innerWidth;
12001
+ const roleOf = (el2) => el2.getAttribute("role") || el2.tagName.toLowerCase();
12002
+ const nameOf = (el2) => {
12003
+ const e = el2;
12004
+ return normalize(
12005
+ e.getAttribute("aria-label") || e.placeholder || e.innerText || e.getAttribute("value") || e.getAttribute("title") || ""
12006
+ ).slice(0, 160);
12007
+ };
12008
+ const textOf = (el2) => normalize(el2.innerText || el2.textContent || "").slice(0, 500);
12009
+ const unionRects = (rects) => {
12010
+ const visible = rects.filter(isVisibleRect);
12011
+ if (!visible.length) return null;
12012
+ const left = Math.min(...visible.map((r) => r.left));
12013
+ const top = Math.min(...visible.map((r) => r.top));
12014
+ const right = Math.max(...visible.map((r) => r.right));
12015
+ const bottom = Math.max(...visible.map((r) => r.bottom));
12016
+ return {
12017
+ left,
12018
+ top,
12019
+ right,
12020
+ bottom,
12021
+ width: right - left,
12022
+ height: bottom - top,
12023
+ x: left,
12024
+ y: top,
12025
+ toJSON: () => ({})
12026
+ };
12027
+ };
12028
+ const elementPayload = (el2, rect) => ({
12029
+ role: roleOf(el2),
12030
+ tag: el2.tagName.toLowerCase(),
12031
+ name: nameOf(el2),
12032
+ text: textOf(el2),
12033
+ x: Math.round(rect.left + rect.width / 2),
12034
+ y: Math.round(rect.top + rect.height / 2),
12035
+ left: Math.round(rect.left),
12036
+ top: Math.round(rect.top),
12037
+ width: Math.round(rect.width),
12038
+ height: Math.round(rect.height),
12039
+ right: Math.round(rect.right),
12040
+ bottom: Math.round(rect.bottom)
12041
+ });
12042
+ const selectorMatches = (selector) => {
12043
+ const nodes = Array.from(document.querySelectorAll(selector));
12044
+ return nodes.map((el2) => ({ el: el2, rect: el2.getBoundingClientRect() })).filter((item) => isVisibleRect(item.rect));
12045
+ };
12046
+ const textMatches = (needle, match) => {
12047
+ const wanted = normalize(needle);
12048
+ const wantedLower = wanted.toLowerCase();
12049
+ if (!wantedLower) return [];
12050
+ const matches = [];
12051
+ const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
12052
+ let node = walker.nextNode();
12053
+ while (node) {
12054
+ const raw = node.textContent || "";
12055
+ const normalizedRaw = normalize(raw);
12056
+ const rawLower = raw.toLowerCase();
12057
+ const normalizedLower = normalizedRaw.toLowerCase();
12058
+ const ok = match === "exact" ? normalizedLower === wantedLower : normalizedLower.includes(wantedLower);
12059
+ if (ok) {
12060
+ const start = match === "exact" ? 0 : rawLower.indexOf(wantedLower);
12061
+ const parent = node.parentElement;
12062
+ if (parent && start >= 0) {
12063
+ try {
12064
+ const range = document.createRange();
12065
+ range.setStart(node, Math.min(start, raw.length));
12066
+ range.setEnd(node, Math.min(raw.length, start + needle.length));
12067
+ const rect = unionRects(Array.from(range.getClientRects()));
12068
+ range.detach();
12069
+ if (rect) matches.push({ el: parent, rect });
12070
+ } catch {
12071
+ const rect = parent.getBoundingClientRect();
12072
+ if (isVisibleRect(rect)) matches.push({ el: parent, rect });
12073
+ }
12074
+ }
12075
+ }
12076
+ node = walker.nextNode();
12077
+ }
12078
+ if (matches.length) return matches;
12079
+ const fallback = Array.from(document.querySelectorAll("body *")).map((el2) => ({ el: el2, rect: el2.getBoundingClientRect(), text: textOf(el2) })).filter((item) => isVisibleRect(item.rect)).filter((item) => {
12080
+ const value = item.text.toLowerCase();
12081
+ return match === "exact" ? value === wantedLower : value.includes(wantedLower);
12082
+ }).sort((a, b) => a.rect.width * a.rect.height - b.rect.width * b.rect.height);
12083
+ return fallback.map(({ el: el2, rect }) => ({ el: el2, rect }));
12084
+ };
12085
+ const viewport = {
12086
+ width: window.innerWidth,
12087
+ height: window.innerHeight,
12088
+ devicePixelRatio: window.devicePixelRatio || 1
12089
+ };
12090
+ const results = rawTargets.map((target) => {
12091
+ const index = Math.max(0, Math.floor(target.index ?? 0));
12092
+ const match = target.match === "exact" ? "exact" : "contains";
12093
+ try {
12094
+ const selector = target.selector?.trim();
12095
+ const text = target.text?.trim();
12096
+ const candidates = selector ? selectorMatches(selector) : text ? textMatches(text, match) : [];
12097
+ const selected = candidates[index];
12098
+ return {
12099
+ name: target.name ?? null,
12100
+ selector: selector || null,
12101
+ text: text || null,
12102
+ match,
12103
+ index,
12104
+ found: Boolean(selected),
12105
+ element: selected ? elementPayload(selected.el, selected.rect) : null,
12106
+ ...selected ? {} : { error: "target not found in current viewport" }
12107
+ };
12108
+ } catch (err) {
12109
+ return {
12110
+ name: target.name ?? null,
12111
+ selector: target.selector ?? null,
12112
+ text: target.text ?? null,
12113
+ match,
12114
+ index,
12115
+ found: false,
12116
+ element: null,
12117
+ error: err instanceof Error ? err.message : String(err)
12118
+ };
12119
+ }
12120
+ });
12121
+ return { viewport, targets: results };
12122
+ }, targets);
12123
+ return { url, title, viewport: data.viewport, targets: data.targets };
12124
+ } finally {
12125
+ await browser.close().catch(() => {
12126
+ });
12127
+ }
12128
+ }
11855
12129
  async function replayStart(runtimeSessionId) {
11856
12130
  const k = client();
11857
12131
  const res = await k.browsers.replays.start(runtimeSessionId);
@@ -11905,6 +12179,22 @@ function failure(err) {
11905
12179
  const msg = err instanceof Error ? err.message : String(err);
11906
12180
  return { error: sanitizeVendorName(msg) };
11907
12181
  }
12182
+ function parseAgentDate(value) {
12183
+ if (!value) return null;
12184
+ const parsed = Date.parse(value);
12185
+ if (Number.isFinite(parsed)) return parsed;
12186
+ const legacyParsed = Date.parse(`${value.replace(" ", "T")}Z`);
12187
+ return Number.isFinite(legacyParsed) ? legacyParsed : null;
12188
+ }
12189
+ function replayClock(row) {
12190
+ if (!row) return null;
12191
+ const startedAtMs = parseAgentDate(row.started_at);
12192
+ return {
12193
+ replay_id: row.replay_id,
12194
+ started_at: row.started_at,
12195
+ replay_elapsed_seconds: startedAtMs == null ? null : Math.max(0, (Date.now() - startedAtMs) / 1e3)
12196
+ };
12197
+ }
11908
12198
  function replayDownloadUrl(sessionId, replayId) {
11909
12199
  return `/agent/sessions/${encodeURIComponent(sessionId)}/replays/${encodeURIComponent(replayId)}/download`;
11910
12200
  }
@@ -12015,13 +12305,15 @@ function buildBrowserAgentRoutes() {
12015
12305
  }
12016
12306
  await charge(row.id, user.id, t0);
12017
12307
  await recordAction({ sessionId: row.id, type: "screenshot", params: null, ok: true });
12308
+ const activeReplay = await getActiveReplayRow(row.id);
12018
12309
  return c.json({
12019
12310
  image_base64: shot.base64,
12020
12311
  mime_type: shot.mimeType,
12021
12312
  url: page?.url ?? null,
12022
12313
  title: page?.title ?? null,
12023
12314
  elements: page?.elements ?? [],
12024
- text: page?.text ?? null
12315
+ text: page?.text ?? null,
12316
+ replay: replayClock(activeReplay)
12025
12317
  });
12026
12318
  } catch (err) {
12027
12319
  await charge(row.id, user.id, t0);
@@ -12038,7 +12330,8 @@ function buildBrowserAgentRoutes() {
12038
12330
  const page = await readPage(row.cdp_ws_url);
12039
12331
  await charge(row.id, user.id, t0);
12040
12332
  await recordAction({ sessionId: row.id, type: "read", params: null, ok: true });
12041
- return c.json(page);
12333
+ const activeReplay = await getActiveReplayRow(row.id);
12334
+ return c.json({ ...page, replay: replayClock(activeReplay) });
12042
12335
  } catch (err) {
12043
12336
  await charge(row.id, user.id, t0);
12044
12337
  return c.json(failure(err), 502);
@@ -12067,6 +12360,27 @@ function buildBrowserAgentRoutes() {
12067
12360
  return c.json(failure(err), 502);
12068
12361
  }
12069
12362
  });
12363
+ app2.post("/sessions/:id/locate", async (c) => {
12364
+ const user = c.get("user");
12365
+ const row = await loadOpenSession(c.req.param("id"), user.id);
12366
+ if (!row) return c.json({ error: "not found" }, 404);
12367
+ const body = await c.req.json().catch(() => ({}));
12368
+ const rawTargets = Array.isArray(body.targets) ? body.targets : [];
12369
+ const targets = rawTargets.filter((value) => value && typeof value === "object").slice(0, 20).map((value) => value);
12370
+ if (!targets.length) return c.json({ error: "targets is required" }, 400);
12371
+ const t0 = Date.now();
12372
+ try {
12373
+ const result = await locatePageTargets(row.cdp_ws_url, targets);
12374
+ await charge(row.id, user.id, t0);
12375
+ await recordAction({ sessionId: row.id, type: "locate", params: { targets: targets.length }, ok: true });
12376
+ const activeReplay = await getActiveReplayRow(row.id);
12377
+ return c.json({ ...result, replay: replayClock(activeReplay) });
12378
+ } catch (err) {
12379
+ await charge(row.id, user.id, t0);
12380
+ await recordAction({ sessionId: row.id, type: "locate", params: { targets: targets.length }, ok: false, error: String(err) });
12381
+ return c.json(failure(err), 502);
12382
+ }
12383
+ });
12070
12384
  app2.post("/sessions/:id/type", async (c) => {
12071
12385
  const user = c.get("user");
12072
12386
  const row = await loadOpenSession(c.req.param("id"), user.id);
@@ -13385,7 +13699,7 @@ app.get("/cron/tick", async (c) => {
13385
13699
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
13386
13700
  return c.json({ error: "Unauthorized" }, 401);
13387
13701
  }
13388
- const { drainQueue } = await import("./worker-NAKGTIF5.js");
13702
+ const { drainQueue } = await import("./worker-5O44YBF4.js");
13389
13703
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
13390
13704
  const [results, sweepResult] = await Promise.all([
13391
13705
  drainQueue(budget),
@@ -13511,4 +13825,4 @@ app.get("/blog/:slug/", (c) => {
13511
13825
  export {
13512
13826
  app
13513
13827
  };
13514
- //# sourceMappingURL=server-MKCU3M7Y.js.map
13828
+ //# sourceMappingURL=server-BTTDFPSQ.js.map