mcp-scraper 0.2.7 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/bin/api-server.cjs +364 -162
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +68 -13
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +2 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -1
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +68 -13
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +42 -6
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-3TF6UT2P.js → chunk-5HMOPP76.js} +2 -2
- package/dist/{chunk-7SI6XIR3.js → chunk-6NEXSNSA.js} +69 -14
- package/dist/chunk-6NEXSNSA.js.map +1 -0
- package/dist/{chunk-MY3S7EX7.js → chunk-CQTAKXBN.js} +43 -7
- package/dist/chunk-CQTAKXBN.js.map +1 -0
- package/dist/chunk-I26QN7WQ.js +7 -0
- package/dist/chunk-I26QN7WQ.js.map +1 -0
- package/dist/index.cjs +42 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-RIKRBDOI.js → server-BTTDFPSQ.js} +238 -133
- package/dist/server-BTTDFPSQ.js.map +1 -0
- package/dist/{worker-NAKGTIF5.js → worker-5O44YBF4.js} +2 -2
- package/docs/mcp-tool-quality-spec.md +3 -1
- package/package.json +1 -1
- package/dist/chunk-7SI6XIR3.js.map +0 -1
- package/dist/chunk-IZE5UW7Y.js +0 -7
- package/dist/chunk-IZE5UW7Y.js.map +0 -1
- package/dist/chunk-MY3S7EX7.js.map +0 -1
- package/dist/server-RIKRBDOI.js.map +0 -1
- /package/dist/{chunk-3TF6UT2P.js.map → chunk-5HMOPP76.js.map} +0 -0
- /package/dist/{worker-NAKGTIF5.js.map → worker-5O44YBF4.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -8682,7 +8682,7 @@ function buildYouTubeChannelVideosUrl(channelInput) {
|
|
|
8682
8682
|
}
|
|
8683
8683
|
return `https://www.youtube.com/${handle}/videos`;
|
|
8684
8684
|
}
|
|
8685
|
-
var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright3, import_sdk4, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
|
|
8685
|
+
var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright3, import_sdk4, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS, SERP_CHALLENGE_SOLVE_POLL_MS, BrowserDriver;
|
|
8686
8686
|
var init_BrowserDriver = __esm({
|
|
8687
8687
|
"src/driver/BrowserDriver.ts"() {
|
|
8688
8688
|
"use strict";
|
|
@@ -8698,6 +8698,8 @@ var init_BrowserDriver = __esm({
|
|
|
8698
8698
|
DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
|
|
8699
8699
|
KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
|
|
8700
8700
|
KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
|
|
8701
|
+
DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS = 2500;
|
|
8702
|
+
SERP_CHALLENGE_SOLVE_POLL_MS = 500;
|
|
8701
8703
|
BrowserDriver = class {
|
|
8702
8704
|
browser = null;
|
|
8703
8705
|
context = null;
|
|
@@ -8947,20 +8949,20 @@ var init_BrowserDriver = __esm({
|
|
|
8947
8949
|
const diag = await this.captureDiagnostics(url);
|
|
8948
8950
|
throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
|
|
8949
8951
|
}
|
|
8950
|
-
|
|
8951
|
-
if (captchaCount > 0) {
|
|
8952
|
+
if (await this.isSerpChallengeVisible()) {
|
|
8952
8953
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
|
|
8953
|
-
|
|
8954
|
+
const solved = await this.waitForSerpChallengeToClear(navDebug, url);
|
|
8955
|
+
if (!solved) throw new CaptchaError(this.captchaMessage());
|
|
8954
8956
|
}
|
|
8955
8957
|
const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
|
|
8956
8958
|
if (fastFound) {
|
|
8957
8959
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: true, captchaDetected: false });
|
|
8958
8960
|
return { hasPaa: true };
|
|
8959
8961
|
}
|
|
8960
|
-
|
|
8961
|
-
if (captchaAfter > 0) {
|
|
8962
|
+
if (await this.isSerpChallengeVisible()) {
|
|
8962
8963
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
|
|
8963
|
-
|
|
8964
|
+
const solved = await this.waitForSerpChallengeToClear(navDebug, url);
|
|
8965
|
+
if (!solved) throw new CaptchaError(this.captchaMessage());
|
|
8964
8966
|
}
|
|
8965
8967
|
for (let i = 1; i <= 6; i++) {
|
|
8966
8968
|
await this.page.evaluate((f) => {
|
|
@@ -8976,6 +8978,35 @@ var init_BrowserDriver = __esm({
|
|
|
8976
8978
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: false });
|
|
8977
8979
|
return { hasPaa: false };
|
|
8978
8980
|
}
|
|
8981
|
+
async isSerpChallengeVisible() {
|
|
8982
|
+
if (!this.page) return false;
|
|
8983
|
+
const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count().catch(() => 0);
|
|
8984
|
+
if (captchaCount > 0) return true;
|
|
8985
|
+
const finalUrl = this.page.url();
|
|
8986
|
+
if (/google\.[^/]+\/sorry\//i.test(finalUrl)) return true;
|
|
8987
|
+
const bodySnippet = await this.page.evaluate(() => {
|
|
8988
|
+
const text = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
|
|
8989
|
+
return text.slice(0, 1e3);
|
|
8990
|
+
}).catch(() => "");
|
|
8991
|
+
return /recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(bodySnippet);
|
|
8992
|
+
}
|
|
8993
|
+
async waitForSerpChallengeToClear(navDebug, requestedUrl) {
|
|
8994
|
+
if (!this.kernelClient || !this.page) return false;
|
|
8995
|
+
const waitMs = positiveIntFromEnv("SERP_CHALLENGE_SOLVE_WAIT_MS", DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS);
|
|
8996
|
+
const deadline = Date.now() + waitMs;
|
|
8997
|
+
while (Date.now() < deadline) {
|
|
8998
|
+
await this.page.waitForTimeout(Math.min(SERP_CHALLENGE_SOLVE_POLL_MS, Math.max(250, deadline - Date.now())));
|
|
8999
|
+
if (!await this.isSerpChallengeVisible()) {
|
|
9000
|
+
await this.page.waitForTimeout(1e3);
|
|
9001
|
+
if (!await this.isSerpChallengeVisible()) {
|
|
9002
|
+
await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: null, captchaDetected: false });
|
|
9003
|
+
return true;
|
|
9004
|
+
}
|
|
9005
|
+
}
|
|
9006
|
+
}
|
|
9007
|
+
await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: false, captchaDetected: true });
|
|
9008
|
+
return false;
|
|
9009
|
+
}
|
|
8979
9010
|
async updateSerpNavigationDebug(navDebug, requestedUrl, state) {
|
|
8980
9011
|
if (!navDebug || !this.page) return;
|
|
8981
9012
|
try {
|
|
@@ -9023,7 +9054,12 @@ var init_BrowserDriver = __esm({
|
|
|
9023
9054
|
async navigateTo(url) {
|
|
9024
9055
|
try {
|
|
9025
9056
|
await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
9057
|
+
if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
|
|
9058
|
+
const solved = await this.waitForSerpChallengeToClear(null, url);
|
|
9059
|
+
if (!solved) throw new CaptchaError(this.captchaMessage());
|
|
9060
|
+
}
|
|
9026
9061
|
} catch (err) {
|
|
9062
|
+
if (err instanceof CaptchaError) throw err;
|
|
9027
9063
|
const diag = await this.captureDiagnostics(url);
|
|
9028
9064
|
throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
|
|
9029
9065
|
}
|
|
@@ -12584,24 +12620,215 @@ var init_MapsSearchExtractor = __esm({
|
|
|
12584
12620
|
}
|
|
12585
12621
|
});
|
|
12586
12622
|
|
|
12587
|
-
// src/
|
|
12588
|
-
function
|
|
12589
|
-
|
|
12590
|
-
|
|
12591
|
-
|
|
12592
|
-
|
|
12593
|
-
|
|
12594
|
-
|
|
12623
|
+
// src/maps-search-rotation.ts
|
|
12624
|
+
function mapsSearchMaxAttemptsForProxyMode(proxyMode) {
|
|
12625
|
+
return proxyMode === "location" ? MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS : MAPS_SEARCH_MAX_ATTEMPTS;
|
|
12626
|
+
}
|
|
12627
|
+
function mapsSearchErrorMessage(err) {
|
|
12628
|
+
return sanitizeVendorName(err instanceof Error ? err.message : String(err));
|
|
12629
|
+
}
|
|
12630
|
+
function looksLikeProxyTunnelFailure(message) {
|
|
12631
|
+
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
12632
|
+
}
|
|
12633
|
+
function looksLikeProxyUnavailable(message) {
|
|
12634
|
+
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
12635
|
+
}
|
|
12636
|
+
function looksLikeBrowserClosed(message) {
|
|
12637
|
+
return /Target page, context or browser has been closed|browser has been closed|context has been closed|page has been closed|browser not found|CDP session closed|Connection closed|WebSocket.*closed|Protocol error.*closed|Browser closed|disconnected/i.test(message);
|
|
12638
|
+
}
|
|
12639
|
+
function mapsSearchFailureOutcome(err) {
|
|
12640
|
+
if (err instanceof CaptchaError) return "captcha";
|
|
12641
|
+
const message = mapsSearchErrorMessage(err);
|
|
12642
|
+
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return "timeout";
|
|
12643
|
+
if (looksLikeProxyTunnelFailure(message)) return "proxy_tunnel_failed";
|
|
12644
|
+
if (looksLikeProxyUnavailable(message)) return "proxy_unavailable";
|
|
12645
|
+
if (looksLikeBrowserClosed(message)) return "browser_closed";
|
|
12646
|
+
return "unknown_error";
|
|
12647
|
+
}
|
|
12648
|
+
function retryableMapsSearchError(err, proxyMode) {
|
|
12649
|
+
if (err instanceof CaptchaError) return true;
|
|
12650
|
+
const message = mapsSearchErrorMessage(err);
|
|
12651
|
+
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
|
|
12652
|
+
if (looksLikeBrowserClosed(message)) return true;
|
|
12653
|
+
return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
|
|
12654
|
+
}
|
|
12655
|
+
function suffix(value) {
|
|
12656
|
+
return value ? value.slice(-8) : null;
|
|
12657
|
+
}
|
|
12658
|
+
function attemptFromDebug(input) {
|
|
12659
|
+
const network = input.debug.networkLocation;
|
|
12660
|
+
const browserSessionId = input.debug.kernel?.sessionId ?? null;
|
|
12661
|
+
const target = input.resolution?.target ?? null;
|
|
12662
|
+
return {
|
|
12663
|
+
attemptNumber: input.attemptIndex + 1,
|
|
12664
|
+
maxAttempts: input.maxAttempts,
|
|
12665
|
+
status: input.status,
|
|
12666
|
+
outcome: input.outcome,
|
|
12667
|
+
willRetry: input.willRetry,
|
|
12668
|
+
durationMs: input.durationMs,
|
|
12669
|
+
resultCount: input.resultCount,
|
|
12670
|
+
error: input.error,
|
|
12671
|
+
proxyMode: input.proxyMode,
|
|
12672
|
+
proxyResolutionSource: input.resolution?.source ?? null,
|
|
12673
|
+
proxyIdSuffix: input.resolution?.proxyIdSuffix ?? null,
|
|
12674
|
+
proxyTargetLevel: target?.level ?? null,
|
|
12675
|
+
proxyTargetLocation: target?.canonicalLocation ?? null,
|
|
12676
|
+
proxyTargetZip: target?.zip ?? null,
|
|
12677
|
+
browserSessionIdSuffix: suffix(browserSessionId),
|
|
12678
|
+
observedIp: network?.ip ?? null,
|
|
12679
|
+
observedCity: network?.city ?? null,
|
|
12680
|
+
observedRegion: network?.region ?? null
|
|
12681
|
+
};
|
|
12595
12682
|
}
|
|
12596
|
-
async function cleanupDisposableProxy(kernelApiKey, proxyId) {
|
|
12683
|
+
async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
|
|
12597
12684
|
if (!kernelApiKey || !proxyId) return;
|
|
12598
|
-
|
|
12685
|
+
try {
|
|
12686
|
+
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
12687
|
+
} catch (err) {
|
|
12599
12688
|
console.warn(JSON.stringify({
|
|
12600
|
-
event:
|
|
12601
|
-
proxy_id_suffix: proxyId
|
|
12602
|
-
message:
|
|
12689
|
+
event: eventName,
|
|
12690
|
+
proxy_id_suffix: suffix(proxyId),
|
|
12691
|
+
message: mapsSearchErrorMessage(err)
|
|
12603
12692
|
}));
|
|
12604
|
-
}
|
|
12693
|
+
}
|
|
12694
|
+
}
|
|
12695
|
+
async function runMapsSearchWithRotation(options) {
|
|
12696
|
+
const attempts = [];
|
|
12697
|
+
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
|
|
12698
|
+
const started = Date.now();
|
|
12699
|
+
let lastError = null;
|
|
12700
|
+
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
|
12701
|
+
const attemptStarted = Date.now();
|
|
12702
|
+
const driver = new BrowserDriver();
|
|
12703
|
+
const extractor = new MapsSearchExtractor(driver);
|
|
12704
|
+
const proxyZip = options.proxyZipForAttempt?.(attemptIndex) ?? options.proxyZip;
|
|
12705
|
+
let resolution2 = null;
|
|
12706
|
+
let disposableProxyId;
|
|
12707
|
+
try {
|
|
12708
|
+
resolution2 = await resolveKernelProxyId({
|
|
12709
|
+
kernelApiKey: options.kernelApiKey,
|
|
12710
|
+
proxyMode: options.proxyMode,
|
|
12711
|
+
configuredKernelProxyId: options.kernelProxyId ?? options.configuredKernelProxyId,
|
|
12712
|
+
location: options.location,
|
|
12713
|
+
proxyZip,
|
|
12714
|
+
gl: options.gl,
|
|
12715
|
+
attemptIndex,
|
|
12716
|
+
fresh: options.proxyMode === "location"
|
|
12717
|
+
});
|
|
12718
|
+
disposableProxyId = resolution2.disposableProxyId;
|
|
12719
|
+
const result = await extractor.extract({
|
|
12720
|
+
...options,
|
|
12721
|
+
proxyZip,
|
|
12722
|
+
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution2.kernelProxyId,
|
|
12723
|
+
kernelProxyResolution: resolution2.resolution
|
|
12724
|
+
});
|
|
12725
|
+
const attempt = attemptFromDebug({
|
|
12726
|
+
attemptIndex,
|
|
12727
|
+
maxAttempts,
|
|
12728
|
+
status: "ok",
|
|
12729
|
+
outcome: "success",
|
|
12730
|
+
willRetry: false,
|
|
12731
|
+
durationMs: Date.now() - attemptStarted,
|
|
12732
|
+
resultCount: result.resultCount,
|
|
12733
|
+
error: null,
|
|
12734
|
+
proxyMode: options.proxyMode,
|
|
12735
|
+
resolution: resolution2.resolution,
|
|
12736
|
+
debug: driver.getDebugSnapshot()
|
|
12737
|
+
});
|
|
12738
|
+
attempts.push(attempt);
|
|
12739
|
+
return {
|
|
12740
|
+
...result,
|
|
12741
|
+
attempts: attempts.slice(),
|
|
12742
|
+
durationMs: Date.now() - started
|
|
12743
|
+
};
|
|
12744
|
+
} catch (err) {
|
|
12745
|
+
lastError = err;
|
|
12746
|
+
const retryable = retryableMapsSearchError(err, options.proxyMode);
|
|
12747
|
+
const willRetry = attemptIndex < maxAttempts - 1 && retryable;
|
|
12748
|
+
const attempt = attemptFromDebug({
|
|
12749
|
+
attemptIndex,
|
|
12750
|
+
maxAttempts,
|
|
12751
|
+
status: "failed",
|
|
12752
|
+
outcome: mapsSearchFailureOutcome(err),
|
|
12753
|
+
willRetry,
|
|
12754
|
+
durationMs: Date.now() - attemptStarted,
|
|
12755
|
+
resultCount: 0,
|
|
12756
|
+
error: mapsSearchErrorMessage(err),
|
|
12757
|
+
proxyMode: options.proxyMode,
|
|
12758
|
+
resolution: resolution2?.resolution ?? null,
|
|
12759
|
+
debug: driver.getDebugSnapshot()
|
|
12760
|
+
});
|
|
12761
|
+
attempts.push(attempt);
|
|
12762
|
+
console.warn(JSON.stringify({
|
|
12763
|
+
event: options.failureEventName ?? "maps_search_attempt_failed",
|
|
12764
|
+
...options.logContext,
|
|
12765
|
+
attempt_number: attempt.attemptNumber,
|
|
12766
|
+
max_attempts: attempt.maxAttempts,
|
|
12767
|
+
outcome: attempt.outcome,
|
|
12768
|
+
will_retry: attempt.willRetry,
|
|
12769
|
+
proxy_resolution_source: attempt.proxyResolutionSource,
|
|
12770
|
+
proxy_id_suffix: attempt.proxyIdSuffix,
|
|
12771
|
+
browser_session_id_suffix: attempt.browserSessionIdSuffix,
|
|
12772
|
+
observed_city: attempt.observedCity,
|
|
12773
|
+
observed_region: attempt.observedRegion,
|
|
12774
|
+
message: attempt.error
|
|
12775
|
+
}));
|
|
12776
|
+
if (!willRetry) {
|
|
12777
|
+
throw new MapsSearchRotationError(
|
|
12778
|
+
mapsSearchErrorMessage(err),
|
|
12779
|
+
attempts,
|
|
12780
|
+
retryable,
|
|
12781
|
+
err instanceof CaptchaError ? "captcha_or_blocked" : "maps_search_failed"
|
|
12782
|
+
);
|
|
12783
|
+
}
|
|
12784
|
+
} finally {
|
|
12785
|
+
await cleanupDisposableProxy(options.kernelApiKey, disposableProxyId, "maps_search_proxy_delete_failed");
|
|
12786
|
+
await driver.close();
|
|
12787
|
+
}
|
|
12788
|
+
}
|
|
12789
|
+
throw new MapsSearchRotationError(
|
|
12790
|
+
lastError ? mapsSearchErrorMessage(lastError) : "Maps search failed",
|
|
12791
|
+
attempts,
|
|
12792
|
+
true
|
|
12793
|
+
);
|
|
12794
|
+
}
|
|
12795
|
+
var MAPS_SEARCH_MAX_ATTEMPTS, MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS, MapsSearchRotationError;
|
|
12796
|
+
var init_maps_search_rotation = __esm({
|
|
12797
|
+
"src/maps-search-rotation.ts"() {
|
|
12798
|
+
"use strict";
|
|
12799
|
+
init_BrowserDriver();
|
|
12800
|
+
init_MapsSearchExtractor();
|
|
12801
|
+
init_errors();
|
|
12802
|
+
init_kernel_proxy_resolver();
|
|
12803
|
+
MAPS_SEARCH_MAX_ATTEMPTS = 3;
|
|
12804
|
+
MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS = 5;
|
|
12805
|
+
MapsSearchRotationError = class extends Error {
|
|
12806
|
+
constructor(message, attempts, retryable, errorCode = "maps_search_failed") {
|
|
12807
|
+
super(message);
|
|
12808
|
+
this.attempts = attempts;
|
|
12809
|
+
this.retryable = retryable;
|
|
12810
|
+
this.errorCode = errorCode;
|
|
12811
|
+
}
|
|
12812
|
+
attempts;
|
|
12813
|
+
retryable;
|
|
12814
|
+
errorCode;
|
|
12815
|
+
name = "MapsSearchRotationError";
|
|
12816
|
+
};
|
|
12817
|
+
}
|
|
12818
|
+
});
|
|
12819
|
+
|
|
12820
|
+
// src/api/maps-routes.ts
|
|
12821
|
+
function mapsErrorResponse(c, err, errorCode) {
|
|
12822
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
12823
|
+
const rotationError = err instanceof MapsSearchRotationError ? err : null;
|
|
12824
|
+
const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
|
|
12825
|
+
const retryable = rotationError?.retryable ?? blocked;
|
|
12826
|
+
return c.json({
|
|
12827
|
+
error: sanitizeVendorName(msg),
|
|
12828
|
+
error_code: rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : errorCode),
|
|
12829
|
+
retryable,
|
|
12830
|
+
attempts: rotationError?.attempts ?? void 0
|
|
12831
|
+
}, retryable ? 503 : 500);
|
|
12605
12832
|
}
|
|
12606
12833
|
var import_hono5, mapsApp;
|
|
12607
12834
|
var init_maps_routes = __esm({
|
|
@@ -12611,13 +12838,12 @@ var init_maps_routes = __esm({
|
|
|
12611
12838
|
init_db();
|
|
12612
12839
|
init_rates();
|
|
12613
12840
|
init_MapsExtractor();
|
|
12614
|
-
init_MapsSearchExtractor();
|
|
12615
12841
|
init_BrowserDriver();
|
|
12616
12842
|
init_schemas3();
|
|
12617
12843
|
init_api_auth();
|
|
12618
12844
|
init_errors();
|
|
12619
12845
|
init_browser_service_env();
|
|
12620
|
-
|
|
12846
|
+
init_maps_search_rotation();
|
|
12621
12847
|
mapsApp = new import_hono5.Hono();
|
|
12622
12848
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12623
12849
|
const user = c.get("user");
|
|
@@ -12636,24 +12862,10 @@ var init_maps_routes = __esm({
|
|
|
12636
12862
|
[parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
|
|
12637
12863
|
);
|
|
12638
12864
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
12639
|
-
const driver = new BrowserDriver();
|
|
12640
|
-
const extractor = new MapsSearchExtractor(driver);
|
|
12641
|
-
let disposableProxyId;
|
|
12642
12865
|
try {
|
|
12643
|
-
const
|
|
12644
|
-
kernelApiKey: parsed.data.kernelApiKey,
|
|
12645
|
-
proxyMode: parsed.data.proxyMode,
|
|
12646
|
-
configuredKernelProxyId: browserServiceProxyId(),
|
|
12647
|
-
location: parsed.data.location,
|
|
12648
|
-
proxyZip: parsed.data.proxyZip,
|
|
12649
|
-
gl: parsed.data.gl,
|
|
12650
|
-
fresh: parsed.data.proxyMode === "location"
|
|
12651
|
-
});
|
|
12652
|
-
disposableProxyId = resolution2.disposableProxyId;
|
|
12653
|
-
const result = await extractor.extract({
|
|
12866
|
+
const result = await runMapsSearchWithRotation({
|
|
12654
12867
|
...parsed.data,
|
|
12655
|
-
|
|
12656
|
-
kernelProxyResolution: resolution2.resolution
|
|
12868
|
+
configuredKernelProxyId: browserServiceProxyId()
|
|
12657
12869
|
});
|
|
12658
12870
|
await logRequestEvent({
|
|
12659
12871
|
userId: user.id,
|
|
@@ -12674,12 +12886,10 @@ var init_maps_routes = __esm({
|
|
|
12674
12886
|
status: "failed",
|
|
12675
12887
|
query: [parsed.data.query, parsed.data.location].filter(Boolean).join(" "),
|
|
12676
12888
|
location: parsed.data.location,
|
|
12677
|
-
error: msg
|
|
12889
|
+
error: msg,
|
|
12890
|
+
result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
|
|
12678
12891
|
});
|
|
12679
|
-
return mapsErrorResponse(c,
|
|
12680
|
-
} finally {
|
|
12681
|
-
await cleanupDisposableProxy(parsed.data.kernelApiKey, disposableProxyId);
|
|
12682
|
-
await driver.close();
|
|
12892
|
+
return mapsErrorResponse(c, err, "maps_search_failed");
|
|
12683
12893
|
}
|
|
12684
12894
|
});
|
|
12685
12895
|
mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
@@ -12886,12 +13096,17 @@ function errorAttemptsSection(body) {
|
|
|
12886
13096
|
const browser = debug.browser ?? {};
|
|
12887
13097
|
const kernel = browser.browserRuntime ?? browser.kernel ?? {};
|
|
12888
13098
|
const proxyResolution = kernel.proxyResolution ?? {};
|
|
12889
|
-
const network = browser.networkLocation ?? {
|
|
13099
|
+
const network = browser.networkLocation ?? {
|
|
13100
|
+
ip: attempt.observedIp ?? attempt.observed_ip,
|
|
13101
|
+
city: attempt.observedCity ?? attempt.observed_city,
|
|
13102
|
+
region: attempt.observedRegion ?? attempt.observed_region
|
|
13103
|
+
};
|
|
12890
13104
|
const nav = browser.serpNavigation ?? {};
|
|
12891
13105
|
const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
|
|
12892
|
-
const sessionId = attempt.browser_session_id ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
|
|
13106
|
+
const sessionId = attempt.browser_session_id ?? attempt.browserSessionIdSuffix ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
|
|
12893
13107
|
const cleanupSucceeded2 = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
|
|
12894
|
-
|
|
13108
|
+
const proxySource = proxyResolution.source ?? attempt.proxyResolutionSource;
|
|
13109
|
+
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"}${proxySource ? `/${proxySource}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded2 === true ? "yes" : cleanupSucceeded2 === false ? "no" : "unknown"}`;
|
|
12895
13110
|
});
|
|
12896
13111
|
return `
|
|
12897
13112
|
|
|
@@ -13352,6 +13567,29 @@ ${rows}`,
|
|
|
13352
13567
|
}
|
|
13353
13568
|
};
|
|
13354
13569
|
}
|
|
13570
|
+
function normalizeMapsAttempts(value) {
|
|
13571
|
+
const attempts = Array.isArray(value) ? value : [];
|
|
13572
|
+
return attempts.map((attempt, index) => ({
|
|
13573
|
+
attemptNumber: attempt.attemptNumber ?? attempt.attempt_number ?? index + 1,
|
|
13574
|
+
maxAttempts: attempt.maxAttempts ?? attempt.max_attempts ?? attempts.length,
|
|
13575
|
+
status: attempt.status === "ok" ? "ok" : "failed",
|
|
13576
|
+
outcome: attempt.outcome ?? attempt.status ?? "unknown",
|
|
13577
|
+
willRetry: attempt.willRetry ?? attempt.will_retry ?? false,
|
|
13578
|
+
durationMs: attempt.durationMs ?? attempt.duration_ms ?? 0,
|
|
13579
|
+
resultCount: attempt.resultCount ?? attempt.result_count ?? 0,
|
|
13580
|
+
error: attempt.error ? sanitizeVendorText(attempt.error) : null,
|
|
13581
|
+
proxyMode: attempt.proxyMode ?? attempt.proxy_mode ?? "location",
|
|
13582
|
+
proxyResolutionSource: attempt.proxyResolutionSource ?? attempt.proxy_resolution_source ?? null,
|
|
13583
|
+
proxyIdSuffix: attempt.proxyIdSuffix ?? attempt.proxy_id_suffix ?? null,
|
|
13584
|
+
proxyTargetLevel: attempt.proxyTargetLevel ?? attempt.proxy_target_level ?? null,
|
|
13585
|
+
proxyTargetLocation: attempt.proxyTargetLocation ?? attempt.proxy_target_location ?? null,
|
|
13586
|
+
proxyTargetZip: attempt.proxyTargetZip ?? attempt.proxy_target_zip ?? null,
|
|
13587
|
+
browserSessionIdSuffix: attempt.browserSessionIdSuffix ?? attempt.browser_session_id ?? null,
|
|
13588
|
+
observedIp: attempt.observedIp ?? attempt.observed_ip ?? null,
|
|
13589
|
+
observedCity: attempt.observedCity ?? attempt.observed_city ?? null,
|
|
13590
|
+
observedRegion: attempt.observedRegion ?? attempt.observed_region ?? null
|
|
13591
|
+
}));
|
|
13592
|
+
}
|
|
13355
13593
|
function formatCreditsInfo(raw, input) {
|
|
13356
13594
|
const parsed = parseData(raw);
|
|
13357
13595
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -13424,6 +13662,8 @@ function formatMapsSearch(raw, input) {
|
|
|
13424
13662
|
const searchQuery = d.searchQuery ?? [input.query, input.location].filter(Boolean).join(" ");
|
|
13425
13663
|
const requestedMax = d.requestedMaxResults ?? input.maxResults ?? 10;
|
|
13426
13664
|
const durationMs = d.durationMs;
|
|
13665
|
+
const attempts = normalizeMapsAttempts(d.attempts);
|
|
13666
|
+
const lastAttempt = attempts.at(-1);
|
|
13427
13667
|
const rows = results.map((r) => {
|
|
13428
13668
|
const rating = [r.rating, r.reviewCount ? `(${r.reviewCount})` : null].filter(Boolean).join(" ");
|
|
13429
13669
|
return `| ${r.position} | ${cell(r.name)} | ${cell(r.category)} | ${cell(rating)} | ${cell(r.address)} | ${r.cidDecimal ? `\`${r.cidDecimal}\`` : "\u2014"} | ${r.websiteUrl ? `[site](${r.websiteUrl})` : "\u2014"} | [maps](${r.placeUrl}) |`;
|
|
@@ -13438,6 +13678,7 @@ ${meta}`;
|
|
|
13438
13678
|
const full = [
|
|
13439
13679
|
`# Google Maps Search: "${searchQuery}"`,
|
|
13440
13680
|
`**Returned:** ${results.length} profile candidate${results.length === 1 ? "" : "s"} \xB7 **Requested max:** ${requestedMax} \xB7 **Limit:** 50`,
|
|
13681
|
+
attempts.length ? `**Attempts:** ${attempts.length}/${lastAttempt?.maxAttempts ?? attempts.length} \xB7 **Proxy:** ${lastAttempt?.proxyMode ?? "unknown"}${lastAttempt?.proxyResolutionSource ? `/${lastAttempt.proxyResolutionSource}` : ""} \xB7 **Observed:** ${[lastAttempt?.observedCity, lastAttempt?.observedRegion].filter(Boolean).join(", ") || "unknown"}` : null,
|
|
13441
13682
|
`
|
|
13442
13683
|
## Results
|
|
13443
13684
|
| # | Name | Category | Rating | Address | CID | Website | Maps |
|
|
@@ -13461,6 +13702,7 @@ ${rows}`,
|
|
|
13461
13702
|
requestedMaxResults: requestedMax,
|
|
13462
13703
|
resultCount: results.length,
|
|
13463
13704
|
results: normalizedResults,
|
|
13705
|
+
attempts,
|
|
13464
13706
|
durationMs: durationMs ?? 0
|
|
13465
13707
|
}
|
|
13466
13708
|
};
|
|
@@ -13471,6 +13713,7 @@ function formatDirectoryWorkflow(raw, input) {
|
|
|
13471
13713
|
const d = parsed.data;
|
|
13472
13714
|
const cities = (d.cities ?? []).map((city) => ({
|
|
13473
13715
|
...city,
|
|
13716
|
+
attempts: normalizeMapsAttempts(city.attempts),
|
|
13474
13717
|
results: city.results.map((result) => ({
|
|
13475
13718
|
...result,
|
|
13476
13719
|
phone: result.phone ?? null,
|
|
@@ -13934,36 +14177,9 @@ var init_location_db = __esm({
|
|
|
13934
14177
|
});
|
|
13935
14178
|
|
|
13936
14179
|
// src/directory/directory-workflow.ts
|
|
13937
|
-
async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
|
|
13938
|
-
if (!kernelApiKey || !proxyId) return;
|
|
13939
|
-
try {
|
|
13940
|
-
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
13941
|
-
} catch (err) {
|
|
13942
|
-
console.warn(JSON.stringify({
|
|
13943
|
-
event: "directory_workflow_proxy_delete_failed",
|
|
13944
|
-
proxy_id_suffix: proxyId.slice(-6),
|
|
13945
|
-
message: err instanceof Error ? err.message : String(err)
|
|
13946
|
-
}));
|
|
13947
|
-
}
|
|
13948
|
-
}
|
|
13949
|
-
function maxAttemptsForProxyMode(proxyMode) {
|
|
13950
|
-
return proxyMode === "location" ? DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS : DIRECTORY_MAX_ATTEMPTS;
|
|
13951
|
-
}
|
|
13952
14180
|
function errorMessage(err) {
|
|
13953
14181
|
return err instanceof Error ? err.message : String(err);
|
|
13954
14182
|
}
|
|
13955
|
-
function looksLikeProxyTunnelFailure(message) {
|
|
13956
|
-
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
13957
|
-
}
|
|
13958
|
-
function looksLikeProxyUnavailable(message) {
|
|
13959
|
-
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
13960
|
-
}
|
|
13961
|
-
function retryableCitySearchError(err, proxyMode) {
|
|
13962
|
-
if (err instanceof CaptchaError) return true;
|
|
13963
|
-
const message = errorMessage(err);
|
|
13964
|
-
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
|
|
13965
|
-
return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
|
|
13966
|
-
}
|
|
13967
14183
|
function proxyZipForAttempt(options, market, attemptIndex) {
|
|
13968
14184
|
if (options.proxyZip) return options.proxyZip;
|
|
13969
14185
|
if (!market.zips.length) return void 0;
|
|
@@ -13982,25 +14198,10 @@ async function mapLimit(items, limit, fn) {
|
|
|
13982
14198
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
13983
14199
|
return out;
|
|
13984
14200
|
}
|
|
13985
|
-
async function
|
|
13986
|
-
const
|
|
13987
|
-
const extractor = new MapsSearchExtractor(driver);
|
|
13988
|
-
const start = Date.now();
|
|
13989
|
-
let disposableProxyId;
|
|
14201
|
+
async function searchCity(options, market) {
|
|
14202
|
+
const started = Date.now();
|
|
13990
14203
|
try {
|
|
13991
|
-
const
|
|
13992
|
-
const resolution2 = await resolveKernelProxyId({
|
|
13993
|
-
kernelApiKey: options.kernelApiKey,
|
|
13994
|
-
proxyMode: options.proxyMode,
|
|
13995
|
-
configuredKernelProxyId: browserServiceProxyId(),
|
|
13996
|
-
location: market.location,
|
|
13997
|
-
proxyZip,
|
|
13998
|
-
gl: options.gl,
|
|
13999
|
-
attemptIndex,
|
|
14000
|
-
fresh: options.proxyMode === "location"
|
|
14001
|
-
});
|
|
14002
|
-
disposableProxyId = resolution2.disposableProxyId;
|
|
14003
|
-
const result = await extractor.extract({
|
|
14204
|
+
const result = await runMapsSearchWithRotation({
|
|
14004
14205
|
query: options.query,
|
|
14005
14206
|
location: market.location,
|
|
14006
14207
|
gl: options.gl,
|
|
@@ -14008,11 +14209,16 @@ async function searchCityAttempt(options, market, attemptIndex) {
|
|
|
14008
14209
|
maxResults: options.maxResultsPerCity,
|
|
14009
14210
|
headless: options.headless,
|
|
14010
14211
|
kernelApiKey: options.kernelApiKey,
|
|
14011
|
-
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution2.kernelProxyId,
|
|
14012
|
-
kernelProxyResolution: resolution2.resolution,
|
|
14013
14212
|
proxyMode: options.proxyMode,
|
|
14014
|
-
proxyZip,
|
|
14015
|
-
debug: options.debug
|
|
14213
|
+
proxyZip: options.proxyZip,
|
|
14214
|
+
debug: options.debug,
|
|
14215
|
+
configuredKernelProxyId: browserServiceProxyId(),
|
|
14216
|
+
proxyZipForAttempt: (attemptIndex) => proxyZipForAttempt(options, market, attemptIndex),
|
|
14217
|
+
failureEventName: "directory_workflow_city_attempt_failed",
|
|
14218
|
+
logContext: {
|
|
14219
|
+
city: market.city,
|
|
14220
|
+
state: market.state
|
|
14221
|
+
}
|
|
14016
14222
|
});
|
|
14017
14223
|
return {
|
|
14018
14224
|
city: market.city,
|
|
@@ -14028,50 +14234,29 @@ async function searchCityAttempt(options, market, attemptIndex) {
|
|
|
14028
14234
|
error: null,
|
|
14029
14235
|
resultCount: result.resultCount,
|
|
14030
14236
|
durationMs: result.durationMs,
|
|
14237
|
+
attempts: result.attempts ?? [],
|
|
14031
14238
|
results: result.results
|
|
14032
14239
|
};
|
|
14033
|
-
}
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
|
|
14038
|
-
|
|
14039
|
-
|
|
14040
|
-
|
|
14041
|
-
|
|
14042
|
-
|
|
14043
|
-
|
|
14044
|
-
|
|
14045
|
-
|
|
14046
|
-
|
|
14047
|
-
|
|
14048
|
-
|
|
14049
|
-
|
|
14050
|
-
|
|
14051
|
-
|
|
14052
|
-
max_attempts: maxAttempts,
|
|
14053
|
-
will_retry: willRetry,
|
|
14054
|
-
message: errorMessage(err)
|
|
14055
|
-
}));
|
|
14056
|
-
if (!willRetry) break;
|
|
14057
|
-
}
|
|
14240
|
+
} catch (err) {
|
|
14241
|
+
const attempts = err instanceof MapsSearchRotationError ? err.attempts : [];
|
|
14242
|
+
return {
|
|
14243
|
+
city: market.city,
|
|
14244
|
+
state: market.state,
|
|
14245
|
+
location: market.location,
|
|
14246
|
+
cityKey: market.cityKey,
|
|
14247
|
+
censusName: market.censusName,
|
|
14248
|
+
population: market.population,
|
|
14249
|
+
populationYear: market.populationYear,
|
|
14250
|
+
zips: market.zips,
|
|
14251
|
+
counties: market.counties,
|
|
14252
|
+
status: "failed",
|
|
14253
|
+
error: errorMessage(err),
|
|
14254
|
+
resultCount: 0,
|
|
14255
|
+
durationMs: Date.now() - started,
|
|
14256
|
+
attempts,
|
|
14257
|
+
results: []
|
|
14258
|
+
};
|
|
14058
14259
|
}
|
|
14059
|
-
return {
|
|
14060
|
-
city: market.city,
|
|
14061
|
-
state: market.state,
|
|
14062
|
-
location: market.location,
|
|
14063
|
-
cityKey: market.cityKey,
|
|
14064
|
-
censusName: market.censusName,
|
|
14065
|
-
population: market.population,
|
|
14066
|
-
populationYear: market.populationYear,
|
|
14067
|
-
zips: market.zips,
|
|
14068
|
-
counties: market.counties,
|
|
14069
|
-
status: "failed",
|
|
14070
|
-
error: lastError ? errorMessage(lastError) : "City Maps search failed",
|
|
14071
|
-
resultCount: 0,
|
|
14072
|
-
durationMs: Date.now() - started,
|
|
14073
|
-
results: []
|
|
14074
|
-
};
|
|
14075
14260
|
}
|
|
14076
14261
|
function csvRowsFor(result) {
|
|
14077
14262
|
const rows = [];
|
|
@@ -14206,23 +14391,18 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
14206
14391
|
const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
|
|
14207
14392
|
return { ...base, csvPath };
|
|
14208
14393
|
}
|
|
14209
|
-
var import_promises5, import_node_path6, import_zod18,
|
|
14394
|
+
var import_promises5, import_node_path6, import_zod18, DirectoryWorkflowOptionsSchema;
|
|
14210
14395
|
var init_directory_workflow = __esm({
|
|
14211
14396
|
"src/directory/directory-workflow.ts"() {
|
|
14212
14397
|
"use strict";
|
|
14213
14398
|
import_promises5 = require("fs/promises");
|
|
14214
14399
|
import_node_path6 = require("path");
|
|
14215
14400
|
import_zod18 = require("zod");
|
|
14216
|
-
init_MapsSearchExtractor();
|
|
14217
|
-
init_BrowserDriver();
|
|
14218
14401
|
init_mcp_response_formatter();
|
|
14219
14402
|
init_browser_service_env();
|
|
14220
|
-
|
|
14221
|
-
init_errors();
|
|
14403
|
+
init_maps_search_rotation();
|
|
14222
14404
|
init_csv();
|
|
14223
14405
|
init_location_db();
|
|
14224
|
-
DIRECTORY_MAX_ATTEMPTS = 3;
|
|
14225
|
-
DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS = 5;
|
|
14226
14406
|
DirectoryWorkflowOptionsSchema = import_zod18.z.object({
|
|
14227
14407
|
query: import_zod18.z.string().min(1),
|
|
14228
14408
|
state: import_zod18.z.string().min(2).default("TN"),
|
|
@@ -16034,7 +16214,7 @@ function classifyAttemptResult(result) {
|
|
|
16034
16214
|
function errorMessage2(err) {
|
|
16035
16215
|
return err instanceof Error ? err.message : String(err);
|
|
16036
16216
|
}
|
|
16037
|
-
function
|
|
16217
|
+
function maxAttemptsForProxyMode(proxyMode) {
|
|
16038
16218
|
return proxyMode === "location" ? LOCATION_PROXY_MAX_ATTEMPTS : MAX_ATTEMPTS2;
|
|
16039
16219
|
}
|
|
16040
16220
|
function looksLikeProxyTunnelFailure2(message) {
|
|
@@ -16061,7 +16241,7 @@ function stripInternalDebug(result, keepDebug) {
|
|
|
16061
16241
|
delete diagnostics.debug;
|
|
16062
16242
|
return { ...result, diagnostics };
|
|
16063
16243
|
}
|
|
16064
|
-
async function
|
|
16244
|
+
async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
|
|
16065
16245
|
if (!kernelApiKey || !proxyId) return;
|
|
16066
16246
|
try {
|
|
16067
16247
|
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
@@ -16146,7 +16326,7 @@ async function harvest(rawOptions) {
|
|
|
16146
16326
|
};
|
|
16147
16327
|
const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
|
|
16148
16328
|
const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
|
|
16149
|
-
const maxAttempts =
|
|
16329
|
+
const maxAttempts = maxAttemptsForProxyMode(proxyMode);
|
|
16150
16330
|
const serializer = new OutputSerializer();
|
|
16151
16331
|
let lastError = null;
|
|
16152
16332
|
for (let i = 0; i < maxAttempts; i++) {
|
|
@@ -16223,7 +16403,7 @@ async function harvest(rawOptions) {
|
|
|
16223
16403
|
debug: attempt.debug,
|
|
16224
16404
|
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16225
16405
|
});
|
|
16226
|
-
await
|
|
16406
|
+
await cleanupDisposableProxy2(kernelApiKey, resolution2.disposableProxyId);
|
|
16227
16407
|
lastError = err;
|
|
16228
16408
|
if (willRetry) continue;
|
|
16229
16409
|
break;
|
|
@@ -16254,7 +16434,7 @@ async function harvest(rawOptions) {
|
|
|
16254
16434
|
debug: attempt.debug,
|
|
16255
16435
|
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16256
16436
|
});
|
|
16257
|
-
await
|
|
16437
|
+
await cleanupDisposableProxy2(kernelApiKey, resolution2.disposableProxyId);
|
|
16258
16438
|
lastError = err;
|
|
16259
16439
|
if (willRetry) continue;
|
|
16260
16440
|
break;
|
|
@@ -16274,7 +16454,7 @@ async function harvest(rawOptions) {
|
|
|
16274
16454
|
debug: attempt.debug,
|
|
16275
16455
|
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16276
16456
|
});
|
|
16277
|
-
await
|
|
16457
|
+
await cleanupDisposableProxy2(kernelApiKey, resolution2.disposableProxyId);
|
|
16278
16458
|
if (attemptOptions.format === "json" || attemptOptions.format === "both") {
|
|
16279
16459
|
await serializer.writeJSON(finalResult, attemptOptions.outputDir);
|
|
16280
16460
|
}
|
|
@@ -17082,12 +17262,12 @@ var PACKAGE_VERSION;
|
|
|
17082
17262
|
var init_version = __esm({
|
|
17083
17263
|
"src/version.ts"() {
|
|
17084
17264
|
"use strict";
|
|
17085
|
-
PACKAGE_VERSION = "0.2.
|
|
17265
|
+
PACKAGE_VERSION = "0.2.8";
|
|
17086
17266
|
}
|
|
17087
17267
|
});
|
|
17088
17268
|
|
|
17089
17269
|
// src/mcp/mcp-tool-schemas.ts
|
|
17090
|
-
var import_zod20, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
17270
|
+
var import_zod20, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
17091
17271
|
var init_mcp_tool_schemas = __esm({
|
|
17092
17272
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
17093
17273
|
"use strict";
|
|
@@ -17099,7 +17279,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17099
17279
|
gl: import_zod20.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
17100
17280
|
hl: import_zod20.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
17101
17281
|
device: import_zod20.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
17102
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt
|
|
17282
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
17103
17283
|
proxyZip: import_zod20.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
17104
17284
|
debug: import_zod20.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
17105
17285
|
};
|
|
@@ -17158,7 +17338,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17158
17338
|
gl: import_zod20.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
17159
17339
|
hl: import_zod20.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
17160
17340
|
maxResults: import_zod20.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
17161
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches;
|
|
17341
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
17162
17342
|
proxyZip: import_zod20.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
17163
17343
|
debug: import_zod20.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
17164
17344
|
};
|
|
@@ -17173,7 +17353,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17173
17353
|
includeZipGroups: import_zod20.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
17174
17354
|
usZipsCsvPath: import_zod20.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
17175
17355
|
saveCsv: import_zod20.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
17176
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches;
|
|
17356
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
17177
17357
|
proxyZip: import_zod20.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
|
|
17178
17358
|
debug: import_zod20.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
17179
17359
|
};
|
|
@@ -17196,6 +17376,26 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17196
17376
|
notes: import_zod20.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
17197
17377
|
};
|
|
17198
17378
|
NullableString = import_zod20.z.string().nullable();
|
|
17379
|
+
MapsSearchAttemptOutput = import_zod20.z.object({
|
|
17380
|
+
attemptNumber: import_zod20.z.number().int().min(1),
|
|
17381
|
+
maxAttempts: import_zod20.z.number().int().min(1),
|
|
17382
|
+
status: import_zod20.z.enum(["ok", "failed"]),
|
|
17383
|
+
outcome: import_zod20.z.string(),
|
|
17384
|
+
willRetry: import_zod20.z.boolean(),
|
|
17385
|
+
durationMs: import_zod20.z.number().int().min(0),
|
|
17386
|
+
resultCount: import_zod20.z.number().int().min(0),
|
|
17387
|
+
error: NullableString,
|
|
17388
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]),
|
|
17389
|
+
proxyResolutionSource: import_zod20.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
17390
|
+
proxyIdSuffix: NullableString,
|
|
17391
|
+
proxyTargetLevel: import_zod20.z.enum(["zip", "city", "state"]).nullable(),
|
|
17392
|
+
proxyTargetLocation: NullableString,
|
|
17393
|
+
proxyTargetZip: NullableString,
|
|
17394
|
+
browserSessionIdSuffix: NullableString,
|
|
17395
|
+
observedIp: NullableString,
|
|
17396
|
+
observedCity: NullableString,
|
|
17397
|
+
observedRegion: NullableString
|
|
17398
|
+
});
|
|
17199
17399
|
MapsSearchOutputSchema = {
|
|
17200
17400
|
query: import_zod20.z.string(),
|
|
17201
17401
|
location: import_zod20.z.string().nullable(),
|
|
@@ -17220,6 +17420,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17220
17420
|
directionsUrl: NullableString,
|
|
17221
17421
|
metadata: import_zod20.z.array(import_zod20.z.string())
|
|
17222
17422
|
})),
|
|
17423
|
+
attempts: import_zod20.z.array(MapsSearchAttemptOutput),
|
|
17223
17424
|
durationMs: import_zod20.z.number().int().min(0)
|
|
17224
17425
|
};
|
|
17225
17426
|
DirectoryMapsBusinessOutput = import_zod20.z.object({
|
|
@@ -17266,6 +17467,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17266
17467
|
error: NullableString,
|
|
17267
17468
|
resultCount: import_zod20.z.number().int().min(0),
|
|
17268
17469
|
durationMs: import_zod20.z.number().int().min(0),
|
|
17470
|
+
attempts: import_zod20.z.array(MapsSearchAttemptOutput),
|
|
17269
17471
|
results: import_zod20.z.array(DirectoryMapsBusinessOutput)
|
|
17270
17472
|
})),
|
|
17271
17473
|
durationMs: import_zod20.z.number().int().min(0)
|
|
@@ -17482,7 +17684,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17482
17684
|
gl: import_zod20.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
17483
17685
|
hl: import_zod20.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
17484
17686
|
device: import_zod20.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
17485
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt
|
|
17687
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
17486
17688
|
proxyZip: import_zod20.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
17487
17689
|
debug: import_zod20.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
17488
17690
|
pages: import_zod20.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
@@ -17493,7 +17695,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17493
17695
|
gl: import_zod20.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
17494
17696
|
hl: import_zod20.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
17495
17697
|
device: import_zod20.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
17496
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt
|
|
17698
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
|
|
17497
17699
|
proxyZip: import_zod20.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
17498
17700
|
pages: import_zod20.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
17499
17701
|
debug: import_zod20.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
@@ -17913,14 +18115,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17913
18115
|
if (savesReports) registerSavedReportResources(server);
|
|
17914
18116
|
server.registerTool("harvest_paa", {
|
|
17915
18117
|
title: "Google PAA + SERP Harvest",
|
|
17916
|
-
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries
|
|
18118
|
+
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or the market evidence is wrong. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
|
|
17917
18119
|
inputSchema: HarvestPaaInputSchema,
|
|
17918
18120
|
outputSchema: HarvestPaaOutputSchema,
|
|
17919
18121
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
17920
18122
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
17921
18123
|
server.registerTool("search_serp", {
|
|
17922
18124
|
title: "Google SERP Lookup",
|
|
17923
|
-
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs
|
|
18125
|
+
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear, the proxy tunnel fails, or location evidence is wrong."),
|
|
17924
18126
|
inputSchema: SearchSerpInputSchema,
|
|
17925
18127
|
outputSchema: SearchSerpOutputSchema,
|
|
17926
18128
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
@@ -17988,14 +18190,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17988
18190
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
17989
18191
|
server.registerTool("maps_search", {
|
|
17990
18192
|
title: "Google Maps Business Search",
|
|
17991
|
-
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service
|
|
18193
|
+
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service creates a fresh residential proxy ID and browser session for the market; retryable failures rotate to a new proxy and new browser session for up to 5 attempts. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
|
|
17992
18194
|
inputSchema: MapsSearchInputSchema,
|
|
17993
18195
|
outputSchema: MapsSearchOutputSchema,
|
|
17994
18196
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
17995
18197
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
17996
18198
|
server.registerTool("directory_workflow", {
|
|
17997
18199
|
title: "Directory Workflow: Markets + Maps",
|
|
17998
|
-
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city
|
|
18200
|
+
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. This workflow captures star ratings from Maps list cards, not profile review counts; use maps_place_intel only when a selected profile needs deeper review details. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
|
|
17999
18201
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
18000
18202
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
18001
18203
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
@@ -18183,7 +18385,7 @@ async function requireMcpCallerKey(c) {
|
|
|
18183
18385
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
18184
18386
|
server.registerTool("capture_serp_snapshot", {
|
|
18185
18387
|
title: "SERP Intelligence Snapshot",
|
|
18186
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries and
|
|
18388
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or location evidence is wrong. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
18187
18389
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
18188
18390
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
18189
18391
|
}, async (input) => executor.captureSerpSnapshot(input));
|