mcp-scraper 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -5
- package/dist/bin/api-server.cjs +2417 -648
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1276 -0
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -0
- package/dist/bin/mcp-scraper-cli.d.cts +1 -0
- package/dist/bin/mcp-scraper-cli.d.ts +1 -0
- package/dist/bin/mcp-scraper-cli.js +476 -0
- package/dist/bin/mcp-scraper-cli.js.map +1 -0
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +69 -14
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +5 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +5 -1
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +69 -14
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +42 -6
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-3TF6UT2P.js → chunk-5E6KJC26.js} +2 -2
- package/dist/{chunk-D4CJBZBY.js → chunk-7GCCOT3M.js} +372 -1
- package/dist/chunk-7GCCOT3M.js.map +1 -0
- package/dist/chunk-BJDCPCIE.js +7 -0
- package/dist/chunk-BJDCPCIE.js.map +1 -0
- package/dist/{chunk-MY3S7EX7.js → chunk-CQTAKXBN.js} +43 -7
- package/dist/chunk-CQTAKXBN.js.map +1 -0
- package/dist/chunk-L6IS63WS.js +869 -0
- package/dist/chunk-L6IS63WS.js.map +1 -0
- package/dist/{chunk-IQOCZGJJ.js → chunk-ROS67BNV.js} +5 -5
- package/dist/{chunk-7SI6XIR3.js → chunk-XAY5U67D.js} +70 -15
- package/dist/chunk-XAY5U67D.js.map +1 -0
- package/dist/{db-YWCNHBLH.js → db-BVHYI57K.js} +38 -2
- package/dist/index.cjs +42 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-RIKRBDOI.js → server-BKJ7LULQ.js} +818 -394
- package/dist/server-BKJ7LULQ.js.map +1 -0
- package/dist/{worker-NAKGTIF5.js → worker-TDJQ6TH3.js} +8 -8
- package/docs/mcp-tool-quality-spec.md +3 -1
- package/docs/specs/agent-ready-seo-packet-spec.md +237 -0
- package/docs/specs/cli-agent-wiring-spec.md +203 -0
- package/docs/specs/deferred-work-spec.md +12 -0
- package/docs/specs/local-competitive-audit-spec.md +312 -0
- package/docs/specs/scheduled-workflows-api-spec.md +304 -0
- package/docs/specs/seo-cli-growth-roadmap-spec.md +179 -0
- package/docs/specs/seo-workflow-runner-and-reports-spec.md +241 -0
- package/package.json +3 -2
- package/dist/chunk-7SI6XIR3.js.map +0 -1
- package/dist/chunk-D4CJBZBY.js.map +0 -1
- package/dist/chunk-IZE5UW7Y.js +0 -7
- package/dist/chunk-IZE5UW7Y.js.map +0 -1
- package/dist/chunk-MY3S7EX7.js.map +0 -1
- package/dist/server-RIKRBDOI.js.map +0 -1
- /package/dist/{chunk-3TF6UT2P.js.map → chunk-5E6KJC26.js.map} +0 -0
- /package/dist/{chunk-IQOCZGJJ.js.map → chunk-ROS67BNV.js.map} +0 -0
- /package/dist/{db-YWCNHBLH.js.map → db-BVHYI57K.js.map} +0 -0
- /package/dist/{worker-NAKGTIF5.js.map → worker-TDJQ6TH3.js.map} +0 -0
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
import {
|
|
3
3
|
HttpMcpToolExecutor,
|
|
4
4
|
registerPaaExtractorMcpTools
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-XAY5U67D.js";
|
|
6
6
|
import "../chunk-M2S27J6Z.js";
|
|
7
7
|
import {
|
|
8
8
|
registerBrowserAgentMcpTools
|
|
9
|
-
} from "../chunk-
|
|
9
|
+
} from "../chunk-5E6KJC26.js";
|
|
10
10
|
import {
|
|
11
11
|
PACKAGE_VERSION
|
|
12
|
-
} from "../chunk-
|
|
12
|
+
} from "../chunk-BJDCPCIE.js";
|
|
13
13
|
|
|
14
14
|
// bin/mcp-scraper-combined-stdio-server.ts
|
|
15
15
|
import { readFileSync } from "fs";
|
|
@@ -52,6 +52,7 @@ function renderInstallTerminal(options) {
|
|
|
52
52
|
colorize(ascii, "amber", color),
|
|
53
53
|
`${colorize("MCP Scraper Agent", "cyan", color)} . v${options.version} . mcpscraper.dev`,
|
|
54
54
|
"1/1 install surfaces ready",
|
|
55
|
+
colorize("Newest: faster search recovery, better location proxy rotation, and directory workflows for city/population + Maps CSV pulls.", "lime", color),
|
|
55
56
|
"",
|
|
56
57
|
`${colorize("Tools", "cyan", color)} ${colorize("(29 loaded)", "muted", color)}`,
|
|
57
58
|
toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
|
|
@@ -63,6 +64,9 @@ function renderInstallTerminal(options) {
|
|
|
63
64
|
"",
|
|
64
65
|
`${colorize("Ready.", "lime", color)} Install the combined MCP server with one of these paths:`,
|
|
65
66
|
"",
|
|
67
|
+
colorize("Setup doctor", "amber", color),
|
|
68
|
+
"npx -y -p mcp-scraper@latest mcp-scraper-cli doctor",
|
|
69
|
+
"",
|
|
66
70
|
colorize("Claude Code", "amber", color),
|
|
67
71
|
claudeCommand,
|
|
68
72
|
"",
|
|
@@ -79,7 +83,7 @@ function renderInstallTerminal(options) {
|
|
|
79
83
|
}
|
|
80
84
|
|
|
81
85
|
// src/version.ts
|
|
82
|
-
var PACKAGE_VERSION = "0.2.
|
|
86
|
+
var PACKAGE_VERSION = "0.2.9";
|
|
83
87
|
|
|
84
88
|
// bin/mcp-scraper-install.ts
|
|
85
89
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n 'claude mcp add mcp-scraper --scope user',\n ` --env MCP_SCRAPER_API_KEY=${apiKeyValue}`,\n ' -- npx -y -p mcp-scraper@latest mcp-scraper-combined',\n ].join(' \\\\\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(29 loaded)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_blueprint', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel'], color),\n toolRow('browser', ['browser_open', 'browser_screenshot', 'browser_click', 'browser_replay_download'], color),\n toolRow('account', ['credits_info', 'local reports', 'MCP resources'], color),\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one of these paths:`,\n '',\n colorize('Claude Code', 'amber', color),\n claudeCommand,\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'This installer can print terminal UI. MCP server commands stay silent so stdio JSON-RPC is not corrupted.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.2.
|
|
1
|
+
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n 'claude mcp add mcp-scraper --scope user',\n ` --env MCP_SCRAPER_API_KEY=${apiKeyValue}`,\n ' -- npx -y -p mcp-scraper@latest mcp-scraper-combined',\n ].join(' \\\\\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: faster search recovery, better location proxy rotation, and directory workflows for city/population + Maps CSV pulls.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(29 loaded)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_blueprint', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel'], color),\n toolRow('browser', ['browser_open', 'browser_screenshot', 'browser_click', 'browser_replay_download'], color),\n toolRow('account', ['credits_info', 'local reports', 'MCP resources'], color),\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one of these paths:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Claude Code', 'amber', color),\n claudeCommand,\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'This installer can print terminal UI. MCP server commands stay silent so stdio JSON-RPC is not corrupted.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.2.9'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'Use mcp-scraper-combined, mcp-scraper, and browser-agent only as MCP stdio servers.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB;AAAA,IACA,+BAA+B,WAAW;AAAA,IAC1C;AAAA,EACF,EAAE,KAAK,OAAO;AAEd,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,iIAAiI,QAAQ,KAAK;AAAA,IACvJ;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,eAAe,SAAS,KAAK,CAAC;AAAA,IAC/E,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,oBAAoB,GAAG,KAAK;AAAA,IAChG,QAAQ,SAAS,CAAC,0BAA0B,aAAa,iBAAiB,GAAG,KAAK;AAAA,IAClF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,qBAAqB,GAAG,KAAK;AAAA,IAC9G,QAAQ,WAAW,CAAC,gBAAgB,sBAAsB,iBAAiB,yBAAyB,GAAG,KAAK;AAAA,IAC5G,QAAQ,WAAW,CAAC,gBAAgB,iBAAiB,eAAe,GAAG,KAAK;AAAA,IAC5E;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;AC5FO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
PACKAGE_VERSION
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-BJDCPCIE.js";
|
|
5
5
|
|
|
6
6
|
// src/install-terminal.ts
|
|
7
7
|
var CODES = {
|
|
@@ -54,6 +54,7 @@ function renderInstallTerminal(options) {
|
|
|
54
54
|
colorize(ascii, "amber", color),
|
|
55
55
|
`${colorize("MCP Scraper Agent", "cyan", color)} . v${options.version} . mcpscraper.dev`,
|
|
56
56
|
"1/1 install surfaces ready",
|
|
57
|
+
colorize("Newest: faster search recovery, better location proxy rotation, and directory workflows for city/population + Maps CSV pulls.", "lime", color),
|
|
57
58
|
"",
|
|
58
59
|
`${colorize("Tools", "cyan", color)} ${colorize("(29 loaded)", "muted", color)}`,
|
|
59
60
|
toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
|
|
@@ -65,6 +66,9 @@ function renderInstallTerminal(options) {
|
|
|
65
66
|
"",
|
|
66
67
|
`${colorize("Ready.", "lime", color)} Install the combined MCP server with one of these paths:`,
|
|
67
68
|
"",
|
|
69
|
+
colorize("Setup doctor", "amber", color),
|
|
70
|
+
"npx -y -p mcp-scraper@latest mcp-scraper-cli doctor",
|
|
71
|
+
"",
|
|
68
72
|
colorize("Claude Code", "amber", color),
|
|
69
73
|
claudeCommand,
|
|
70
74
|
"",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/install-terminal.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n 'claude mcp add mcp-scraper --scope user',\n ` --env MCP_SCRAPER_API_KEY=${apiKeyValue}`,\n ' -- npx -y -p mcp-scraper@latest mcp-scraper-combined',\n ].join(' \\\\\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(29 loaded)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_blueprint', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel'], color),\n toolRow('browser', ['browser_open', 'browser_screenshot', 'browser_click', 'browser_replay_download'], color),\n toolRow('account', ['credits_info', 'local reports', 'MCP resources'], color),\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one of these paths:`,\n '',\n colorize('Claude Code', 'amber', color),\n claudeCommand,\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'This installer can print terminal UI. MCP server commands stay silent so stdio JSON-RPC is not corrupted.',\n '',\n ].join('\\n')\n}\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'Use mcp-scraper-combined, mcp-scraper, and browser-agent only as MCP stdio servers.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB;AAAA,IACA,+BAA+B,WAAW;AAAA,IAC1C;AAAA,EACF,EAAE,KAAK,OAAO;AAEd,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,eAAe,SAAS,KAAK,CAAC;AAAA,IAC/E,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,oBAAoB,GAAG,KAAK;AAAA,IAChG,QAAQ,SAAS,CAAC,0BAA0B,aAAa,iBAAiB,GAAG,KAAK;AAAA,IAClF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,qBAAqB,GAAG,KAAK;AAAA,IAC9G,QAAQ,WAAW,CAAC,gBAAgB,sBAAsB,iBAAiB,yBAAyB,GAAG,KAAK;AAAA,IAC5G,QAAQ,WAAW,CAAC,gBAAgB,iBAAiB,eAAe,GAAG,KAAK;AAAA,IAC5E;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;
|
|
1
|
+
{"version":3,"sources":["../../src/install-terminal.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n 'claude mcp add mcp-scraper --scope user',\n ` --env MCP_SCRAPER_API_KEY=${apiKeyValue}`,\n ' -- npx -y -p mcp-scraper@latest mcp-scraper-combined',\n ].join(' \\\\\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: faster search recovery, better location proxy rotation, and directory workflows for city/population + Maps CSV pulls.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(29 loaded)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_blueprint', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel'], color),\n toolRow('browser', ['browser_open', 'browser_screenshot', 'browser_click', 'browser_replay_download'], color),\n toolRow('account', ['credits_info', 'local reports', 'MCP resources'], color),\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one of these paths:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Claude Code', 'amber', color),\n claudeCommand,\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'This installer can print terminal UI. MCP server commands stay silent so stdio JSON-RPC is not corrupted.',\n '',\n ].join('\\n')\n}\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'Use mcp-scraper-combined, mcp-scraper, and browser-agent only as MCP stdio servers.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB;AAAA,IACA,+BAA+B,WAAW;AAAA,IAC1C;AAAA,EACF,EAAE,KAAK,OAAO;AAEd,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,iIAAiI,QAAQ,KAAK;AAAA,IACvJ;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,eAAe,SAAS,KAAK,CAAC;AAAA,IAC/E,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,oBAAoB,GAAG,KAAK;AAAA,IAChG,QAAQ,SAAS,CAAC,0BAA0B,aAAa,iBAAiB,GAAG,KAAK;AAAA,IAClF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,qBAAqB,GAAG,KAAK;AAAA,IAC9G,QAAQ,WAAW,CAAC,gBAAgB,sBAAsB,iBAAiB,yBAAyB,GAAG,KAAK;AAAA,IAC5G,QAAQ,WAAW,CAAC,gBAAgB,iBAAiB,eAAe,GAAG,KAAK;AAAA,IAC5E;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACxFA,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
@@ -136,7 +136,7 @@ var import_node_fs2 = require("fs");
|
|
|
136
136
|
var import_node_path2 = require("path");
|
|
137
137
|
|
|
138
138
|
// src/version.ts
|
|
139
|
-
var PACKAGE_VERSION = "0.2.
|
|
139
|
+
var PACKAGE_VERSION = "0.2.9";
|
|
140
140
|
|
|
141
141
|
// src/mcp/mcp-response-formatter.ts
|
|
142
142
|
var import_node_fs = require("fs");
|
|
@@ -276,12 +276,17 @@ function errorAttemptsSection(body) {
|
|
|
276
276
|
const browser = debug.browser ?? {};
|
|
277
277
|
const kernel = browser.browserRuntime ?? browser.kernel ?? {};
|
|
278
278
|
const proxyResolution = kernel.proxyResolution ?? {};
|
|
279
|
-
const network = browser.networkLocation ?? {
|
|
279
|
+
const network = browser.networkLocation ?? {
|
|
280
|
+
ip: attempt.observedIp ?? attempt.observed_ip,
|
|
281
|
+
city: attempt.observedCity ?? attempt.observed_city,
|
|
282
|
+
region: attempt.observedRegion ?? attempt.observed_region
|
|
283
|
+
};
|
|
280
284
|
const nav = browser.serpNavigation ?? {};
|
|
281
285
|
const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
|
|
282
|
-
const sessionId = attempt.browser_session_id ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
|
|
286
|
+
const sessionId = attempt.browser_session_id ?? attempt.browserSessionIdSuffix ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
|
|
283
287
|
const cleanupSucceeded = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
|
|
284
|
-
|
|
288
|
+
const proxySource = proxyResolution.source ?? attempt.proxyResolutionSource;
|
|
289
|
+
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"}${proxySource ? `/${proxySource}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded === true ? "yes" : cleanupSucceeded === false ? "no" : "unknown"}`;
|
|
285
290
|
});
|
|
286
291
|
return `
|
|
287
292
|
|
|
@@ -742,6 +747,29 @@ ${rows}`,
|
|
|
742
747
|
}
|
|
743
748
|
};
|
|
744
749
|
}
|
|
750
|
+
function normalizeMapsAttempts(value) {
|
|
751
|
+
const attempts = Array.isArray(value) ? value : [];
|
|
752
|
+
return attempts.map((attempt, index) => ({
|
|
753
|
+
attemptNumber: attempt.attemptNumber ?? attempt.attempt_number ?? index + 1,
|
|
754
|
+
maxAttempts: attempt.maxAttempts ?? attempt.max_attempts ?? attempts.length,
|
|
755
|
+
status: attempt.status === "ok" ? "ok" : "failed",
|
|
756
|
+
outcome: attempt.outcome ?? attempt.status ?? "unknown",
|
|
757
|
+
willRetry: attempt.willRetry ?? attempt.will_retry ?? false,
|
|
758
|
+
durationMs: attempt.durationMs ?? attempt.duration_ms ?? 0,
|
|
759
|
+
resultCount: attempt.resultCount ?? attempt.result_count ?? 0,
|
|
760
|
+
error: attempt.error ? sanitizeVendorText(attempt.error) : null,
|
|
761
|
+
proxyMode: attempt.proxyMode ?? attempt.proxy_mode ?? "location",
|
|
762
|
+
proxyResolutionSource: attempt.proxyResolutionSource ?? attempt.proxy_resolution_source ?? null,
|
|
763
|
+
proxyIdSuffix: attempt.proxyIdSuffix ?? attempt.proxy_id_suffix ?? null,
|
|
764
|
+
proxyTargetLevel: attempt.proxyTargetLevel ?? attempt.proxy_target_level ?? null,
|
|
765
|
+
proxyTargetLocation: attempt.proxyTargetLocation ?? attempt.proxy_target_location ?? null,
|
|
766
|
+
proxyTargetZip: attempt.proxyTargetZip ?? attempt.proxy_target_zip ?? null,
|
|
767
|
+
browserSessionIdSuffix: attempt.browserSessionIdSuffix ?? attempt.browser_session_id ?? null,
|
|
768
|
+
observedIp: attempt.observedIp ?? attempt.observed_ip ?? null,
|
|
769
|
+
observedCity: attempt.observedCity ?? attempt.observed_city ?? null,
|
|
770
|
+
observedRegion: attempt.observedRegion ?? attempt.observed_region ?? null
|
|
771
|
+
}));
|
|
772
|
+
}
|
|
745
773
|
function formatCreditsInfo(raw, input) {
|
|
746
774
|
const parsed = parseData(raw);
|
|
747
775
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -814,6 +842,8 @@ function formatMapsSearch(raw, input) {
|
|
|
814
842
|
const searchQuery = d.searchQuery ?? [input.query, input.location].filter(Boolean).join(" ");
|
|
815
843
|
const requestedMax = d.requestedMaxResults ?? input.maxResults ?? 10;
|
|
816
844
|
const durationMs = d.durationMs;
|
|
845
|
+
const attempts = normalizeMapsAttempts(d.attempts);
|
|
846
|
+
const lastAttempt = attempts.at(-1);
|
|
817
847
|
const rows = results.map((r) => {
|
|
818
848
|
const rating = [r.rating, r.reviewCount ? `(${r.reviewCount})` : null].filter(Boolean).join(" ");
|
|
819
849
|
return `| ${r.position} | ${cell(r.name)} | ${cell(r.category)} | ${cell(rating)} | ${cell(r.address)} | ${r.cidDecimal ? `\`${r.cidDecimal}\`` : "\u2014"} | ${r.websiteUrl ? `[site](${r.websiteUrl})` : "\u2014"} | [maps](${r.placeUrl}) |`;
|
|
@@ -828,6 +858,7 @@ ${meta}`;
|
|
|
828
858
|
const full = [
|
|
829
859
|
`# Google Maps Search: "${searchQuery}"`,
|
|
830
860
|
`**Returned:** ${results.length} profile candidate${results.length === 1 ? "" : "s"} \xB7 **Requested max:** ${requestedMax} \xB7 **Limit:** 50`,
|
|
861
|
+
attempts.length ? `**Attempts:** ${attempts.length}/${lastAttempt?.maxAttempts ?? attempts.length} \xB7 **Proxy:** ${lastAttempt?.proxyMode ?? "unknown"}${lastAttempt?.proxyResolutionSource ? `/${lastAttempt.proxyResolutionSource}` : ""} \xB7 **Observed:** ${[lastAttempt?.observedCity, lastAttempt?.observedRegion].filter(Boolean).join(", ") || "unknown"}` : null,
|
|
831
862
|
`
|
|
832
863
|
## Results
|
|
833
864
|
| # | Name | Category | Rating | Address | CID | Website | Maps |
|
|
@@ -851,6 +882,7 @@ ${rows}`,
|
|
|
851
882
|
requestedMaxResults: requestedMax,
|
|
852
883
|
resultCount: results.length,
|
|
853
884
|
results: normalizedResults,
|
|
885
|
+
attempts,
|
|
854
886
|
durationMs: durationMs ?? 0
|
|
855
887
|
}
|
|
856
888
|
};
|
|
@@ -861,6 +893,7 @@ function formatDirectoryWorkflow(raw, input) {
|
|
|
861
893
|
const d = parsed.data;
|
|
862
894
|
const cities = (d.cities ?? []).map((city) => ({
|
|
863
895
|
...city,
|
|
896
|
+
attempts: normalizeMapsAttempts(city.attempts),
|
|
864
897
|
results: city.results.map((result) => ({
|
|
865
898
|
...result,
|
|
866
899
|
phone: result.phone ?? null,
|
|
@@ -1088,7 +1121,7 @@ var HarvestPaaInputSchema = {
|
|
|
1088
1121
|
gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
1089
1122
|
hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
1090
1123
|
device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
1091
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt
|
|
1124
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
1092
1125
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
1093
1126
|
debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
1094
1127
|
};
|
|
@@ -1147,7 +1180,7 @@ var MapsSearchInputSchema = {
|
|
|
1147
1180
|
gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
1148
1181
|
hl: import_zod.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
1149
1182
|
maxResults: import_zod.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
1150
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches;
|
|
1183
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
1151
1184
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
1152
1185
|
debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
1153
1186
|
};
|
|
@@ -1161,8 +1194,8 @@ var DirectoryWorkflowInputSchema = {
|
|
|
1161
1194
|
concurrency: import_zod.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
|
|
1162
1195
|
includeZipGroups: import_zod.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
1163
1196
|
usZipsCsvPath: import_zod.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
1164
|
-
saveCsv: import_zod.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
1165
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches;
|
|
1197
|
+
saveCsv: import_zod.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
1198
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
1166
1199
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
|
|
1167
1200
|
debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
1168
1201
|
};
|
|
@@ -1185,6 +1218,26 @@ var RankTrackerBlueprintInputSchema = {
|
|
|
1185
1218
|
notes: import_zod.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
1186
1219
|
};
|
|
1187
1220
|
var NullableString = import_zod.z.string().nullable();
|
|
1221
|
+
var MapsSearchAttemptOutput = import_zod.z.object({
|
|
1222
|
+
attemptNumber: import_zod.z.number().int().min(1),
|
|
1223
|
+
maxAttempts: import_zod.z.number().int().min(1),
|
|
1224
|
+
status: import_zod.z.enum(["ok", "failed"]),
|
|
1225
|
+
outcome: import_zod.z.string(),
|
|
1226
|
+
willRetry: import_zod.z.boolean(),
|
|
1227
|
+
durationMs: import_zod.z.number().int().min(0),
|
|
1228
|
+
resultCount: import_zod.z.number().int().min(0),
|
|
1229
|
+
error: NullableString,
|
|
1230
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]),
|
|
1231
|
+
proxyResolutionSource: import_zod.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
1232
|
+
proxyIdSuffix: NullableString,
|
|
1233
|
+
proxyTargetLevel: import_zod.z.enum(["zip", "city", "state"]).nullable(),
|
|
1234
|
+
proxyTargetLocation: NullableString,
|
|
1235
|
+
proxyTargetZip: NullableString,
|
|
1236
|
+
browserSessionIdSuffix: NullableString,
|
|
1237
|
+
observedIp: NullableString,
|
|
1238
|
+
observedCity: NullableString,
|
|
1239
|
+
observedRegion: NullableString
|
|
1240
|
+
});
|
|
1188
1241
|
var MapsSearchOutputSchema = {
|
|
1189
1242
|
query: import_zod.z.string(),
|
|
1190
1243
|
location: import_zod.z.string().nullable(),
|
|
@@ -1209,6 +1262,7 @@ var MapsSearchOutputSchema = {
|
|
|
1209
1262
|
directionsUrl: NullableString,
|
|
1210
1263
|
metadata: import_zod.z.array(import_zod.z.string())
|
|
1211
1264
|
})),
|
|
1265
|
+
attempts: import_zod.z.array(MapsSearchAttemptOutput),
|
|
1212
1266
|
durationMs: import_zod.z.number().int().min(0)
|
|
1213
1267
|
};
|
|
1214
1268
|
var DirectoryMapsBusinessOutput = import_zod.z.object({
|
|
@@ -1255,6 +1309,7 @@ var DirectoryWorkflowOutputSchema = {
|
|
|
1255
1309
|
error: NullableString,
|
|
1256
1310
|
resultCount: import_zod.z.number().int().min(0),
|
|
1257
1311
|
durationMs: import_zod.z.number().int().min(0),
|
|
1312
|
+
attempts: import_zod.z.array(MapsSearchAttemptOutput),
|
|
1258
1313
|
results: import_zod.z.array(DirectoryMapsBusinessOutput)
|
|
1259
1314
|
})),
|
|
1260
1315
|
durationMs: import_zod.z.number().int().min(0)
|
|
@@ -1471,7 +1526,7 @@ var SearchSerpInputSchema = {
|
|
|
1471
1526
|
gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
1472
1527
|
hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
1473
1528
|
device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
1474
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt
|
|
1529
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
1475
1530
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
1476
1531
|
debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
1477
1532
|
pages: import_zod.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
@@ -1482,7 +1537,7 @@ var CaptureSerpSnapshotInputSchema = {
|
|
|
1482
1537
|
gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
1483
1538
|
hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
1484
1539
|
device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
1485
|
-
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt
|
|
1540
|
+
proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
|
|
1486
1541
|
proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
1487
1542
|
pages: import_zod.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
1488
1543
|
debug: import_zod.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
@@ -1894,14 +1949,14 @@ function registerPaaExtractorMcpTools(server2, executor2, options = {}) {
|
|
|
1894
1949
|
if (savesReports) registerSavedReportResources(server2);
|
|
1895
1950
|
server2.registerTool("harvest_paa", {
|
|
1896
1951
|
title: "Google PAA + SERP Harvest",
|
|
1897
|
-
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries
|
|
1952
|
+
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or the market evidence is wrong. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
|
|
1898
1953
|
inputSchema: HarvestPaaInputSchema,
|
|
1899
1954
|
outputSchema: HarvestPaaOutputSchema,
|
|
1900
1955
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
1901
1956
|
}, async (input) => formatHarvestPaa(await executor2.harvestPaa(input), input));
|
|
1902
1957
|
server2.registerTool("search_serp", {
|
|
1903
1958
|
title: "Google SERP Lookup",
|
|
1904
|
-
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs
|
|
1959
|
+
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear, the proxy tunnel fails, or location evidence is wrong."),
|
|
1905
1960
|
inputSchema: SearchSerpInputSchema,
|
|
1906
1961
|
outputSchema: SearchSerpOutputSchema,
|
|
1907
1962
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
@@ -1969,14 +2024,14 @@ function registerPaaExtractorMcpTools(server2, executor2, options = {}) {
|
|
|
1969
2024
|
}, async (input) => formatMapsPlaceIntel(await executor2.mapsPlaceIntel(input), input));
|
|
1970
2025
|
server2.registerTool("maps_search", {
|
|
1971
2026
|
title: "Google Maps Business Search",
|
|
1972
|
-
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service
|
|
2027
|
+
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service creates a fresh residential proxy ID and browser session for the market; retryable failures rotate to a new proxy and new browser session for up to 5 attempts. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
|
|
1973
2028
|
inputSchema: MapsSearchInputSchema,
|
|
1974
2029
|
outputSchema: MapsSearchOutputSchema,
|
|
1975
2030
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
1976
2031
|
}, async (input) => formatMapsSearch(await executor2.mapsSearch(input), input));
|
|
1977
2032
|
server2.registerTool("directory_workflow", {
|
|
1978
2033
|
title: "Directory Workflow: Markets + Maps",
|
|
1979
|
-
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city
|
|
2034
|
+
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
|
|
1980
2035
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
1981
2036
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
1982
2037
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|