mcp-scraper 0.21.2 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +1061 -308
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +101 -34
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +22 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-ZJDANZTQ.js → chunk-E5J4HJBO.js} +3 -2
- package/dist/chunk-E5J4HJBO.js.map +1 -0
- package/dist/{chunk-DZBGULWW.js → chunk-ICT7DDHL.js} +399 -3
- package/dist/chunk-ICT7DDHL.js.map +1 -0
- package/dist/{chunk-2FF4GBQ4.js → chunk-JHI373VB.js} +49 -7
- package/dist/chunk-JHI373VB.js.map +1 -0
- package/dist/{chunk-GMZFQRH2.js → chunk-MTSBI7ZH.js} +23 -6
- package/dist/chunk-MTSBI7ZH.js.map +1 -0
- package/dist/{chunk-ZNNYPE2U.js → chunk-N4JHXPW6.js} +89 -27
- package/dist/chunk-N4JHXPW6.js.map +1 -0
- package/dist/{chunk-X3NIXHHG.js → chunk-PNBYYAEK.js} +3 -3
- package/dist/{chunk-JBME5WJR.js → chunk-SDHPQTFM.js} +2 -2
- package/dist/chunk-WQI7JIIN.js +7 -0
- package/dist/chunk-WQI7JIIN.js.map +1 -0
- package/dist/{db-EONGTNJM.js → db-WRILVOFY.js} +6 -2
- package/dist/{extract-bundle-HMCYUVBJ.js → extract-bundle-NFFVMUMT.js} +2 -2
- package/dist/index.cjs +22 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-CHTX6YQA.js → server-OLOOFN5W.js} +499 -271
- package/dist/server-OLOOFN5W.js.map +1 -0
- package/dist/{site-extract-repository-5V5EEQ72.js → site-extract-repository-2QQSL2QC.js} +4 -4
- package/dist/{worker-GE5JMHE6.js → worker-RV2TYXE3.js} +15 -8
- package/dist/worker-RV2TYXE3.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +90 -11
- package/docs/specs/oauth-mcp-spec.md +2 -2
- package/package.json +1 -1
- package/dist/chunk-2FF4GBQ4.js.map +0 -1
- package/dist/chunk-DZBGULWW.js.map +0 -1
- package/dist/chunk-GMZFQRH2.js.map +0 -1
- package/dist/chunk-NBF4F4UT.js +0 -7
- package/dist/chunk-NBF4F4UT.js.map +0 -1
- package/dist/chunk-ZJDANZTQ.js.map +0 -1
- package/dist/chunk-ZNNYPE2U.js.map +0 -1
- package/dist/server-CHTX6YQA.js.map +0 -1
- package/dist/worker-GE5JMHE6.js.map +0 -1
- /package/dist/{chunk-X3NIXHHG.js.map → chunk-PNBYYAEK.js.map} +0 -0
- /package/dist/{chunk-JBME5WJR.js.map → chunk-SDHPQTFM.js.map} +0 -0
- /package/dist/{db-EONGTNJM.js.map → db-WRILVOFY.js.map} +0 -0
- /package/dist/{extract-bundle-HMCYUVBJ.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
- /package/dist/{site-extract-repository-5V5EEQ72.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
|
@@ -106,7 +106,7 @@ function renderInstallTerminal(options) {
|
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
// src/version.ts
|
|
109
|
-
var PACKAGE_VERSION = "0.21.
|
|
109
|
+
var PACKAGE_VERSION = "0.21.4";
|
|
110
110
|
|
|
111
111
|
// bin/mcp-scraper-install.ts
|
|
112
112
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(159 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.21.2'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACnHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(159 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.21.4'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACnHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
} from "../chunk-YNHPZU6B.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-WQI7JIIN.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -30,6 +30,9 @@ var import_node_path8 = require("path");
|
|
|
30
30
|
var import_mcp3 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
31
31
|
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
32
32
|
|
|
33
|
+
// src/mcp/http-mcp-tool-executor.ts
|
|
34
|
+
var import_node_crypto = require("crypto");
|
|
35
|
+
|
|
33
36
|
// src/harvest-timeout.ts
|
|
34
37
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
35
38
|
var CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
|
|
@@ -120,6 +123,7 @@ var HttpMcpToolExecutor = class {
|
|
|
120
123
|
timeoutMs;
|
|
121
124
|
httpTimeoutOverrideMs;
|
|
122
125
|
serpIntelligenceTimeoutMs;
|
|
126
|
+
pendingSerpCaptureBillingKeys = /* @__PURE__ */ new Map();
|
|
123
127
|
constructor(baseUrl2, apiKey2) {
|
|
124
128
|
this.baseUrl = baseUrl2.replace(/\/$/, "");
|
|
125
129
|
this.apiKey = apiKey2;
|
|
@@ -130,13 +134,14 @@ var HttpMcpToolExecutor = class {
|
|
|
130
134
|
const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
|
|
131
135
|
this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
|
|
132
136
|
}
|
|
133
|
-
async call(path, body, timeoutMs = this.timeoutMs, method = "POST") {
|
|
137
|
+
async call(path, body, timeoutMs = this.timeoutMs, method = "POST", requestHeaders = {}) {
|
|
134
138
|
try {
|
|
135
139
|
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
136
140
|
method,
|
|
137
141
|
headers: {
|
|
138
142
|
"Content-Type": "application/json",
|
|
139
|
-
"x-api-key": this.apiKey
|
|
143
|
+
"x-api-key": this.apiKey,
|
|
144
|
+
...requestHeaders
|
|
140
145
|
},
|
|
141
146
|
body: JSON.stringify(body),
|
|
142
147
|
signal: AbortSignal.timeout(timeoutMs)
|
|
@@ -384,8 +389,27 @@ var HttpMcpToolExecutor = class {
|
|
|
384
389
|
connections: input.connections
|
|
385
390
|
}, void 0, "PUT");
|
|
386
391
|
}
|
|
387
|
-
captureSerpSnapshot(input) {
|
|
388
|
-
|
|
392
|
+
async captureSerpSnapshot(input) {
|
|
393
|
+
const fingerprint = (0, import_node_crypto.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
394
|
+
const now = Date.now();
|
|
395
|
+
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
396
|
+
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
397
|
+
}
|
|
398
|
+
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
|
|
399
|
+
const idempotencyKey = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto.randomUUID)();
|
|
400
|
+
this.pendingSerpCaptureBillingKeys.set(fingerprint, {
|
|
401
|
+
key: idempotencyKey,
|
|
402
|
+
expiresAt: now + 15 * 6e4
|
|
403
|
+
});
|
|
404
|
+
const result = await this.call(
|
|
405
|
+
"/serp-intelligence/capture",
|
|
406
|
+
input,
|
|
407
|
+
this.serpIntelligenceTimeoutMs,
|
|
408
|
+
"POST",
|
|
409
|
+
{ "Idempotency-Key": idempotencyKey }
|
|
410
|
+
);
|
|
411
|
+
if (!result.isError) this.pendingSerpCaptureBillingKeys.delete(fingerprint);
|
|
412
|
+
return result;
|
|
389
413
|
}
|
|
390
414
|
captureSerpPageSnapshots(input) {
|
|
391
415
|
return this.call("/serp-intelligence/page-snapshots", input, this.serpIntelligenceTimeoutMs);
|
|
@@ -528,7 +552,7 @@ render();
|
|
|
528
552
|
}
|
|
529
553
|
|
|
530
554
|
// src/version.ts
|
|
531
|
-
var PACKAGE_VERSION = "0.21.
|
|
555
|
+
var PACKAGE_VERSION = "0.21.4";
|
|
532
556
|
|
|
533
557
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
534
558
|
var import_zod = require("zod");
|
|
@@ -2045,7 +2069,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2045
2069
|
var import_mcp2 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
2046
2070
|
var import_node_fs5 = require("fs");
|
|
2047
2071
|
var import_node_path7 = require("path");
|
|
2048
|
-
var
|
|
2072
|
+
var import_node_crypto6 = require("crypto");
|
|
2049
2073
|
|
|
2050
2074
|
// src/mcp/mcp-response-formatter.ts
|
|
2051
2075
|
var import_node_fs4 = require("fs");
|
|
@@ -2602,7 +2626,7 @@ function renderImageSection(audit) {
|
|
|
2602
2626
|
}
|
|
2603
2627
|
|
|
2604
2628
|
// src/mcp/report-artifact-offload.ts
|
|
2605
|
-
var
|
|
2629
|
+
var import_node_crypto3 = require("crypto");
|
|
2606
2630
|
|
|
2607
2631
|
// src/api/blob-store.ts
|
|
2608
2632
|
var import_node_fs3 = require("fs");
|
|
@@ -2679,7 +2703,7 @@ var VercelBlobStore = class {
|
|
|
2679
2703
|
};
|
|
2680
2704
|
|
|
2681
2705
|
// src/api/connected-data-artifacts.ts
|
|
2682
|
-
var
|
|
2706
|
+
var import_node_crypto2 = require("crypto");
|
|
2683
2707
|
var import_promises2 = require("fs/promises");
|
|
2684
2708
|
var import_node_os5 = require("os");
|
|
2685
2709
|
var import_node_path5 = require("path");
|
|
@@ -2761,7 +2785,7 @@ var PREVIEW_CHARS = 2e3;
|
|
|
2761
2785
|
var ARTIFACT_OFFLOAD_ENABLED = process.env.MCP_SCRAPER_ARTIFACT_OFFLOAD !== "false";
|
|
2762
2786
|
async function offloadReport(toolName, ownerId, report) {
|
|
2763
2787
|
const timestamp = Date.now();
|
|
2764
|
-
const random = (0,
|
|
2788
|
+
const random = (0, import_node_crypto3.randomBytes)(6).toString("hex");
|
|
2765
2789
|
const key = `${REPORT_BLOB_PREFIX}${ownerId}/${toolName}/${timestamp}-${random}.md`;
|
|
2766
2790
|
const stored = await getBlobStore().put(key, report, "text/markdown");
|
|
2767
2791
|
return {
|
|
@@ -2805,7 +2829,9 @@ function summaryEnvelope(executiveSummary, offloaded) {
|
|
|
2805
2829
|
|
|
2806
2830
|
// src/api/rates.ts
|
|
2807
2831
|
var MC_COSTS = {
|
|
2808
|
-
serp:
|
|
2832
|
+
serp: 400,
|
|
2833
|
+
serp_headless: 400,
|
|
2834
|
+
serp_headful: 1400,
|
|
2809
2835
|
paa: 300,
|
|
2810
2836
|
paa_base: 5500,
|
|
2811
2837
|
page_scrape: 100,
|
|
@@ -2846,9 +2872,9 @@ var CREDIT_COST_CATALOG = [
|
|
|
2846
2872
|
key: "serp",
|
|
2847
2873
|
label: "SERP search",
|
|
2848
2874
|
aliases: ["search_serp", "serp", "google search", "organic results"],
|
|
2849
|
-
credits: mcToCredits(MC_COSTS.
|
|
2850
|
-
unit: "per search",
|
|
2851
|
-
notes:
|
|
2875
|
+
credits: mcToCredits(MC_COSTS.serp_headless),
|
|
2876
|
+
unit: "per search (headless); escalates to headful only when needed",
|
|
2877
|
+
notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded). Optional SERP Intelligence page snapshots add ${mcToCredits(MC_COSTS.page_scrape)} credit per attempted URL.`
|
|
2852
2878
|
},
|
|
2853
2879
|
{
|
|
2854
2880
|
key: "paa",
|
|
@@ -3078,14 +3104,15 @@ var MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc
|
|
|
3078
3104
|
var import_client = require("@fal-ai/client");
|
|
3079
3105
|
|
|
3080
3106
|
// src/api/cost-telemetry.ts
|
|
3081
|
-
var
|
|
3107
|
+
var import_node_crypto5 = require("crypto");
|
|
3082
3108
|
|
|
3083
3109
|
// src/api/db.ts
|
|
3084
3110
|
var import_http = require("@libsql/client/http");
|
|
3085
|
-
var
|
|
3111
|
+
var import_node_crypto4 = require("crypto");
|
|
3086
3112
|
var import_zod2 = require("zod");
|
|
3087
3113
|
var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
|
|
3088
3114
|
var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
|
|
3115
|
+
var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
|
|
3089
3116
|
var SiteAuditJobRowSchema = import_zod2.z.object({
|
|
3090
3117
|
id: import_zod2.z.string(),
|
|
3091
3118
|
user_id: import_zod2.z.number(),
|
|
@@ -5437,21 +5464,33 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
5437
5464
|
const parsed = parseData(raw);
|
|
5438
5465
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
5439
5466
|
const d = parsed.data;
|
|
5467
|
+
const harvestResult = d.harvestResult && typeof d.harvestResult === "object" ? d.harvestResult : d;
|
|
5440
5468
|
const providerPayload = d;
|
|
5441
|
-
const organic = Array.isArray(
|
|
5442
|
-
const localPack = Array.isArray(
|
|
5443
|
-
const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
5444
|
-
const
|
|
5445
|
-
const
|
|
5446
|
-
const
|
|
5447
|
-
const
|
|
5469
|
+
const organic = Array.isArray(harvestResult.organicResults) ? harvestResult.organicResults : Array.isArray(harvestResult.organic) ? harvestResult.organic : [];
|
|
5470
|
+
const localPack = Array.isArray(harvestResult.localPack) ? harvestResult.localPack : [];
|
|
5471
|
+
const artifacts = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
5472
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
5473
|
+
const diagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null;
|
|
5474
|
+
const completionStatus = diagnostics && typeof diagnostics.completionStatus === "string" ? diagnostics.completionStatus : null;
|
|
5475
|
+
const status = String(d.status ?? d.captureStatus ?? completionStatus ?? "captured");
|
|
5476
|
+
const query = typeof harvestResult.seed === "string" ? harvestResult.seed : typeof harvestResult.query === "string" ? harvestResult.query : typeof input.query === "string" ? input.query : null;
|
|
5477
|
+
const location = typeof harvestResult.location === "string" ? harvestResult.location : typeof input.location === "string" ? input.location : null;
|
|
5478
|
+
const capturedAt = typeof harvestResult.extractedAt === "string" ? harvestResult.extractedAt : typeof harvestResult.capturedAt === "string" ? harvestResult.capturedAt : null;
|
|
5448
5479
|
const resultCount = organic.length + localPack.length;
|
|
5480
|
+
const pageSnapshotFailedCount = artifacts.filter((artifact) => artifact.status === "failed" || Boolean(artifact.error)).length;
|
|
5481
|
+
const rawBilling = d.billing && typeof d.billing === "object" ? d.billing : null;
|
|
5482
|
+
const billing = rawBilling ? {
|
|
5483
|
+
creditsUsed: typeof rawBilling.creditsUsed === "number" ? rawBilling.creditsUsed : null,
|
|
5484
|
+
requestId: typeof rawBilling.requestId === "string" ? rawBilling.requestId : null,
|
|
5485
|
+
jobId: typeof rawBilling.jobId === "string" ? rawBilling.jobId : null
|
|
5486
|
+
} : null;
|
|
5449
5487
|
const full = [
|
|
5450
5488
|
`# SERP Intelligence Snapshot: ${query ?? "query"}`,
|
|
5451
5489
|
`**Status:** ${status}`,
|
|
5452
5490
|
location ? `**Location:** ${location}` : "",
|
|
5453
|
-
`**
|
|
5454
|
-
|
|
5491
|
+
`**SERP results:** ${resultCount} (${organic.length} organic, ${localPack.length} local pack)`,
|
|
5492
|
+
`**Page snapshots:** ${artifacts.length} (${pageSnapshotFailedCount} failed)`,
|
|
5493
|
+
billing?.creditsUsed !== null && billing?.creditsUsed !== void 0 ? `**Credits used:** ${billing.creditsUsed}` : "",
|
|
5455
5494
|
"",
|
|
5456
5495
|
"Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
|
|
5457
5496
|
].filter(Boolean).join("\n");
|
|
@@ -5464,6 +5503,10 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
5464
5503
|
location,
|
|
5465
5504
|
capturedAt,
|
|
5466
5505
|
resultCount,
|
|
5506
|
+
organicResultCount: organic.length,
|
|
5507
|
+
localPackResultCount: localPack.length,
|
|
5508
|
+
pageSnapshotCount: artifacts.length,
|
|
5509
|
+
pageSnapshotFailedCount,
|
|
5467
5510
|
snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
|
|
5468
5511
|
resolvedInputs: {
|
|
5469
5512
|
query: input.query ?? null,
|
|
@@ -5476,7 +5519,9 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
5476
5519
|
pages: input.pages ?? null
|
|
5477
5520
|
},
|
|
5478
5521
|
artifacts,
|
|
5479
|
-
|
|
5522
|
+
attempts,
|
|
5523
|
+
billing,
|
|
5524
|
+
diagnostics,
|
|
5480
5525
|
providerPayload
|
|
5481
5526
|
}
|
|
5482
5527
|
};
|
|
@@ -5485,9 +5530,13 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
5485
5530
|
const parsed = parseData(raw);
|
|
5486
5531
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
5487
5532
|
const d = parsed.data;
|
|
5488
|
-
const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
5533
|
+
const captures = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
5489
5534
|
const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
|
|
5490
|
-
const
|
|
5535
|
+
const capturedCount = captures.filter((c) => c.status === "captured" || c.status === "ok").length;
|
|
5536
|
+
const skippedCount = captures.filter((c) => c.status === "skipped").length;
|
|
5537
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
5538
|
+
const inferredStatus = captures.length === 0 ? "empty" : failedCount === captures.length ? "failed" : failedCount > 0 || skippedCount > 0 ? "partial" : "captured";
|
|
5539
|
+
const status = String(d.status ?? inferredStatus);
|
|
5491
5540
|
const rows = captures.slice(0, 25).map((capture, i) => {
|
|
5492
5541
|
const url = typeof capture.url === "string" ? capture.url : "";
|
|
5493
5542
|
const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
|
|
@@ -5496,8 +5545,10 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
5496
5545
|
const full = [
|
|
5497
5546
|
"# SERP Intelligence Page Snapshots",
|
|
5498
5547
|
`**Status:** ${status}`,
|
|
5499
|
-
`**
|
|
5548
|
+
`**Attempted:** ${captures.length}`,
|
|
5549
|
+
`**Captured:** ${capturedCount}`,
|
|
5500
5550
|
`**Failed:** ${failedCount}`,
|
|
5551
|
+
skippedCount ? `**Skipped:** ${skippedCount}` : "",
|
|
5501
5552
|
captures.length ? `
|
|
5502
5553
|
| # | URL | Source | Status |
|
|
5503
5554
|
|---|-----|--------|--------|
|
|
@@ -5509,8 +5560,11 @@ ${rows}` : ""
|
|
|
5509
5560
|
schemaVersion: "serp-intelligence.page-snapshots.v1",
|
|
5510
5561
|
status,
|
|
5511
5562
|
count: captures.length,
|
|
5563
|
+
capturedCount,
|
|
5512
5564
|
failedCount,
|
|
5565
|
+
skippedCount,
|
|
5513
5566
|
captures,
|
|
5567
|
+
attempts,
|
|
5514
5568
|
resolvedInputs: {
|
|
5515
5569
|
urls: input.urls ?? null,
|
|
5516
5570
|
targets: input.targets ?? null,
|
|
@@ -6877,9 +6931,19 @@ var CaptureSerpSnapshotOutputSchema = {
|
|
|
6877
6931
|
location: NullableString2,
|
|
6878
6932
|
capturedAt: NullableString2,
|
|
6879
6933
|
resultCount: import_zod4.z.number().int().min(0).nullable(),
|
|
6934
|
+
organicResultCount: import_zod4.z.number().int().min(0),
|
|
6935
|
+
localPackResultCount: import_zod4.z.number().int().min(0),
|
|
6936
|
+
pageSnapshotCount: import_zod4.z.number().int().min(0),
|
|
6937
|
+
pageSnapshotFailedCount: import_zod4.z.number().int().min(0),
|
|
6880
6938
|
snapshotId: NullableString2,
|
|
6881
6939
|
resolvedInputs: import_zod4.z.record(import_zod4.z.unknown()),
|
|
6882
6940
|
artifacts: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6941
|
+
attempts: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6942
|
+
billing: import_zod4.z.object({
|
|
6943
|
+
creditsUsed: import_zod4.z.number().min(0).nullable(),
|
|
6944
|
+
requestId: NullableString2,
|
|
6945
|
+
jobId: NullableString2
|
|
6946
|
+
}).nullable(),
|
|
6883
6947
|
diagnostics: import_zod4.z.record(import_zod4.z.unknown()).nullable(),
|
|
6884
6948
|
providerPayload: import_zod4.z.record(import_zod4.z.unknown())
|
|
6885
6949
|
};
|
|
@@ -6887,8 +6951,11 @@ var CaptureSerpPageSnapshotsOutputSchema = {
|
|
|
6887
6951
|
schemaVersion: import_zod4.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
6888
6952
|
status: import_zod4.z.string(),
|
|
6889
6953
|
count: import_zod4.z.number().int().min(0),
|
|
6954
|
+
capturedCount: import_zod4.z.number().int().min(0),
|
|
6890
6955
|
failedCount: import_zod4.z.number().int().min(0),
|
|
6956
|
+
skippedCount: import_zod4.z.number().int().min(0),
|
|
6891
6957
|
captures: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6958
|
+
attempts: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6892
6959
|
resolvedInputs: import_zod4.z.record(import_zod4.z.unknown()),
|
|
6893
6960
|
diagnostics: import_zod4.z.record(import_zod4.z.unknown()).nullable(),
|
|
6894
6961
|
providerPayload: import_zod4.z.record(import_zod4.z.unknown())
|
|
@@ -7013,8 +7080,8 @@ var CaptureSerpSnapshotInputSchema = {
|
|
|
7013
7080
|
proxyZip: import_zod4.z.string().regex(/^\d{5}$/).optional().describe('US ZIP for residential geo-IP targeting. Only meaningful with proxyMode "location".'),
|
|
7014
7081
|
pages: import_zod4.z.number().int().min(1).max(2).default(1).describe("Google result pages to capture. Use 2 only for deeper ranking evidence."),
|
|
7015
7082
|
debug: import_zod4.z.boolean().default(false).describe("Include sanitized browser/proxy/location diagnostics."),
|
|
7016
|
-
includePageSnapshots: import_zod4.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs."),
|
|
7017
|
-
pageSnapshotLimit: import_zod4.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true.")
|
|
7083
|
+
includePageSnapshots: import_zod4.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."),
|
|
7084
|
+
pageSnapshotLimit: import_zod4.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded.")
|
|
7018
7085
|
};
|
|
7019
7086
|
var ScreenshotInputSchema = {
|
|
7020
7087
|
url: import_zod4.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
@@ -7022,12 +7089,12 @@ var ScreenshotInputSchema = {
|
|
|
7022
7089
|
allowLocal: import_zod4.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
7023
7090
|
};
|
|
7024
7091
|
var CaptureSerpPageSnapshotsInputSchema = {
|
|
7025
|
-
urls: import_zod4.z.array(import_zod4.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
7092
|
+
urls: import_zod4.z.array(import_zod4.z.string().url()).min(1).max(25).optional().describe("Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
7026
7093
|
targets: import_zod4.z.array(import_zod4.z.object({
|
|
7027
7094
|
url: import_zod4.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
7028
7095
|
sourceKind: import_zod4.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured."),
|
|
7029
7096
|
sourcePosition: import_zod4.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
7030
|
-
}).strict()).min(1).max(25).optional().describe("Structured targets.
|
|
7097
|
+
}).strict()).min(1).max(25).optional().describe("Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."),
|
|
7031
7098
|
maxConcurrency: import_zod4.z.number().int().min(1).max(5).default(2).describe("Parallel page captures."),
|
|
7032
7099
|
timeoutMs: import_zod4.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
7033
7100
|
debug: import_zod4.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
@@ -7666,14 +7733,14 @@ function liveWebToolAnnotations(title) {
|
|
|
7666
7733
|
function registerSerpIntelligenceCaptureTools(server2, executor) {
|
|
7667
7734
|
server2.registerTool("capture_serp_snapshot", {
|
|
7668
7735
|
title: "SERP Intelligence Snapshot",
|
|
7669
|
-
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
7736
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
|
|
7670
7737
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
7671
7738
|
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
7672
7739
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
7673
7740
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
7674
7741
|
server2.registerTool("capture_serp_page_snapshots", {
|
|
7675
7742
|
title: "SERP Intelligence Page Snapshots",
|
|
7676
|
-
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls
|
|
7743
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
|
|
7677
7744
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
7678
7745
|
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
7679
7746
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|