mcp-scraper 0.21.3 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +728 -181
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +97 -32
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +21 -0
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-JJ5RMEGX.js → chunk-E5J4HJBO.js} +2 -2
- package/dist/{chunk-H7APH5UG.js → chunk-ICT7DDHL.js} +366 -50
- package/dist/chunk-ICT7DDHL.js.map +1 -0
- package/dist/{chunk-OBNKOYOE.js → chunk-JHI373VB.js} +7 -5
- package/dist/chunk-JHI373VB.js.map +1 -0
- package/dist/{chunk-5IU5ZFVO.js → chunk-MTSBI7ZH.js} +23 -6
- package/dist/chunk-MTSBI7ZH.js.map +1 -0
- package/dist/{chunk-7LO6KLKD.js → chunk-N4JHXPW6.js} +89 -27
- package/dist/chunk-N4JHXPW6.js.map +1 -0
- package/dist/{chunk-XDRHBGJR.js → chunk-PNBYYAEK.js} +3 -3
- package/dist/{chunk-2P5VFIJM.js → chunk-SDHPQTFM.js} +2 -2
- package/dist/chunk-WQI7JIIN.js +7 -0
- package/dist/chunk-WQI7JIIN.js.map +1 -0
- package/dist/{db-Z35S224E.js → db-WRILVOFY.js} +4 -2
- package/dist/{extract-bundle-4RO3YU4Q.js → extract-bundle-NFFVMUMT.js} +2 -2
- package/dist/index.cjs +21 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-PBBRSLBP.js → server-OLOOFN5W.js} +246 -100
- package/dist/server-OLOOFN5W.js.map +1 -0
- package/dist/{site-extract-repository-FDH6ZXKT.js → site-extract-repository-2QQSL2QC.js} +4 -4
- package/dist/{worker-BXJQ7J4N.js → worker-RV2TYXE3.js} +9 -8
- package/dist/worker-RV2TYXE3.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +90 -11
- package/docs/specs/oauth-mcp-spec.md +2 -2
- package/package.json +1 -1
- package/dist/chunk-5IU5ZFVO.js.map +0 -1
- package/dist/chunk-7LO6KLKD.js.map +0 -1
- package/dist/chunk-H7APH5UG.js.map +0 -1
- package/dist/chunk-OBNKOYOE.js.map +0 -1
- package/dist/chunk-VYUWWIVR.js +0 -7
- package/dist/chunk-VYUWWIVR.js.map +0 -1
- package/dist/server-PBBRSLBP.js.map +0 -1
- package/dist/worker-BXJQ7J4N.js.map +0 -1
- /package/dist/{chunk-JJ5RMEGX.js.map → chunk-E5J4HJBO.js.map} +0 -0
- /package/dist/{chunk-XDRHBGJR.js.map → chunk-PNBYYAEK.js.map} +0 -0
- /package/dist/{chunk-2P5VFIJM.js.map → chunk-SDHPQTFM.js.map} +0 -0
- /package/dist/{db-Z35S224E.js.map → db-WRILVOFY.js.map} +0 -0
- /package/dist/{extract-bundle-4RO3YU4Q.js.map → extract-bundle-NFFVMUMT.js.map} +0 -0
- /package/dist/{site-extract-repository-FDH6ZXKT.js.map → site-extract-repository-2QQSL2QC.js.map} +0 -0
|
@@ -106,7 +106,7 @@ function renderInstallTerminal(options) {
|
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
// src/version.ts
|
|
109
|
-
var PACKAGE_VERSION = "0.21.
|
|
109
|
+
var PACKAGE_VERSION = "0.21.4";
|
|
110
110
|
|
|
111
111
|
// bin/mcp-scraper-install.ts
|
|
112
112
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(159 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.21.3'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACnHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(159 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.21.4'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACnHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
} from "../chunk-YNHPZU6B.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-WQI7JIIN.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -30,6 +30,9 @@ var import_node_path8 = require("path");
|
|
|
30
30
|
var import_mcp3 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
31
31
|
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
32
32
|
|
|
33
|
+
// src/mcp/http-mcp-tool-executor.ts
|
|
34
|
+
var import_node_crypto = require("crypto");
|
|
35
|
+
|
|
33
36
|
// src/harvest-timeout.ts
|
|
34
37
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
35
38
|
var CLIENT_OVER_SERVER_MARGIN_MS = 15e3;
|
|
@@ -120,6 +123,7 @@ var HttpMcpToolExecutor = class {
|
|
|
120
123
|
timeoutMs;
|
|
121
124
|
httpTimeoutOverrideMs;
|
|
122
125
|
serpIntelligenceTimeoutMs;
|
|
126
|
+
pendingSerpCaptureBillingKeys = /* @__PURE__ */ new Map();
|
|
123
127
|
constructor(baseUrl2, apiKey2) {
|
|
124
128
|
this.baseUrl = baseUrl2.replace(/\/$/, "");
|
|
125
129
|
this.apiKey = apiKey2;
|
|
@@ -130,13 +134,14 @@ var HttpMcpToolExecutor = class {
|
|
|
130
134
|
const configuredSerpIntelligenceTimeoutMs = Number(process.env.MCP_SCRAPER_SERP_INTELLIGENCE_HTTP_TIMEOUT_MS ?? this.timeoutMs);
|
|
131
135
|
this.serpIntelligenceTimeoutMs = Number.isFinite(configuredSerpIntelligenceTimeoutMs) && configuredSerpIntelligenceTimeoutMs > 0 ? configuredSerpIntelligenceTimeoutMs : this.timeoutMs;
|
|
132
136
|
}
|
|
133
|
-
async call(path, body, timeoutMs = this.timeoutMs, method = "POST") {
|
|
137
|
+
async call(path, body, timeoutMs = this.timeoutMs, method = "POST", requestHeaders = {}) {
|
|
134
138
|
try {
|
|
135
139
|
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
136
140
|
method,
|
|
137
141
|
headers: {
|
|
138
142
|
"Content-Type": "application/json",
|
|
139
|
-
"x-api-key": this.apiKey
|
|
143
|
+
"x-api-key": this.apiKey,
|
|
144
|
+
...requestHeaders
|
|
140
145
|
},
|
|
141
146
|
body: JSON.stringify(body),
|
|
142
147
|
signal: AbortSignal.timeout(timeoutMs)
|
|
@@ -384,8 +389,27 @@ var HttpMcpToolExecutor = class {
|
|
|
384
389
|
connections: input.connections
|
|
385
390
|
}, void 0, "PUT");
|
|
386
391
|
}
|
|
387
|
-
captureSerpSnapshot(input) {
|
|
388
|
-
|
|
392
|
+
async captureSerpSnapshot(input) {
|
|
393
|
+
const fingerprint = (0, import_node_crypto.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
394
|
+
const now = Date.now();
|
|
395
|
+
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
396
|
+
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
397
|
+
}
|
|
398
|
+
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
|
|
399
|
+
const idempotencyKey = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto.randomUUID)();
|
|
400
|
+
this.pendingSerpCaptureBillingKeys.set(fingerprint, {
|
|
401
|
+
key: idempotencyKey,
|
|
402
|
+
expiresAt: now + 15 * 6e4
|
|
403
|
+
});
|
|
404
|
+
const result = await this.call(
|
|
405
|
+
"/serp-intelligence/capture",
|
|
406
|
+
input,
|
|
407
|
+
this.serpIntelligenceTimeoutMs,
|
|
408
|
+
"POST",
|
|
409
|
+
{ "Idempotency-Key": idempotencyKey }
|
|
410
|
+
);
|
|
411
|
+
if (!result.isError) this.pendingSerpCaptureBillingKeys.delete(fingerprint);
|
|
412
|
+
return result;
|
|
389
413
|
}
|
|
390
414
|
captureSerpPageSnapshots(input) {
|
|
391
415
|
return this.call("/serp-intelligence/page-snapshots", input, this.serpIntelligenceTimeoutMs);
|
|
@@ -528,7 +552,7 @@ render();
|
|
|
528
552
|
}
|
|
529
553
|
|
|
530
554
|
// src/version.ts
|
|
531
|
-
var PACKAGE_VERSION = "0.21.
|
|
555
|
+
var PACKAGE_VERSION = "0.21.4";
|
|
532
556
|
|
|
533
557
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
534
558
|
var import_zod = require("zod");
|
|
@@ -2045,7 +2069,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
2045
2069
|
var import_mcp2 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
2046
2070
|
var import_node_fs5 = require("fs");
|
|
2047
2071
|
var import_node_path7 = require("path");
|
|
2048
|
-
var
|
|
2072
|
+
var import_node_crypto6 = require("crypto");
|
|
2049
2073
|
|
|
2050
2074
|
// src/mcp/mcp-response-formatter.ts
|
|
2051
2075
|
var import_node_fs4 = require("fs");
|
|
@@ -2602,7 +2626,7 @@ function renderImageSection(audit) {
|
|
|
2602
2626
|
}
|
|
2603
2627
|
|
|
2604
2628
|
// src/mcp/report-artifact-offload.ts
|
|
2605
|
-
var
|
|
2629
|
+
var import_node_crypto3 = require("crypto");
|
|
2606
2630
|
|
|
2607
2631
|
// src/api/blob-store.ts
|
|
2608
2632
|
var import_node_fs3 = require("fs");
|
|
@@ -2679,7 +2703,7 @@ var VercelBlobStore = class {
|
|
|
2679
2703
|
};
|
|
2680
2704
|
|
|
2681
2705
|
// src/api/connected-data-artifacts.ts
|
|
2682
|
-
var
|
|
2706
|
+
var import_node_crypto2 = require("crypto");
|
|
2683
2707
|
var import_promises2 = require("fs/promises");
|
|
2684
2708
|
var import_node_os5 = require("os");
|
|
2685
2709
|
var import_node_path5 = require("path");
|
|
@@ -2761,7 +2785,7 @@ var PREVIEW_CHARS = 2e3;
|
|
|
2761
2785
|
var ARTIFACT_OFFLOAD_ENABLED = process.env.MCP_SCRAPER_ARTIFACT_OFFLOAD !== "false";
|
|
2762
2786
|
async function offloadReport(toolName, ownerId, report) {
|
|
2763
2787
|
const timestamp = Date.now();
|
|
2764
|
-
const random = (0,
|
|
2788
|
+
const random = (0, import_node_crypto3.randomBytes)(6).toString("hex");
|
|
2765
2789
|
const key = `${REPORT_BLOB_PREFIX}${ownerId}/${toolName}/${timestamp}-${random}.md`;
|
|
2766
2790
|
const stored = await getBlobStore().put(key, report, "text/markdown");
|
|
2767
2791
|
return {
|
|
@@ -2805,7 +2829,7 @@ function summaryEnvelope(executiveSummary, offloaded) {
|
|
|
2805
2829
|
|
|
2806
2830
|
// src/api/rates.ts
|
|
2807
2831
|
var MC_COSTS = {
|
|
2808
|
-
serp:
|
|
2832
|
+
serp: 400,
|
|
2809
2833
|
serp_headless: 400,
|
|
2810
2834
|
serp_headful: 1400,
|
|
2811
2835
|
paa: 300,
|
|
@@ -2850,7 +2874,7 @@ var CREDIT_COST_CATALOG = [
|
|
|
2850
2874
|
aliases: ["search_serp", "serp", "google search", "organic results"],
|
|
2851
2875
|
credits: mcToCredits(MC_COSTS.serp_headless),
|
|
2852
2876
|
unit: "per search (headless); escalates to headful only when needed",
|
|
2853
|
-
notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded).`
|
|
2877
|
+
notes: `Returns AI Overview, PAA snippet, videos, forums, and local pack. Billed ${mcToCredits(MC_COSTS.serp_headless)} credits when served headless (the common case), ${mcToCredits(MC_COSTS.serp_headful)} credits if an anti-bot escalation forces headful mode (overestimate refunded). Optional SERP Intelligence page snapshots add ${mcToCredits(MC_COSTS.page_scrape)} credit per attempted URL.`
|
|
2854
2878
|
},
|
|
2855
2879
|
{
|
|
2856
2880
|
key: "paa",
|
|
@@ -3080,14 +3104,15 @@ var MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc
|
|
|
3080
3104
|
var import_client = require("@fal-ai/client");
|
|
3081
3105
|
|
|
3082
3106
|
// src/api/cost-telemetry.ts
|
|
3083
|
-
var
|
|
3107
|
+
var import_node_crypto5 = require("crypto");
|
|
3084
3108
|
|
|
3085
3109
|
// src/api/db.ts
|
|
3086
3110
|
var import_http = require("@libsql/client/http");
|
|
3087
|
-
var
|
|
3111
|
+
var import_node_crypto4 = require("crypto");
|
|
3088
3112
|
var import_zod2 = require("zod");
|
|
3089
3113
|
var DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
|
|
3090
3114
|
var DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
|
|
3115
|
+
var REQUEST_EVENT_RESULT_MAX_BYTES = 32 * 1024;
|
|
3091
3116
|
var SiteAuditJobRowSchema = import_zod2.z.object({
|
|
3092
3117
|
id: import_zod2.z.string(),
|
|
3093
3118
|
user_id: import_zod2.z.number(),
|
|
@@ -5439,21 +5464,33 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
5439
5464
|
const parsed = parseData(raw);
|
|
5440
5465
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
5441
5466
|
const d = parsed.data;
|
|
5467
|
+
const harvestResult = d.harvestResult && typeof d.harvestResult === "object" ? d.harvestResult : d;
|
|
5442
5468
|
const providerPayload = d;
|
|
5443
|
-
const organic = Array.isArray(
|
|
5444
|
-
const localPack = Array.isArray(
|
|
5445
|
-
const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
5446
|
-
const
|
|
5447
|
-
const
|
|
5448
|
-
const
|
|
5449
|
-
const
|
|
5469
|
+
const organic = Array.isArray(harvestResult.organicResults) ? harvestResult.organicResults : Array.isArray(harvestResult.organic) ? harvestResult.organic : [];
|
|
5470
|
+
const localPack = Array.isArray(harvestResult.localPack) ? harvestResult.localPack : [];
|
|
5471
|
+
const artifacts = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
5472
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
5473
|
+
const diagnostics = harvestResult.diagnostics && typeof harvestResult.diagnostics === "object" ? harvestResult.diagnostics : d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null;
|
|
5474
|
+
const completionStatus = diagnostics && typeof diagnostics.completionStatus === "string" ? diagnostics.completionStatus : null;
|
|
5475
|
+
const status = String(d.status ?? d.captureStatus ?? completionStatus ?? "captured");
|
|
5476
|
+
const query = typeof harvestResult.seed === "string" ? harvestResult.seed : typeof harvestResult.query === "string" ? harvestResult.query : typeof input.query === "string" ? input.query : null;
|
|
5477
|
+
const location = typeof harvestResult.location === "string" ? harvestResult.location : typeof input.location === "string" ? input.location : null;
|
|
5478
|
+
const capturedAt = typeof harvestResult.extractedAt === "string" ? harvestResult.extractedAt : typeof harvestResult.capturedAt === "string" ? harvestResult.capturedAt : null;
|
|
5450
5479
|
const resultCount = organic.length + localPack.length;
|
|
5480
|
+
const pageSnapshotFailedCount = artifacts.filter((artifact) => artifact.status === "failed" || Boolean(artifact.error)).length;
|
|
5481
|
+
const rawBilling = d.billing && typeof d.billing === "object" ? d.billing : null;
|
|
5482
|
+
const billing = rawBilling ? {
|
|
5483
|
+
creditsUsed: typeof rawBilling.creditsUsed === "number" ? rawBilling.creditsUsed : null,
|
|
5484
|
+
requestId: typeof rawBilling.requestId === "string" ? rawBilling.requestId : null,
|
|
5485
|
+
jobId: typeof rawBilling.jobId === "string" ? rawBilling.jobId : null
|
|
5486
|
+
} : null;
|
|
5451
5487
|
const full = [
|
|
5452
5488
|
`# SERP Intelligence Snapshot: ${query ?? "query"}`,
|
|
5453
5489
|
`**Status:** ${status}`,
|
|
5454
5490
|
location ? `**Location:** ${location}` : "",
|
|
5455
|
-
`**
|
|
5456
|
-
|
|
5491
|
+
`**SERP results:** ${resultCount} (${organic.length} organic, ${localPack.length} local pack)`,
|
|
5492
|
+
`**Page snapshots:** ${artifacts.length} (${pageSnapshotFailedCount} failed)`,
|
|
5493
|
+
billing?.creditsUsed !== null && billing?.creditsUsed !== void 0 ? `**Credits used:** ${billing.creditsUsed}` : "",
|
|
5457
5494
|
"",
|
|
5458
5495
|
"Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
|
|
5459
5496
|
].filter(Boolean).join("\n");
|
|
@@ -5466,6 +5503,10 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
5466
5503
|
location,
|
|
5467
5504
|
capturedAt,
|
|
5468
5505
|
resultCount,
|
|
5506
|
+
organicResultCount: organic.length,
|
|
5507
|
+
localPackResultCount: localPack.length,
|
|
5508
|
+
pageSnapshotCount: artifacts.length,
|
|
5509
|
+
pageSnapshotFailedCount,
|
|
5469
5510
|
snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
|
|
5470
5511
|
resolvedInputs: {
|
|
5471
5512
|
query: input.query ?? null,
|
|
@@ -5478,7 +5519,9 @@ function formatCaptureSerpSnapshot(raw, input) {
|
|
|
5478
5519
|
pages: input.pages ?? null
|
|
5479
5520
|
},
|
|
5480
5521
|
artifacts,
|
|
5481
|
-
|
|
5522
|
+
attempts,
|
|
5523
|
+
billing,
|
|
5524
|
+
diagnostics,
|
|
5482
5525
|
providerPayload
|
|
5483
5526
|
}
|
|
5484
5527
|
};
|
|
@@ -5487,9 +5530,13 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
5487
5530
|
const parsed = parseData(raw);
|
|
5488
5531
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
5489
5532
|
const d = parsed.data;
|
|
5490
|
-
const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
5533
|
+
const captures = Array.isArray(d.pageSnapshotArtifacts) ? d.pageSnapshotArtifacts : Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
5491
5534
|
const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
|
|
5492
|
-
const
|
|
5535
|
+
const capturedCount = captures.filter((c) => c.status === "captured" || c.status === "ok").length;
|
|
5536
|
+
const skippedCount = captures.filter((c) => c.status === "skipped").length;
|
|
5537
|
+
const attempts = Array.isArray(d.attempts) ? d.attempts : [];
|
|
5538
|
+
const inferredStatus = captures.length === 0 ? "empty" : failedCount === captures.length ? "failed" : failedCount > 0 || skippedCount > 0 ? "partial" : "captured";
|
|
5539
|
+
const status = String(d.status ?? inferredStatus);
|
|
5493
5540
|
const rows = captures.slice(0, 25).map((capture, i) => {
|
|
5494
5541
|
const url = typeof capture.url === "string" ? capture.url : "";
|
|
5495
5542
|
const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
|
|
@@ -5498,8 +5545,10 @@ function formatCaptureSerpPageSnapshots(raw, input) {
|
|
|
5498
5545
|
const full = [
|
|
5499
5546
|
"# SERP Intelligence Page Snapshots",
|
|
5500
5547
|
`**Status:** ${status}`,
|
|
5501
|
-
`**
|
|
5548
|
+
`**Attempted:** ${captures.length}`,
|
|
5549
|
+
`**Captured:** ${capturedCount}`,
|
|
5502
5550
|
`**Failed:** ${failedCount}`,
|
|
5551
|
+
skippedCount ? `**Skipped:** ${skippedCount}` : "",
|
|
5503
5552
|
captures.length ? `
|
|
5504
5553
|
| # | URL | Source | Status |
|
|
5505
5554
|
|---|-----|--------|--------|
|
|
@@ -5511,8 +5560,11 @@ ${rows}` : ""
|
|
|
5511
5560
|
schemaVersion: "serp-intelligence.page-snapshots.v1",
|
|
5512
5561
|
status,
|
|
5513
5562
|
count: captures.length,
|
|
5563
|
+
capturedCount,
|
|
5514
5564
|
failedCount,
|
|
5565
|
+
skippedCount,
|
|
5515
5566
|
captures,
|
|
5567
|
+
attempts,
|
|
5516
5568
|
resolvedInputs: {
|
|
5517
5569
|
urls: input.urls ?? null,
|
|
5518
5570
|
targets: input.targets ?? null,
|
|
@@ -6879,9 +6931,19 @@ var CaptureSerpSnapshotOutputSchema = {
|
|
|
6879
6931
|
location: NullableString2,
|
|
6880
6932
|
capturedAt: NullableString2,
|
|
6881
6933
|
resultCount: import_zod4.z.number().int().min(0).nullable(),
|
|
6934
|
+
organicResultCount: import_zod4.z.number().int().min(0),
|
|
6935
|
+
localPackResultCount: import_zod4.z.number().int().min(0),
|
|
6936
|
+
pageSnapshotCount: import_zod4.z.number().int().min(0),
|
|
6937
|
+
pageSnapshotFailedCount: import_zod4.z.number().int().min(0),
|
|
6882
6938
|
snapshotId: NullableString2,
|
|
6883
6939
|
resolvedInputs: import_zod4.z.record(import_zod4.z.unknown()),
|
|
6884
6940
|
artifacts: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6941
|
+
attempts: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6942
|
+
billing: import_zod4.z.object({
|
|
6943
|
+
creditsUsed: import_zod4.z.number().min(0).nullable(),
|
|
6944
|
+
requestId: NullableString2,
|
|
6945
|
+
jobId: NullableString2
|
|
6946
|
+
}).nullable(),
|
|
6885
6947
|
diagnostics: import_zod4.z.record(import_zod4.z.unknown()).nullable(),
|
|
6886
6948
|
providerPayload: import_zod4.z.record(import_zod4.z.unknown())
|
|
6887
6949
|
};
|
|
@@ -6889,8 +6951,11 @@ var CaptureSerpPageSnapshotsOutputSchema = {
|
|
|
6889
6951
|
schemaVersion: import_zod4.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
6890
6952
|
status: import_zod4.z.string(),
|
|
6891
6953
|
count: import_zod4.z.number().int().min(0),
|
|
6954
|
+
capturedCount: import_zod4.z.number().int().min(0),
|
|
6892
6955
|
failedCount: import_zod4.z.number().int().min(0),
|
|
6956
|
+
skippedCount: import_zod4.z.number().int().min(0),
|
|
6893
6957
|
captures: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6958
|
+
attempts: import_zod4.z.array(import_zod4.z.record(import_zod4.z.unknown())),
|
|
6894
6959
|
resolvedInputs: import_zod4.z.record(import_zod4.z.unknown()),
|
|
6895
6960
|
diagnostics: import_zod4.z.record(import_zod4.z.unknown()).nullable(),
|
|
6896
6961
|
providerPayload: import_zod4.z.record(import_zod4.z.unknown())
|
|
@@ -7015,8 +7080,8 @@ var CaptureSerpSnapshotInputSchema = {
|
|
|
7015
7080
|
proxyZip: import_zod4.z.string().regex(/^\d{5}$/).optional().describe('US ZIP for residential geo-IP targeting. Only meaningful with proxyMode "location".'),
|
|
7016
7081
|
pages: import_zod4.z.number().int().min(1).max(2).default(1).describe("Google result pages to capture. Use 2 only for deeper ranking evidence."),
|
|
7017
7082
|
debug: import_zod4.z.boolean().default(false).describe("Include sanitized browser/proxy/location diagnostics."),
|
|
7018
|
-
includePageSnapshots: import_zod4.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs."),
|
|
7019
|
-
pageSnapshotLimit: import_zod4.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true.")
|
|
7083
|
+
includePageSnapshots: import_zod4.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs. Each attempted snapshot adds 1 Credit."),
|
|
7084
|
+
pageSnapshotLimit: import_zod4.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots when includePageSnapshots is true. This capacity is held up front and unused capacity is refunded.")
|
|
7020
7085
|
};
|
|
7021
7086
|
var ScreenshotInputSchema = {
|
|
7022
7087
|
url: import_zod4.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
@@ -7024,12 +7089,12 @@ var ScreenshotInputSchema = {
|
|
|
7024
7089
|
allowLocal: import_zod4.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
7025
7090
|
};
|
|
7026
7091
|
var CaptureSerpPageSnapshotsInputSchema = {
|
|
7027
|
-
urls: import_zod4.z.array(import_zod4.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
7092
|
+
urls: import_zod4.z.array(import_zod4.z.string().url()).min(1).max(25).optional().describe("Public HTTP/HTTPS URLs to capture. Provide exactly one of urls or targets. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
7028
7093
|
targets: import_zod4.z.array(import_zod4.z.object({
|
|
7029
7094
|
url: import_zod4.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
7030
7095
|
sourceKind: import_zod4.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured."),
|
|
7031
7096
|
sourcePosition: import_zod4.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
7032
|
-
}).strict()).min(1).max(25).optional().describe("Structured targets.
|
|
7097
|
+
}).strict()).min(1).max(25).optional().describe("Structured targets. Provide exactly one of targets or urls; use targets when source kind or position should be preserved."),
|
|
7033
7098
|
maxConcurrency: import_zod4.z.number().int().min(1).max(5).default(2).describe("Parallel page captures."),
|
|
7034
7099
|
timeoutMs: import_zod4.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
7035
7100
|
debug: import_zod4.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
@@ -7668,14 +7733,14 @@ function liveWebToolAnnotations(title) {
|
|
|
7668
7733
|
function registerSerpIntelligenceCaptureTools(server2, executor) {
|
|
7669
7734
|
server2.registerTool("capture_serp_snapshot", {
|
|
7670
7735
|
title: "SERP Intelligence Snapshot",
|
|
7671
|
-
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
7736
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset. Costs 4 Credits when headless or 14 if anti-bot escalation requires headful mode; the 14-Credit hold is settled to the mode used. Optional page snapshots add 1 Credit per attempted URL.",
|
|
7672
7737
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
7673
7738
|
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
7674
7739
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
7675
7740
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
7676
7741
|
server2.registerTool("capture_serp_page_snapshots", {
|
|
7677
7742
|
title: "SERP Intelligence Page Snapshots",
|
|
7678
|
-
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls
|
|
7743
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide exactly one of urls or targets; use targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected. Costs 1 Credit per attempted URL.",
|
|
7679
7744
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
7680
7745
|
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
7681
7746
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|