mcp-scraper 0.64.1 → 0.65.1

package/dist/bin/mcp-scraper-cli.js

@@ -13,7 +13,7 @@ import "../chunk-FG6LJGJ7.js";
 import "../chunk-UVFB24RZ.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-3WNPHRGG.js";
+} from "../chunk-CGKWNXJR.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-install.cjs

@@ -53,7 +53,7 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(258 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(259 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
     toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
     toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
@@ -107,7 +107,7 @@ function renderInstallTerminal(options) {
 }
 
 // src/version.ts
-var PACKAGE_VERSION = "0.64.1";
+var PACKAGE_VERSION = "0.65.1";
 
 // bin/mcp-scraper-install.ts
 var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;

package/dist/bin/mcp-scraper-install.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(258 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n    toolRow('browser', ['serp_identity_create', 'serp_identity_list', 'browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n    toolRow('commons', ['commons_search_entities', 'commons_get_entity_linkset', 'commons_prepare_entity', 'commons_submit_entity', 'commons_prepare_publication', 'commons_claim_publication', 'commons_publish_editorial', 'commons_get_publication'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Hosted browser sessions use direct/no-proxy egress by default.',\n    'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n    'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n    'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n    'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.64.1'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n  process.argv.includes('--no-color') ||\n  process.env.NO_COLOR !== undefined ||\n  process.env.FORCE_COLOR === '0' ||\n  !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n  process.stdout.write([\n    'Usage: mcp-scraper-install [--no-color]',\n    '',\n    'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n    'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n    '',\n  ].join('\\n'))\n  process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n  version: PACKAGE_VERSION,\n  color: !noColor,\n  apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,wBAAwB,sBAAsB,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACtQ,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,2BAA2B,8BAA8B,0BAA0B,yBAAyB,+BAA+B,6BAA6B,6BAA6B,yBAAyB,GAAG,KAAK;AAAA,IAC1P,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACpHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
+{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(259 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n    toolRow('browser', ['serp_identity_create', 'serp_identity_list', 'browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n    toolRow('commons', ['commons_search_entities', 'commons_get_entity_linkset', 'commons_prepare_entity', 'commons_submit_entity', 'commons_prepare_publication', 'commons_claim_publication', 'commons_publish_editorial', 'commons_get_publication'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Hosted browser sessions use direct/no-proxy egress by default.',\n    'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n    'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n    'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n    'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.65.1'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n  process.argv.includes('--no-color') ||\n  process.env.NO_COLOR !== undefined ||\n  process.env.FORCE_COLOR === '0' ||\n  !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n  process.stdout.write([\n    'Usage: mcp-scraper-install [--no-color]',\n    '',\n    'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n    'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n    '',\n  ].join('\\n'))\n  process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n  version: PACKAGE_VERSION,\n  color: !noColor,\n  apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,wBAAwB,sBAAsB,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACtQ,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,2BAA2B,8BAA8B,0BAA0B,yBAAyB,+BAA+B,6BAA6B,6BAA6B,yBAAyB,GAAG,KAAK;AAAA,IAC1P,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACpHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}

package/dist/bin/mcp-scraper-install.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import {
   renderInstallTerminal
-} from "../chunk-3NXLGTC6.js";
+} from "../chunk-DCWXVAQT.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-3WNPHRGG.js";
+} from "../chunk-CGKWNXJR.js";
 
 // bin/mcp-scraper-install.ts
 var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;

package/dist/bin/mcp-stdio-server.cjs

@@ -624,6 +624,19 @@ var HttpMcpToolExecutor = class {
     const { idempotencyKey, ...body } = input;
     return this.callSiteExtractStart("extract_site", { ...body, background: true }, idempotencyKey);
   }
+  analyzeSiteSimilarity(input) {
+    const { idempotencyKey, similarityThreshold, similarityMaxPairs, ...body } = input;
+    return this.callSiteExtractStart("analyze_site_similarity", {
+      ...body,
+      background: true,
+      formats: ["markdown", "links", "json"],
+      renderJavaScript: true,
+      captureRenderedDom: true,
+      semanticSimilarity: true,
+      similarityThreshold,
+      similarityMaxPairs
+    }, idempotencyKey);
+  }
   auditSite(input) {
     const { idempotencyKey, ...body } = input;
     const requestBody = {
@@ -1209,7 +1222,7 @@ render();
 }
 
 // src/version.ts
-var PACKAGE_VERSION = "0.64.1";
+var PACKAGE_VERSION = "0.65.1";
 
 // src/mcp/input-field-descriptions.ts
 var import_zod = require("zod");
@@ -2217,6 +2230,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
 var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
   "extract_url",
   "extract_site",
+  "analyze_site_similarity",
   "audit_site",
   "check_site_export",
   "site_export_read",
@@ -5720,7 +5734,7 @@ function buildSeoExport(siteUrl, pages, branding) {
     branding: branding ?? void 0
   };
 }
-function formatBackgroundJobStarted(toolLabel, data, delivery) {
+function formatBackgroundJobStarted(toolLabel, url, data, delivery) {
   if (data.status !== "pending" || typeof data.jobId !== "string") return null;
   const jobId = data.jobId;
   const full = [
@@ -5735,6 +5749,7 @@ Poll \`check_site_export\` with this jobId. When ready, use \`site_export_read\`
   return {
     content: [{ type: "text", text: full }],
     structuredContent: {
+      url,
       jobId,
       status: "pending",
       requestedMaxPages: data.requestedMaxPages,
@@ -5755,7 +5770,8 @@ async function formatExtractSite(raw, input, ctx) {
   const parsed = parseData(raw);
   if ("error" in parsed) return formattedErrorResult(parsed.error);
   const started = formatBackgroundJobStarted(
-    "Multi-Page Site Content Crawl",
+    input.toolLabel ?? "Multi-Page Site Content Crawl",
+    input.url,
     parsed.data,
     {
       requested: input.delivery ?? "auto",
@@ -5886,6 +5902,7 @@ async function formatAuditSite(raw, input, ctx) {
   if ("error" in parsed) return formattedErrorResult(parsed.error);
   const started = formatBackgroundJobStarted(
     "Technical SEO Audit",
+    input.url,
     parsed.data,
     {
       requested: input.delivery ?? "auto",
@@ -8185,6 +8202,8 @@ seam is noted so you can chain them.
 - Whole site -> **extract_site** (takes a url). It durably retains complete per-page JSON, acquired HTML,
   and Markdown. Poll **check_site_export**, then call **site_export_read** for the manifest or a page view;
   call **site_export_image** for downloaded image IDs.
+- JavaScript-rendered page overlap/cannibalization -> **analyze_site_similarity**. It returns dedicated
+  raw-cosine page pairs, corpus percentiles, threshold clusters, and rendered-page artifacts.
 - Wayback replay URLs work with the same tools: \`extract_url\` removes playback chrome and can return
   a featured image; \`extract_site\` batches nearby archived HTML captures for the replayed site.
 - For multiple archive months, pass \`extract_site.wayback\` with explicit \`months\` or a \`from\`/\`to\`
@@ -9739,6 +9758,13 @@ var ExtractSiteInputSchema = {
   preserveMedia: import_zod7.z.boolean().default(false).describe("Include supported images in the export bundle. This is the preferred replacement for downloadImages."),
   downloadImages: import_zod7.z.boolean().optional().describe("Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, image preservation defaults to false.")
 };
+var AnalyzeSiteSimilarityInputSchema = {
+  url: WebsiteUrlOrDomainSchema.describe("Public live site to render and compare. Bare domains default to https://."),
+  maxPages: import_zod7.z.number().int().min(2).max(500).default(100).describe("Maximum rendered pages to compare; default 100, maximum 500."),
+  similarityThreshold: import_zod7.z.number().min(0).max(1).default(0.9).describe("Minimum raw cosine score retained in the pair table; default 0.90."),
+  similarityMaxPairs: import_zod7.z.number().int().min(1).max(5e4).default(1e4).describe("Maximum scored pairs retained, highest first; default 10,000."),
+  idempotencyKey: import_zod7.z.string().trim().min(8).max(200).describe("Required unique opaque ID for this intended analysis. Reuse only when retrying the same call; use a new value for an intentional rerun.")
+};
 var AuditSiteInputSchema = {
   url: WebsiteUrlOrDomainSchema.describe("Public website URL or domain for a full technical SEO audit (issues, link graph, indexability, headings, images). Bare domains default to https://. For plain content use extract_site instead."),
   maxPages: import_zod7.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. MCP audits always run as durable background exports and return a jobId; poll check_site_export for the hosted audit ZIP."),
@@ -9751,17 +9777,17 @@ var AuditSiteInputSchema = {
   downloadImages: import_zod7.z.boolean().optional().describe("Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, image preservation defaults to false.")
 };
 var CheckSiteExportInputSchema = {
-  jobId: import_zod7.z.string().min(1).describe("The jobId returned by extract_site or audit_site. Poll until status is complete, partial, or failed; partial jobs still return a downloadable bundle with successful pages and failure details.")
+  jobId: import_zod7.z.string().min(1).describe("The jobId returned by extract_site, analyze_site_similarity, or audit_site. Poll until status is complete, partial, or failed; partial jobs still return successful content and failure details.")
 };
 var SiteExportReadInputSchema = {
-  jobId: import_zod7.z.string().min(1).describe("Site export job ID returned by extract_site or audit_site."),
+  jobId: import_zod7.z.string().min(1).describe("Site export job ID returned by extract_site, analyze_site_similarity, or audit_site."),
   pageId: import_zod7.z.string().regex(/^[a-f0-9]{64}$/).optional().describe("Page ID returned by a manifest read. Omit to list the export manifest."),
   format: import_zod7.z.enum(["manifest", "json", "html", "markdown"]).default("manifest").describe("manifest lists page/image IDs; JSON, HTML, and Markdown read one page representation."),
   offset: import_zod7.z.number().int().min(0).default(0).describe("UTF-8 byte offset. Continue from nextOffset until it is null."),
   maxBytes: import_zod7.z.number().int().min(1).max(1e6).default(64e3).describe("Maximum UTF-8 bytes returned in this window.")
 };
 var SiteExportImageInputSchema = {
-  jobId: import_zod7.z.string().min(1).describe("Site export job ID returned by extract_site or audit_site."),
+  jobId: import_zod7.z.string().min(1).describe("Site export job ID returned by extract_site, analyze_site_similarity, or audit_site."),
   imageId: import_zod7.z.string().regex(/^[a-f0-9]{64}$/).describe("Downloaded image ID returned by a site_export_read manifest.")
 };
 var ArchiveReadInputSchema = {
@@ -13577,6 +13603,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     await formatExtractSite(await executor.extractSite(input), input, ctx),
     requestContext
   ));
+  server.registerTool("analyze_site_similarity", {
+    title: "Rendered Site Content Similarity",
+    description: "Find overlapping or competing live pages. Forces JavaScript rendering, retains bounded inert DOM plus clean Markdown, and returns raw-cosine pairs, corpus percentiles, clusters, and table-ready files in a private ZIP. Poll check_site_export; the scores measure content overlap, not intent or proof of cannibalization.",
+    inputSchema: AnalyzeSiteSimilarityInputSchema,
+    outputSchema: recordOutputSchema("analyze_site_similarity", ExtractSiteOutputSchema),
+    annotations: { ...liveWebToolAnnotations("Rendered Site Content Similarity"), readOnlyHint: false }
+  }, async (input, requestContext) => tasks.taskify(
+    "site_export",
+    await formatExtractSite(
+      await executor.analyzeSiteSimilarity(input),
+      { ...input, toolLabel: "Rendered Site Content Similarity" },
+      ctx
+    ),
+    requestContext
+  ));
   server.registerTool("audit_site", {
     title: "Technical SEO Audit",
     description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. Pass a new idempotencyKey for each intended audit and reuse it only when retrying that call. Every MCP audit starts a durable export; poll check_site_export for discovered, attempted, successful, failed, and remaining counts plus ${fileBehavior("the saved ZIP.", "the owner-scoped downloadable ZIP.")} Use extract_site instead for plain page content.`,
@@ -13590,7 +13631,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   ));
   server.registerTool("check_site_export", {
     title: "Check Site Export",
-    description: "Poll a background extract_site or audit_site job. Reports discovered, attempted, successful, failed, and remaining pages plus a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback through site_export_read/site_export_image and also return a downloadable ZIP.",
+    description: "Poll a background extract_site, analyze_site_similarity, or audit_site job. Reports page counters and a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback and a downloadable ZIP.",
     inputSchema: CheckSiteExportInputSchema,
     outputSchema: recordOutputSchema("check_site_export", CheckSiteExportOutputSchema),
     annotations: { ...liveWebToolAnnotations("Check Site Export"), readOnlyHint: false }
@@ -17001,7 +17042,7 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(258 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(259 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
     toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
     toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),