mcp-scraper 0.40.2 → 0.40.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +1517 -236
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +155 -87
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +6 -5
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +31 -9
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +4 -4
- package/dist/{chunk-XORPNO3Z.js → chunk-27DUCRAZ.js} +2 -2
- package/dist/{chunk-44HZLHDV.js → chunk-5UN33CGU.js} +38 -1
- package/dist/chunk-5UN33CGU.js.map +1 -0
- package/dist/{chunk-X2LKCX6H.js → chunk-6J57U6HA.js} +3 -3
- package/dist/{chunk-LP6E462I.js → chunk-6SZ52BQ5.js} +49 -101
- package/dist/chunk-6SZ52BQ5.js.map +1 -0
- package/dist/chunk-BBI7RGOT.js +99 -0
- package/dist/chunk-BBI7RGOT.js.map +1 -0
- package/dist/{chunk-NVXNEOUQ.js → chunk-CS4HE6QY.js} +47 -11
- package/dist/chunk-CS4HE6QY.js.map +1 -0
- package/dist/{chunk-3PIWJS6Y.js → chunk-EHES33KB.js} +2 -2
- package/dist/{chunk-GUVKHCKE.js → chunk-IQTT7CAB.js} +94 -3
- package/dist/chunk-IQTT7CAB.js.map +1 -0
- package/dist/{chunk-7XBBFBYY.js → chunk-JVIW4GK2.js} +44 -12
- package/dist/chunk-JVIW4GK2.js.map +1 -0
- package/dist/chunk-XU4ZLIB2.js +7 -0
- package/dist/chunk-XU4ZLIB2.js.map +1 -0
- package/dist/{db-W3CP562I.js → db-LVVU6NOK.js} +16 -2
- package/dist/{extract-bundle-K4PG3RZJ.js → extract-bundle-SKXEDG4Z.js} +4 -4
- package/dist/index.cjs +31 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +4 -4
- package/dist/{location-data-repository-RLQX6SNM.js → location-data-repository-NBTCOA7B.js} +3 -3
- package/dist/{server-KM3CFWCF.js → server-W76RUYD3.js} +1108 -66
- package/dist/server-W76RUYD3.js.map +1 -0
- package/dist/{site-extract-repository-2SMMFKKL.js → site-extract-repository-6GX72U4N.js} +4 -4
- package/dist/{worker-FXAGFYOE.js → worker-5RWLFSAJ.js} +27 -28
- package/dist/worker-5RWLFSAJ.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-44HZLHDV.js.map +0 -1
- package/dist/chunk-7XBBFBYY.js.map +0 -1
- package/dist/chunk-GUVKHCKE.js.map +0 -1
- package/dist/chunk-LP6E462I.js.map +0 -1
- package/dist/chunk-NVXNEOUQ.js.map +0 -1
- package/dist/chunk-ZUJLSICT.js +0 -7
- package/dist/chunk-ZUJLSICT.js.map +0 -1
- package/dist/server-KM3CFWCF.js.map +0 -1
- package/dist/worker-FXAGFYOE.js.map +0 -1
- /package/dist/{chunk-XORPNO3Z.js.map → chunk-27DUCRAZ.js.map} +0 -0
- /package/dist/{chunk-X2LKCX6H.js.map → chunk-6J57U6HA.js.map} +0 -0
- /package/dist/{chunk-3PIWJS6Y.js.map → chunk-EHES33KB.js.map} +0 -0
- /package/dist/{db-W3CP562I.js.map → db-LVVU6NOK.js.map} +0 -0
- /package/dist/{extract-bundle-K4PG3RZJ.js.map → extract-bundle-SKXEDG4Z.js.map} +0 -0
- /package/dist/{location-data-repository-RLQX6SNM.js.map → location-data-repository-NBTCOA7B.js.map} +0 -0
- /package/dist/{site-extract-repository-2SMMFKKL.js.map → site-extract-repository-6GX72U4N.js.map} +0 -0
|
@@ -106,7 +106,7 @@ function renderInstallTerminal(options) {
|
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
// src/version.ts
|
|
109
|
-
var PACKAGE_VERSION = "0.40.
|
|
109
|
+
var PACKAGE_VERSION = "0.40.3";
|
|
110
110
|
|
|
111
111
|
// bin/mcp-scraper-install.ts
|
|
112
112
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(175 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.40.2'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACnHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/install-terminal.ts","../../src/version.ts","../../bin/mcp-scraper-install.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(175 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n","export const PACKAGE_VERSION = '0.40.3'\n","#!/usr/bin/env node\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\nconst help = process.argv.includes('--help') || process.argv.includes('-h')\nif (help) {\n process.stdout.write([\n 'Usage: mcp-scraper-install [--no-color]',\n '',\n 'Prints the branded MCP Scraper terminal install card and copyable install commands.',\n 'mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.',\n '',\n ].join('\\n'))\n process.exit(0)\n}\n\nprocess.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n}))\n"],"mappings":";;;;AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;;;ACnHO,IAAM,kBAAkB;;;ACI/B,IAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,IAAM,OAAO,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAC1E,IAAI,MAAM;AACR,UAAQ,OAAO,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI,CAAC;AACZ,UAAQ,KAAK,CAAC;AAChB;AAEA,QAAQ,OAAO,MAAM,sBAAsB;AAAA,EACzC,SAAS;AAAA,EACT,OAAO,CAAC;AAAA,EACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AACnE,CAAC,CAAC;","names":[]}
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
} from "../chunk-XELSA2MS.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-XU4ZLIB2.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -60,6 +60,120 @@ function browserServiceProfileSaveChanges() {
|
|
|
60
60
|
return void 0;
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
// src/errors.ts
|
|
64
|
+
var PUBLIC_SERVICE_UNAVAILABLE_MESSAGE = "This operation is temporarily unavailable. Please retry later.";
|
|
65
|
+
var PUBLIC_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
66
|
+
"request_aborted",
|
|
67
|
+
"captcha_exhausted",
|
|
68
|
+
"captcha_or_blocked",
|
|
69
|
+
"location_mismatch",
|
|
70
|
+
"proxy_tunnel_failed",
|
|
71
|
+
"proxy_unavailable",
|
|
72
|
+
"harvest_timeout",
|
|
73
|
+
"mcp_request_timeout",
|
|
74
|
+
"service_unavailable"
|
|
75
|
+
]);
|
|
76
|
+
function normalizePublicErrorCode(code) {
|
|
77
|
+
return typeof code === "string" && PUBLIC_ERROR_CODES.has(code) ? code : "service_unavailable";
|
|
78
|
+
}
|
|
79
|
+
function publicErrorMessage(code) {
|
|
80
|
+
switch (normalizePublicErrorCode(code)) {
|
|
81
|
+
case "request_aborted":
|
|
82
|
+
return "The request was cancelled before it completed.";
|
|
83
|
+
case "captcha_exhausted":
|
|
84
|
+
case "captcha_or_blocked":
|
|
85
|
+
return "The target site returned a verification challenge. Please retry later.";
|
|
86
|
+
case "location_mismatch":
|
|
87
|
+
return "The target site returned results for a different location. Please retry.";
|
|
88
|
+
case "proxy_tunnel_failed":
|
|
89
|
+
case "proxy_unavailable":
|
|
90
|
+
return "The web request could not establish a working connection. Please retry later.";
|
|
91
|
+
case "harvest_timeout":
|
|
92
|
+
case "mcp_request_timeout":
|
|
93
|
+
return "The request timed out before it completed. Please retry with a smaller request.";
|
|
94
|
+
default:
|
|
95
|
+
return PUBLIC_SERVICE_UNAVAILABLE_MESSAGE;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function sanitizeVendorName(message) {
|
|
99
|
+
return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/ +/g, " ").trim();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// src/api/outbound-sanitize.ts
|
|
103
|
+
var KEY_RENAMES = {
|
|
104
|
+
kernel: "browserRuntime"
|
|
105
|
+
};
|
|
106
|
+
var SANITIZED_VALUE_KEYS = /error|message/i;
|
|
107
|
+
var INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
|
|
108
|
+
"kernel_session_id",
|
|
109
|
+
"browser_session_id",
|
|
110
|
+
"kernelSessionId",
|
|
111
|
+
"browserSessionId",
|
|
112
|
+
"sessionId",
|
|
113
|
+
"kernel_delete_started",
|
|
114
|
+
"session_cleanup_started",
|
|
115
|
+
"kernelDeleteStarted",
|
|
116
|
+
"sessionCleanupStarted",
|
|
117
|
+
"kernel_delete_succeeded",
|
|
118
|
+
"session_cleanup_succeeded",
|
|
119
|
+
"kernelDeleteSucceeded",
|
|
120
|
+
"sessionCleanupSucceeded",
|
|
121
|
+
"kernel_delete_error",
|
|
122
|
+
"session_cleanup_error",
|
|
123
|
+
"kernelDeleteError",
|
|
124
|
+
"sessionCleanupError",
|
|
125
|
+
"kernelProxyId",
|
|
126
|
+
"proxyId",
|
|
127
|
+
"requestedProxyIdSuffix",
|
|
128
|
+
"retrievedProxyIdSuffix",
|
|
129
|
+
"provider",
|
|
130
|
+
"providerName",
|
|
131
|
+
"service",
|
|
132
|
+
"serviceName",
|
|
133
|
+
"vendor",
|
|
134
|
+
"vendorName",
|
|
135
|
+
"organization",
|
|
136
|
+
"organizationId",
|
|
137
|
+
"accountId"
|
|
138
|
+
]);
|
|
139
|
+
var VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
|
|
140
|
+
"hosted_url",
|
|
141
|
+
"hostedUrl",
|
|
142
|
+
"live_view_url",
|
|
143
|
+
"liveViewUrl",
|
|
144
|
+
"cdp_ws_url",
|
|
145
|
+
"cdpWsUrl",
|
|
146
|
+
"browser_live_view_url"
|
|
147
|
+
]);
|
|
148
|
+
var VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
|
|
149
|
+
function redactVendorUrls(value) {
|
|
150
|
+
return value.replace(VENDOR_URL_RE, "[browser-service]");
|
|
151
|
+
}
|
|
152
|
+
function sanitizeOutboundDiagnostics(value, parentKey = "") {
|
|
153
|
+
if (typeof value === "string") {
|
|
154
|
+
let out = redactVendorUrls(value);
|
|
155
|
+
if (SANITIZED_VALUE_KEYS.test(parentKey)) {
|
|
156
|
+
out = publicErrorMessage("service_unavailable");
|
|
157
|
+
}
|
|
158
|
+
return out;
|
|
159
|
+
}
|
|
160
|
+
if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
|
|
161
|
+
if (value !== null && typeof value === "object") {
|
|
162
|
+
const out = {};
|
|
163
|
+
for (const [key, val] of Object.entries(value)) {
|
|
164
|
+
if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
|
|
165
|
+
const renamed = KEY_RENAMES[key] ?? key;
|
|
166
|
+
if (VENDOR_URL_KEYS.has(key)) {
|
|
167
|
+
out[renamed] = null;
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
out[renamed] = sanitizeOutboundDiagnostics(val, key);
|
|
171
|
+
}
|
|
172
|
+
return out;
|
|
173
|
+
}
|
|
174
|
+
return value;
|
|
175
|
+
}
|
|
176
|
+
|
|
63
177
|
// src/mcp/http-mcp-tool-executor.ts
|
|
64
178
|
function youtubeVideoIdFromUrl(url) {
|
|
65
179
|
if (!url) return null;
|
|
@@ -94,7 +208,20 @@ async function readResponseData(res) {
|
|
|
94
208
|
function httpErrorPayload(path, res, data) {
|
|
95
209
|
const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
|
|
96
210
|
const rawCode = objectData?.code ?? objectData?.errorCode ?? objectData?.error_code;
|
|
97
|
-
const
|
|
211
|
+
const candidateCode = typeof rawCode === "string" && /^[a-z0-9][a-z0-9_-]{0,99}$/.test(rawCode) ? rawCode : "mcp_http_error";
|
|
212
|
+
if (res.status >= 500) {
|
|
213
|
+
const safeCode2 = normalizePublicErrorCode(candidateCode);
|
|
214
|
+
return {
|
|
215
|
+
error: safeCode2,
|
|
216
|
+
error_code: safeCode2,
|
|
217
|
+
error_type: "http",
|
|
218
|
+
retryable: true,
|
|
219
|
+
status: res.status,
|
|
220
|
+
path,
|
|
221
|
+
message: publicErrorMessage(safeCode2)
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
const safeCode = candidateCode;
|
|
98
225
|
const bodyMessage = objectData ? objectData.message ?? objectData.error ?? rawCode : data;
|
|
99
226
|
return {
|
|
100
227
|
...objectData ?? { body: data },
|
|
@@ -153,7 +280,6 @@ var HttpMcpToolExecutor = class {
|
|
|
153
280
|
}
|
|
154
281
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
155
282
|
} catch (err) {
|
|
156
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
157
283
|
if (err instanceof DOMException && err.name === "TimeoutError") {
|
|
158
284
|
return {
|
|
159
285
|
content: [{
|
|
@@ -170,7 +296,7 @@ var HttpMcpToolExecutor = class {
|
|
|
170
296
|
isError: true
|
|
171
297
|
};
|
|
172
298
|
}
|
|
173
|
-
return { content: [{ type: "text", text:
|
|
299
|
+
return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
|
|
174
300
|
}
|
|
175
301
|
}
|
|
176
302
|
async callConnectedMutation(path, body, timeoutMs = this.timeoutMs) {
|
|
@@ -219,9 +345,8 @@ var HttpMcpToolExecutor = class {
|
|
|
219
345
|
return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
|
|
220
346
|
}
|
|
221
347
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
222
|
-
} catch
|
|
223
|
-
|
|
224
|
-
return { content: [{ type: "text", text: msg }], isError: true };
|
|
348
|
+
} catch {
|
|
349
|
+
return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
|
|
225
350
|
}
|
|
226
351
|
}
|
|
227
352
|
async getTextArtifact(path, maxBytes, timeoutMs = this.timeoutMs) {
|
|
@@ -235,7 +360,13 @@ var HttpMcpToolExecutor = class {
|
|
|
235
360
|
});
|
|
236
361
|
if (!res.ok) {
|
|
237
362
|
const data = await res.json().catch(async () => ({ error: await res.text().catch(() => `HTTP ${res.status}`) }));
|
|
238
|
-
return {
|
|
363
|
+
return {
|
|
364
|
+
content: [{
|
|
365
|
+
type: "text",
|
|
366
|
+
text: JSON.stringify(res.status >= 500 ? httpErrorPayload(path, res, data) : sanitizeOutboundDiagnostics(data))
|
|
367
|
+
}],
|
|
368
|
+
isError: true
|
|
369
|
+
};
|
|
239
370
|
}
|
|
240
371
|
const bytes = Buffer.from(await res.arrayBuffer());
|
|
241
372
|
const sliced = bytes.subarray(0, Math.min(maxBytes, bytes.length));
|
|
@@ -252,8 +383,7 @@ var HttpMcpToolExecutor = class {
|
|
|
252
383
|
}]
|
|
253
384
|
};
|
|
254
385
|
} catch (err) {
|
|
255
|
-
|
|
256
|
-
return { content: [{ type: "text", text: msg }], isError: true };
|
|
386
|
+
return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
|
|
257
387
|
}
|
|
258
388
|
}
|
|
259
389
|
harvestPaa(input) {
|
|
@@ -635,7 +765,7 @@ render();
|
|
|
635
765
|
}
|
|
636
766
|
|
|
637
767
|
// src/version.ts
|
|
638
|
-
var PACKAGE_VERSION = "0.40.
|
|
768
|
+
var PACKAGE_VERSION = "0.40.3";
|
|
639
769
|
|
|
640
770
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
641
771
|
var import_zod = require("zod");
|
|
@@ -1282,62 +1412,6 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
|
1282
1412
|
}
|
|
1283
1413
|
}
|
|
1284
1414
|
|
|
1285
|
-
// src/errors.ts
|
|
1286
|
-
function sanitizeVendorName(message) {
|
|
1287
|
-
return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/ +/g, " ").trim();
|
|
1288
|
-
}
|
|
1289
|
-
|
|
1290
|
-
// src/api/outbound-sanitize.ts
|
|
1291
|
-
var KEY_RENAMES = {
|
|
1292
|
-
kernel: "browserRuntime",
|
|
1293
|
-
kernel_session_id: "browser_session_id",
|
|
1294
|
-
kernel_delete_started: "session_cleanup_started",
|
|
1295
|
-
kernel_delete_succeeded: "session_cleanup_succeeded",
|
|
1296
|
-
kernel_delete_error: "session_cleanup_error",
|
|
1297
|
-
kernelSessionId: "browserSessionId",
|
|
1298
|
-
kernelDeleteStarted: "sessionCleanupStarted",
|
|
1299
|
-
kernelDeleteSucceeded: "sessionCleanupSucceeded",
|
|
1300
|
-
kernelDeleteError: "sessionCleanupError",
|
|
1301
|
-
kernelProxyId: "proxyId"
|
|
1302
|
-
};
|
|
1303
|
-
var SANITIZED_VALUE_KEYS = /error|message/i;
|
|
1304
|
-
var VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
|
|
1305
|
-
"hosted_url",
|
|
1306
|
-
"hostedUrl",
|
|
1307
|
-
"live_view_url",
|
|
1308
|
-
"liveViewUrl",
|
|
1309
|
-
"cdp_ws_url",
|
|
1310
|
-
"cdpWsUrl",
|
|
1311
|
-
"browser_live_view_url"
|
|
1312
|
-
]);
|
|
1313
|
-
var VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
|
|
1314
|
-
function redactVendorUrls(value) {
|
|
1315
|
-
return value.replace(VENDOR_URL_RE, "[browser-service]");
|
|
1316
|
-
}
|
|
1317
|
-
function sanitizeOutboundDiagnostics(value, parentKey = "") {
|
|
1318
|
-
if (typeof value === "string") {
|
|
1319
|
-
let out = redactVendorUrls(value);
|
|
1320
|
-
if (SANITIZED_VALUE_KEYS.test(parentKey) && /kernel/i.test(out)) {
|
|
1321
|
-
out = sanitizeVendorName(out);
|
|
1322
|
-
}
|
|
1323
|
-
return out;
|
|
1324
|
-
}
|
|
1325
|
-
if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
|
|
1326
|
-
if (value !== null && typeof value === "object") {
|
|
1327
|
-
const out = {};
|
|
1328
|
-
for (const [key, val] of Object.entries(value)) {
|
|
1329
|
-
const renamed = KEY_RENAMES[key] ?? key;
|
|
1330
|
-
if (VENDOR_URL_KEYS.has(key)) {
|
|
1331
|
-
out[renamed] = null;
|
|
1332
|
-
continue;
|
|
1333
|
-
}
|
|
1334
|
-
out[renamed] = sanitizeOutboundDiagnostics(val, key);
|
|
1335
|
-
}
|
|
1336
|
-
return out;
|
|
1337
|
-
}
|
|
1338
|
-
return value;
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1341
1415
|
// src/mcp/output-schema-registry.ts
|
|
1342
1416
|
var ADVERTISE_OUTPUT_SCHEMAS = process.env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true";
|
|
1343
1417
|
var OUTPUT_SCHEMAS = {};
|
|
@@ -1369,8 +1443,7 @@ function errorResult(tool, value, sessionId = null, replayId = null) {
|
|
|
1369
1443
|
tool,
|
|
1370
1444
|
session_id: sessionId,
|
|
1371
1445
|
...replayId !== null ? { replay_id: replayId } : {},
|
|
1372
|
-
error: errorMessage(value)
|
|
1373
|
-
raw: value && typeof value === "object" ? value : { value }
|
|
1446
|
+
error: errorMessage(value)
|
|
1374
1447
|
}, true);
|
|
1375
1448
|
}
|
|
1376
1449
|
function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_screenshot") {
|
|
@@ -1455,8 +1528,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1455
1528
|
});
|
|
1456
1529
|
const data = await res.json().catch(() => ({}));
|
|
1457
1530
|
return { ok: res.ok, data };
|
|
1458
|
-
} catch
|
|
1459
|
-
return { ok: false, data: { error:
|
|
1531
|
+
} catch {
|
|
1532
|
+
return { ok: false, data: { error: publicErrorMessage("service_unavailable") } };
|
|
1460
1533
|
}
|
|
1461
1534
|
}
|
|
1462
1535
|
async function downloadReplay(sessionId, replayId, filename) {
|
|
@@ -1485,8 +1558,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1485
1558
|
download_url: `${baseUrl2}${path}`
|
|
1486
1559
|
}
|
|
1487
1560
|
};
|
|
1488
|
-
} catch
|
|
1489
|
-
return { ok: false, data: { error:
|
|
1561
|
+
} catch {
|
|
1562
|
+
return { ok: false, data: { error: publicErrorMessage("service_unavailable") } };
|
|
1490
1563
|
}
|
|
1491
1564
|
}
|
|
1492
1565
|
const annotations = (title, readOnly = false) => ({
|
|
@@ -3818,15 +3891,15 @@ function formatStructuredError(body, fallback) {
|
|
|
3818
3891
|
return `Insufficient credits. Balance: ${body.balance_credits} credits. This call requires ${body.required_credits} credits. Top up at ${body.topup_url}`;
|
|
3819
3892
|
}
|
|
3820
3893
|
if (body.error === "mcp_request_timeout") {
|
|
3821
|
-
return
|
|
3894
|
+
return publicErrorMessage("mcp_request_timeout");
|
|
3822
3895
|
}
|
|
3823
3896
|
if (typeof body.error_code === "string") {
|
|
3824
|
-
const
|
|
3897
|
+
const code = normalizePublicErrorCode(body.error_code);
|
|
3898
|
+
const message = publicErrorMessage(code);
|
|
3825
3899
|
const retryable = body.retryable === true ? " Retryable: yes." : "";
|
|
3826
|
-
return `${
|
|
3900
|
+
return `${code}: ${message}${retryable}${errorAttemptsSection(body)}`;
|
|
3827
3901
|
}
|
|
3828
|
-
|
|
3829
|
-
return fallback || "Tool error";
|
|
3902
|
+
return publicErrorMessage("service_unavailable");
|
|
3830
3903
|
}
|
|
3831
3904
|
function parseData(raw) {
|
|
3832
3905
|
const first = raw.content.find((b) => b.type === "text");
|
|
@@ -3837,7 +3910,7 @@ function parseData(raw) {
|
|
|
3837
3910
|
const data = parsed.result ?? parsed;
|
|
3838
3911
|
return { data };
|
|
3839
3912
|
} catch {
|
|
3840
|
-
if (raw.isError) return { error:
|
|
3913
|
+
if (raw.isError) return { error: publicErrorMessage("service_unavailable") };
|
|
3841
3914
|
return { error: "Failed to parse tool response" };
|
|
3842
3915
|
}
|
|
3843
3916
|
}
|
|
@@ -3879,10 +3952,9 @@ function debugSection(debug) {
|
|
|
3879
3952
|
const candidates = Array.isArray(locationEvidence?.candidates) ? locationEvidence.candidates.slice(0, 4).map((c) => `${c.city}, ${c.regionCode} (${c.count})`).join(", ") : "";
|
|
3880
3953
|
const lines = [
|
|
3881
3954
|
"\n## Debug",
|
|
3882
|
-
`- Proxy mode: ${request.proxyMode ?? kernel.proxyMode ?? "unknown"}
|
|
3883
|
-
`- Proxy resolution: ${proxyResolution.
|
|
3884
|
-
`- Browser
|
|
3885
|
-
`- Browser IP geo: ${[network.ip, network.city, network.region, network.country].filter(Boolean).join(" \xB7 ") || network.error || "unknown"}`,
|
|
3955
|
+
`- Proxy mode: ${request.proxyMode ?? kernel.proxyMode ?? "unknown"}`,
|
|
3956
|
+
`- Proxy resolution: ${proxyResolution.target ? `${proxyResolution.target.level ?? "city"} ${proxyResolution.target.city}, ${proxyResolution.target.state}` : proxyResolution.error ? publicErrorMessage("service_unavailable") : "unknown"}`,
|
|
3957
|
+
`- Browser IP geo: ${[network.ip, network.city, network.region, network.country].filter(Boolean).join(" \xB7 ") || (network.error ? publicErrorMessage("service_unavailable") : "unknown")}`,
|
|
3886
3958
|
`- Google URL: ${truncate(nav.requestedUrl, 240) || "unknown"}`,
|
|
3887
3959
|
`- Final URL: ${truncate(nav.finalUrl, 240) || "unknown"} \xB7 CAPTCHA: ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 redirected: ${nav.redirected === true ? "yes" : nav.redirected === false ? "no" : "unknown"}`
|
|
3888
3960
|
];
|
|
@@ -3898,7 +3970,6 @@ function errorAttemptsSection(body) {
|
|
|
3898
3970
|
const debug = attempt.debug ?? {};
|
|
3899
3971
|
const browser = debug.browser ?? {};
|
|
3900
3972
|
const kernel = browser.browserRuntime ?? browser.kernel ?? {};
|
|
3901
|
-
const proxyResolution = kernel.proxyResolution ?? {};
|
|
3902
3973
|
const network = browser.networkLocation ?? {
|
|
3903
3974
|
ip: attempt.observedIp ?? attempt.observed_ip,
|
|
3904
3975
|
city: attempt.observedCity ?? attempt.observed_city,
|
|
@@ -3906,10 +3977,7 @@ function errorAttemptsSection(body) {
|
|
|
3906
3977
|
};
|
|
3907
3978
|
const nav = browser.serpNavigation ?? {};
|
|
3908
3979
|
const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
|
|
3909
|
-
|
|
3910
|
-
const cleanupSucceeded = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
|
|
3911
|
-
const proxySource = proxyResolution.source ?? attempt.proxyResolutionSource;
|
|
3912
|
-
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"}${proxySource ? `/${proxySource}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded === true ? "yes" : cleanupSucceeded === false ? "no" : "unknown"}`;
|
|
3980
|
+
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 proxy mode ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"} \xB7 ${geo} \xB7 verification challenge ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"}`;
|
|
3913
3981
|
});
|
|
3914
3982
|
return `
|
|
3915
3983
|
|