mcp-scraper 0.33.6 → 0.34.0

package/dist/bin/paa-harvest.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-A54ELKJK.js";
+} from "../chunk-4KQVIYIH.js";
 import {
   browserServiceApiKey
-} from "../chunk-WJJTGRK4.js";
+} from "../chunk-V36LS5YV.js";
 import "../chunk-K443GQY5.js";
-import "../chunk-V73MPRU6.js";
+import "../chunk-CB5C3BPB.js";
 import "../chunk-M2S27J6Z.js";
-import "../chunk-AIK7WQD5.js";
+import "../chunk-62DQAWPF.js";
 
 // src/cli.ts
 import { Command } from "commander";

package/dist/{chunk-DUS4ENOY.js → chunk-2EDFOQD7.js}

@@ -50,7 +50,7 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(165 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(166 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
     toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
     toolRow("build", ["rank_tracker_workflow", "cron plan", "database prompt"], color),
@@ -105,4 +105,4 @@ function renderInstallTerminal(options) {
 export {
   renderInstallTerminal
 };
-//# sourceMappingURL=chunk-DUS4ENOY.js.map
+//# sourceMappingURL=chunk-2EDFOQD7.js.map

package/dist/{chunk-DUS4ENOY.js.map → chunk-2EDFOQD7.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(165 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n    toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Hosted browser sessions use direct/no-proxy egress by default.',\n    'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n    'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n    'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n    'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}
+{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(166 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n    toolRow('browser', ['browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Hosted browser sessions use direct/no-proxy egress by default.',\n    'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n    'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n    'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n    'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACxN,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}

package/dist/{chunk-C2OD4ELT.js → chunk-3HBPKR5G.js}

@@ -6,7 +6,7 @@ import {
 import {
   finishHarvestAttempt,
   startHarvestAttempt
-} from "./chunk-AIK7WQD5.js";
+} from "./chunk-62DQAWPF.js";
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -155,4 +155,4 @@ export {
   harvestProblemResponse,
   createHarvestAttemptRecorder
 };
-//# sourceMappingURL=chunk-C2OD4ELT.js.map
+//# sourceMappingURL=chunk-3HBPKR5G.js.map

package/dist/{chunk-A54ELKJK.js → chunk-4KQVIYIH.js}

@@ -5,15 +5,16 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-WJJTGRK4.js";
+} from "./chunk-V36LS5YV.js";
 import {
   postToMemoryLibrary
 } from "./chunk-K443GQY5.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
-  RawPAAItemSchema
-} from "./chunk-V73MPRU6.js";
+  RawPAAItemSchema,
+  recencyToTbs
+} from "./chunk-CB5C3BPB.js";
 import {
   CaptchaError,
   ExtractionError,
@@ -484,6 +485,7 @@ var BrowserDriver = class {
   async navigateToSERP(query, uule, gl, hl, options) {
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
     if (options?.num) params.set("num", String(options.num));
+    if (options?.tbs) params.set("tbs", options.tbs);
     if (uule) params.set("uule", uule);
     const url = "https://www.google.com/search?" + params.toString();
     const navUrl = serpArrivalUrl(url);
@@ -2050,6 +2052,7 @@ var PAAExtractor = class {
         options.hl,
         {
           ...options.serpOnly ? { num: 100 } : {},
+          ...recencyToTbs(options.recency) ? { tbs: recencyToTbs(options.recency) } : {},
           debug: options.debug
         }
       );
@@ -2069,6 +2072,8 @@ var PAAExtractor = class {
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
           const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
           await this.throwIfCaptcha(page, "Google SERP page 2");
@@ -2128,6 +2133,8 @@ var PAAExtractor = class {
         let locationEvidence2 = initialLocationEvidence;
         if ((options.pages ?? 1) >= 2) {
           const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
           await this.throwIfCaptcha(page, "Google SERP page 2");
@@ -2194,6 +2201,7 @@ var PAAExtractor = class {
       let locationEvidence = initialLocationEvidence;
       if ((options.pages ?? 1) >= 2) {
         const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+        if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
         if (uule) p2params.set("uule", uule);
         await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
         await this.throwIfCaptcha(page, "Google SERP page 2");
@@ -2357,7 +2365,92 @@ var ProgressReporter = class {
 };
 
 // src/kernel-proxy-resolver.ts
+import Kernel3 from "@onkernel/sdk";
+
+// src/backup-proxy.ts
+import { randomInt } from "crypto";
 import Kernel2 from "@onkernel/sdk";
+var BACKUP_PROXY_HOST = "pr.oxylabs.io";
+var BACKUP_PROXY_PORT = 7777;
+var BACKUP_STICKY_SESSION_MINUTES = 10;
+var BACKUP_LAST_RESORT_ATTEMPTS = 2;
+function backupProxyUsername() {
+  const value = process.env.OXYLABS_USERNAME?.trim();
+  return value || void 0;
+}
+function backupProxyPassword() {
+  const value = process.env.OXYLABS_PASSWORD?.trim();
+  return value || void 0;
+}
+function backupProxyCountry() {
+  const value = process.env.OXYLABS_COUNTRY?.trim();
+  return (value || "US").toUpperCase();
+}
+function backupProxyAvailable() {
+  return Boolean(backupProxyUsername() && backupProxyPassword());
+}
+function randomStickySessionId() {
+  let digits = "";
+  for (let i = 0; i < 10; i++) digits += String(randomInt(0, 10));
+  return digits;
+}
+function buildBackupUsername(opts = {}) {
+  const user = backupProxyUsername();
+  if (!user) return void 0;
+  const base = user.startsWith("customer-") ? user : `customer-${user}`;
+  const country = (opts.country?.trim() || backupProxyCountry()).toUpperCase();
+  const sticky = opts.sticky ? `-sessid-${randomStickySessionId()}-sesstime-${BACKUP_STICKY_SESSION_MINUTES}` : "";
+  return `${base}-cc-${country}${sticky}`;
+}
+function scrubBackupProxyText(message) {
+  return message.replace(/pr\.oxylabs\.io(:\d+)?/gi, "backup proxy").replace(/oxylabs/gi, "backup proxy");
+}
+function backupProxyName(attemptIndex) {
+  const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
+  return `mcp-backup-custom-${stamp}`;
+}
+async function createBackupProxyId(kernelApiKey, opts = {}) {
+  const username = buildBackupUsername({ sticky: opts.sticky });
+  const password = backupProxyPassword();
+  if (!username || !password) throw new Error("backup proxy pool is not configured");
+  const kernel = new Kernel2({ apiKey: kernelApiKey });
+  try {
+    const created = await kernel.proxies.create({
+      type: "custom",
+      name: backupProxyName(opts.attemptIndex),
+      protocol: "http",
+      config: {
+        host: BACKUP_PROXY_HOST,
+        port: BACKUP_PROXY_PORT,
+        username,
+        password
+      }
+    });
+    if (!created.id) throw new Error("backup proxy creation did not return a proxy id");
+    return created.id;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(scrubBackupProxyText(message));
+  }
+}
+async function createBackupProxyIdSafe(kernelApiKey, attemptIndex) {
+  if (!kernelApiKey || !backupProxyAvailable()) return void 0;
+  try {
+    return await createBackupProxyId(kernelApiKey, { attemptIndex });
+  } catch {
+    return void 0;
+  }
+}
+async function cleanupBackupProxyId(kernelApiKey, proxyId) {
+  if (!kernelApiKey || !proxyId) return;
+  try {
+    const kernel = new Kernel2({ apiKey: kernelApiKey });
+    await kernel.proxies.delete(proxyId);
+  } catch {
+  }
+}
+
+// src/kernel-proxy-resolver.ts
 var US_STATE_CODES = {
   alabama: "AL",
   alaska: "AK",
@@ -2455,7 +2548,7 @@ function proxyIdSuffix2(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
 function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
-  const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
+  const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "backup_custom" ? "custom" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
   return {
     kernelProxyId: proxyId,
     ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
@@ -2639,6 +2732,26 @@ function freshTargetCandidates(target, explicitZip, attemptIndex) {
   out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
   return out;
 }
+async function settleCreateExhaustion(options, target, createErrors) {
+  if (options.backupOnExhaustion && createErrors.length > 0 && options.kernelApiKey && backupProxyAvailable()) {
+    try {
+      const backupId = await createBackupProxyId(options.kernelApiKey, { attemptIndex: options.attemptIndex });
+      return resolution("backup_custom", options.proxyMode, backupId, target, null, true);
+    } catch (err) {
+      createErrors.push(`backup: ${scrubBackupProxyText(errorText2(err))}`);
+    }
+  }
+  return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+}
+async function resolveBackupKernelProxyId(options) {
+  if (!options.kernelApiKey || !backupProxyAvailable()) return null;
+  try {
+    const backupId = await createBackupProxyId(options.kernelApiKey, { attemptIndex: options.attemptIndex });
+    return resolution("backup_custom", options.proxyMode, backupId, null, null, true);
+  } catch {
+    return null;
+  }
+}
 async function createFreshLocationProxy(kernel, options, target) {
   const createErrors = [];
   for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
@@ -2672,11 +2785,11 @@ async function createFreshLocationProxy(kernel, options, target) {
       createErrors.push(`isp: ${errorText2(err)}`);
     }
   }
-  return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+  return settleCreateExhaustion(options, target, createErrors);
 }
 async function deleteKernelProxyId(kernelApiKey, proxyId) {
   if (!kernelApiKey || !proxyId) return;
-  const kernel = new Kernel2({ apiKey: kernelApiKey });
+  const kernel = new Kernel3({ apiKey: kernelApiKey });
   await kernel.proxies.delete(proxyId);
 }
 async function resolveKernelProxyId(options) {
@@ -2690,7 +2803,7 @@ async function resolveKernelProxyId(options) {
   if (!target || !options.kernelApiKey) {
     return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
   }
-  const kernel = new Kernel2({ apiKey: options.kernelApiKey });
+  const kernel = new Kernel3({ apiKey: options.kernelApiKey });
   try {
     const attemptIndex = options.attemptIndex ?? 0;
     if (options.fresh) {
@@ -2712,7 +2825,7 @@ async function resolveKernelProxyId(options) {
       } catch (err) {
         createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
       }
-      return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
+      return settleCreateExhaustion(options, escalatedTarget, createErrors2);
     }
     const proxies = await kernel.proxies.list();
     const zip = knownZipFor(target, options.proxyZip);
@@ -2791,7 +2904,7 @@ async function resolveKernelProxyId(options) {
     } catch (err) {
       createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
     }
-    return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+    return settleCreateExhaustion(options, target, createErrors);
   } catch (err) {
     return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
   }
@@ -2956,7 +3069,9 @@ async function harvest(rawOptions) {
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
   const requestedMaxAttempts = raw.maxAttempts;
-  const maxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
+  const baseMaxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
+  const maxAttempts = baseMaxAttempts + (backupProxyAvailable() ? BACKUP_LAST_RESORT_ATTEMPTS : 0);
+  const softDeadlineMs = typeof raw.softDeadlineMs === "number" ? raw.softDeadlineMs : null;
   const serializer = new OutputSerializer();
   let lastError = null;
   let effectiveProxyMode = proxyMode;
@@ -2966,12 +3081,16 @@ async function harvest(rawOptions) {
     const startedAtMs = Date.now();
     try {
       if (signal?.aborted) throw abortReason(signal);
-      const resolution2 = await resolveKernelProxyId({
+      const isBackupAttempt = i >= baseMaxAttempts;
+      if (isBackupAttempt && softDeadlineMs !== null && Date.now() >= softDeadlineMs) break;
+      const resolution2 = isBackupAttempt ? await resolveBackupKernelProxyId({ kernelApiKey, proxyMode: effectiveProxyMode, attemptIndex: i }) : await resolveKernelProxyId({
         ...proxyOpts,
         proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: effectiveProxyMode === "location"
+        fresh: effectiveProxyMode === "location",
+        backupOnExhaustion: true
       });
+      if (!resolution2) break;
       const mergedAttempt = {
         ...raw,
         kernelApiKey,
@@ -2980,7 +3099,7 @@ async function harvest(rawOptions) {
         proxyMode: effectiveProxyMode,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none" && !isBackupAttempt) mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",
@@ -2999,10 +3118,8 @@ async function harvest(rawOptions) {
         location: attemptOptions.location ?? null,
         max_questions: attemptOptions.maxQuestions
       }));
-      const isHeadfulEscalation = i === maxAttempts - 1;
-      const needHeadful = isHeadfulEscalation || !attemptOptions.serpOnly;
       const baseCtx = currentCostContext();
-      const attempt = needHeadful && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
+      const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome = classifyAttemptError(err);
@@ -3175,8 +3292,13 @@ export {
   serpArrivalUrl,
   buildYouTubeChannelVideosUrl,
   BrowserDriver,
+  BACKUP_LAST_RESORT_ATTEMPTS,
+  backupProxyAvailable,
+  createBackupProxyIdSafe,
+  cleanupBackupProxyId,
+  resolveBackupKernelProxyId,
   deleteKernelProxyId,
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-A54ELKJK.js.map
+//# sourceMappingURL=chunk-4KQVIYIH.js.map