mcp-scraper 0.89.9 → 0.90.0

package/CHANGELOG.md

@@ -4,6 +4,24 @@ All notable changes to MCP Scraper are documented here. The format is based on [
 
 ## [Unreleased]
 
+## [0.90.0] - 2026-09-21
+
+### Added
+
+- Scale customers can buy expiring Credits in whole $10 increments at the Scale plan's existing per-dollar rate, with payment and Checkout-return fulfillment sharing one replay-safe grant.
+
+### Fixed
+
+- Make existing-subscriber upgrades invoice and attempt the proration immediately, grant the new plan Credits synchronously only after the invoice is paid, and block further plan changes while an earlier invoice remains outstanding.
+
+## [0.89.10] - 2026-09-21
+
+### Fixed
+
+- Bound browser-provider connection setup by the logical call deadline so a stalled CDP connection cannot survive a ten-second SERP request.
+- Bound browser-close acknowledgement to 1.5 seconds after issuing close, preventing cleanup from holding a completed or cancelled function open until Vercel's 800-second ceiling.
+- Reserve explicit time inside the ten-second SERP limit for browser cleanup, billing settlement, and the structured response.
+
 ## [0.89.9] - 2026-09-21
 
 ### Changed

package/README.md

@@ -175,7 +175,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.89.9`, SHA-256 `00b8ca9ffabe4f7525b0083bf1beaa29bb6bf69815ca1ab1f0674154246b1016`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.90.0`, SHA-256 `7e5bfe35dddabebb5c2566bd304dd93dee25ed28b2847c14f76597ee2a8b92b4`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.js

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
 import{readFileSync as a}from"fs";function c(){try{for(let t of a(".env","utf8").split(`
-`)){let o=t.indexOf("=");if(o<1||t.trimStart().startsWith("#"))continue;let e=t.slice(0,o).trim();process.env[e]||(process.env[e]=t.slice(o+1).trim())}}catch{}}c();async function p(){let[{serve:t},{app:o,personalAssistantProductionStartup:e},{startWorker:i},{migrate:n}]=await Promise.all([import("@hono/node-server"),import("../server-EYSV2RRD.js"),import("../worker-577GJULQ.js"),import("../db-UKQ74OVT.js")]),s=parseInt(process.env.PORT??"3001");try{if(await e,await n(),process.env.ANALYTICS_DATABASE_URL){let{migrateAnalytics:r}=await import("../analytics-repository-UX6VU22H.js");await r()}i(),t({fetch:o.fetch,port:s},r=>{console.log(`[server] http://localhost:${r.port}`),console.log(`[server] admin auth: ${process.env.ADMIN_KEY?"configured":"not configured"}`)})}catch(r){console.error("[startup] server preflight failed",r instanceof Error?r.name:"unknown_error"),process.exit(1)}}p();
+`)){let o=t.indexOf("=");if(o<1||t.trimStart().startsWith("#"))continue;let e=t.slice(0,o).trim();process.env[e]||(process.env[e]=t.slice(o+1).trim())}}catch{}}c();async function p(){let[{serve:t},{app:o,personalAssistantProductionStartup:e},{startWorker:i},{migrate:n}]=await Promise.all([import("@hono/node-server"),import("../server-TI7YYGK5.js"),import("../worker-IUJVVXIG.js"),import("../db-UKQ74OVT.js")]),s=parseInt(process.env.PORT??"3001");try{if(await e,await n(),process.env.ANALYTICS_DATABASE_URL){let{migrateAnalytics:r}=await import("../analytics-repository-UX6VU22H.js");await r()}i(),t({fetch:o.fetch,port:s},r=>{console.log(`[server] http://localhost:${r.port}`),console.log(`[server] admin auth: ${process.env.ADMIN_KEY?"configured":"not configured"}`)})}catch(r){console.error("[startup] server preflight failed",r instanceof Error?r.name:"unknown_error"),process.exit(1)}}p();

package/dist/bin/mcp-scraper-cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-JHMI6HEO.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-2TZBO52D.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-2PXP7TMY.js";import{a as P}from"../chunk-PVTKGAZL.js";import{Command as he}from"commander";import{spawn as ne}from"child_process";import{mkdir as ke,writeFile as Pe}from"fs/promises";import{basename as Ce,join as Z}from"path";function se(e){return e.apiKey?.trim()||"sk_live_your_key"}function ce(e){return e.packageSpec?.trim()||"mcp-scraper@latest"}function A(e={}){return["-y","--package",ce(e),"mcp-scraper"]}function pe(e){let n={MCP_SCRAPER_API_KEY:se(e)},c=e.browserProfileName?.trim();return c&&(n.BROWSER_AGENT_PROFILE_NAME=c),e.browserProfileSaveChanges===!0&&(n.BROWSER_AGENT_PROFILE_SAVE_CHANGES="true"),n}function q(){return["mcp","remove","mcp-scraper","-s","user"]}function J(){return["mcp","get","mcp-scraper"]}function B(e){let n=e.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];if(!n)return null;let c=e.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1]??"",i=c.length?c.split(/\s+/):[],p={},u=e.split(/^\s*Environment:\s*$/m)[1];if(u)for(let a of u.split(`
+import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-JHMI6HEO.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-2TZBO52D.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-2PXP7TMY.js";import{a as P}from"../chunk-45Z77CM7.js";import{Command as he}from"commander";import{spawn as ne}from"child_process";import{mkdir as ke,writeFile as Pe}from"fs/promises";import{basename as Ce,join as Z}from"path";function se(e){return e.apiKey?.trim()||"sk_live_your_key"}function ce(e){return e.packageSpec?.trim()||"mcp-scraper@latest"}function A(e={}){return["-y","--package",ce(e),"mcp-scraper"]}function pe(e){let n={MCP_SCRAPER_API_KEY:se(e)},c=e.browserProfileName?.trim();return c&&(n.BROWSER_AGENT_PROFILE_NAME=c),e.browserProfileSaveChanges===!0&&(n.BROWSER_AGENT_PROFILE_SAVE_CHANGES="true"),n}function q(){return["mcp","remove","mcp-scraper","-s","user"]}function J(){return["mcp","get","mcp-scraper"]}function B(e){let n=e.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];if(!n)return null;let c=e.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1]??"",i=c.length?c.split(/\s+/):[],p={},u=e.split(/^\s*Environment:\s*$/m)[1];if(u)for(let a of u.split(`
 `)){let l=a.match(/^\s{2,}([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);if(!l){if(a.trim().length&&!/^\s{2,}/.test(a))break;continue}p[l[1]]=l[2]}return{command:n,args:i,env:p}}function j(e){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(e.env))n.push("--env",`${c}=${i}`);return n.push("--",e.command,...e.args),n}function G(e={}){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(pe(e)))n.push("--env",`${c}=${i}`);return n.push("--","npx",...A(e)),n}function O(e){if(e==="claude-code")return"claude";if(e==="claude"||D.hosts.some(n=>n.id===e))return e;throw new Error('Unknown host "'+e+'". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only')}function ue(e){return K(e==="claude"?"claude-code":e)}function W(e,n={}){let c=O(e),i=ue(c),p="Restart the MCP client so it starts a fresh npx process.",u='MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply',a=`X-Ray install protocol: ${v} (${b})`;return c==="codex"?["# Codex MCP config",a,i.exactConfig,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
 `):c==="claude"?["# Claude Code command",a,i.exactConfig,"","# One-command Claude Code setup",u,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
 `):c==="claude-desktop"?["# Claude Desktop config",a,i.exactConfig,"","Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,p].join(`

package/dist/bin/mcp-scraper-core.js

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-import{a as e}from"../chunk-2WWKTVCO.js";import"../chunk-XATD7HDY.js";import"../chunk-W2BVJ7S2.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-D6BTAT5Y.js";import"../chunk-7AC66V3T.js";import"../chunk-E4ME7J6G.js";import"../chunk-4FROKQJN.js";import"../chunk-DKQZNSYM.js";import"../chunk-S3N55WHK.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-2PXP7TMY.js";import"../chunk-PVTKGAZL.js";import"../chunk-7BEWBRMT.js";var _=["harvest_paa","search_serp","extract_url","diff_page","map_site_urls","map_wayback_snapshots","extract_site","analyze_site_similarity","audit_site","check_site_export","site_export_read","site_export_image","archive_read","youtube_harvest","youtube_transcribe","facebook_page_intel","facebook_ad_search","reddit_thread","reddit_trending","video_frame_analysis","video_frame_analysis_status","facebook_ad_transcribe","google_ads_search","google_ads_page_intel","google_ads_transcribe","facebook_video_transcribe","instagram_profile_content","instagram_media_download","maps_place_intel","maps_search","trustpilot_reviews","g2_reviews","capture_serp_snapshot","capture_serp_page_snapshots"];e({toolsets:new Set(["paa","serp"]),allowedToolNames:_});
+import{a as e}from"../chunk-SWKUC7YD.js";import"../chunk-RAG2WBBL.js";import"../chunk-W2BVJ7S2.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-D6BTAT5Y.js";import"../chunk-7AC66V3T.js";import"../chunk-YEFXVTMJ.js";import"../chunk-4FROKQJN.js";import"../chunk-DKQZNSYM.js";import"../chunk-XGZJGKEG.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-2PXP7TMY.js";import"../chunk-45Z77CM7.js";import"../chunk-7BEWBRMT.js";var _=["harvest_paa","search_serp","extract_url","diff_page","map_site_urls","map_wayback_snapshots","extract_site","analyze_site_similarity","audit_site","check_site_export","site_export_read","site_export_image","archive_read","youtube_harvest","youtube_transcribe","facebook_page_intel","facebook_ad_search","reddit_thread","reddit_trending","video_frame_analysis","video_frame_analysis_status","facebook_ad_transcribe","google_ads_search","google_ads_page_intel","google_ads_transcribe","facebook_video_transcribe","instagram_profile_content","instagram_media_download","maps_place_intel","maps_search","trustpilot_reviews","g2_reviews","capture_serp_snapshot","capture_serp_page_snapshots"];e({toolsets:new Set(["paa","serp"]),allowedToolNames:_});

package/dist/bin/mcp-scraper-install.js

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
-import{a as s}from"../chunk-S3N55WHK.js";import{a as e}from"../chunk-PVTKGAZL.js";var r=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY,n=process.argv.includes("--help")||process.argv.includes("-h");n&&(process.stdout.write(["Usage: mcp-scraper-install [--no-color]","","Prints the branded MCP Scraper terminal install card and copyable install commands.","mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.",""].join(`
+import{a as s}from"../chunk-XGZJGKEG.js";import{a as e}from"../chunk-45Z77CM7.js";var r=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY,n=process.argv.includes("--help")||process.argv.includes("-h");n&&(process.stdout.write(["Usage: mcp-scraper-install [--no-color]","","Prints the branded MCP Scraper terminal install card and copyable install commands.","mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.",""].join(`
 `)),process.exit(0));process.stdout.write(s({version:e,color:!r,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()}));

package/dist/bin/mcp-stdio-server.js

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-import{a as r}from"../chunk-2WWKTVCO.js";import"../chunk-XATD7HDY.js";import"../chunk-W2BVJ7S2.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-D6BTAT5Y.js";import"../chunk-7AC66V3T.js";import"../chunk-E4ME7J6G.js";import"../chunk-4FROKQJN.js";import"../chunk-DKQZNSYM.js";import"../chunk-S3N55WHK.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-2PXP7TMY.js";import"../chunk-PVTKGAZL.js";import"../chunk-7BEWBRMT.js";r();
+import{a as r}from"../chunk-SWKUC7YD.js";import"../chunk-RAG2WBBL.js";import"../chunk-W2BVJ7S2.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-D6BTAT5Y.js";import"../chunk-7AC66V3T.js";import"../chunk-YEFXVTMJ.js";import"../chunk-4FROKQJN.js";import"../chunk-DKQZNSYM.js";import"../chunk-XGZJGKEG.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-2PXP7TMY.js";import"../chunk-45Z77CM7.js";import"../chunk-7BEWBRMT.js";r();

package/dist/bin/paa-harvest.js

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-import{u as t}from"../chunk-F4LIB772.js";import"../chunk-M5QHXNFZ.js";import{a as r}from"../chunk-4FROKQJN.js";import"../chunk-DKQZNSYM.js";import"../chunk-2TZBO52D.js";import"../chunk-2PXP7TMY.js";import"../chunk-7BEWBRMT.js";import{Command as s,Option as a}from"commander";var i=new s;i.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>","Seed query").option("-l, --location <location>",'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>","Google country code","us").option("--hl <hl>","Google language code","en").option("-d, --depth <depth>","BFS depth (1-30)","3").option("-m, --max-questions <n>","Max questions to harvest","100").option("-o, --output <dir>","Output directory","./paa-output").option("-f, --format <format>","Output format: json, csv, or both","both").option("--headless","Run browser in headless mode",!1).option("--profile <dir>","Persistent browser profile directory").option("--proxy <url>","Proxy server URL").option("--browser-api-key <key>","Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").addOption(new a("--\u006b\u0065\u0072\u006e\u0065\u006c-api-key <key>").hideHelp()).action(async e=>{try{let o=await t({query:e.query,location:e.location,gl:e.gl,hl:e.hl,depth:parseInt(e.depth,10),maxQuestions:parseInt(e.maxQuestions,10),outputDir:e.output,format:e.format,headless:e.headless,profileDir:e.profile,proxy:e.proxy,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.browserApiKey??e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey??r()});console.log(JSON.stringify({totalQuestions:o.totalQuestions,outputDir:o.stats.seed}))}catch(o){console.error(o instanceof Error?o.message:String(o)),process.exit(1)}});async function n(){await i.parseAsync()}n();
+import{u as t}from"../chunk-M7EXIKT7.js";import"../chunk-M5QHXNFZ.js";import{a as r}from"../chunk-4FROKQJN.js";import"../chunk-DKQZNSYM.js";import"../chunk-2TZBO52D.js";import"../chunk-2PXP7TMY.js";import"../chunk-7BEWBRMT.js";import{Command as s,Option as a}from"commander";var i=new s;i.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>","Seed query").option("-l, --location <location>",'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>","Google country code","us").option("--hl <hl>","Google language code","en").option("-d, --depth <depth>","BFS depth (1-30)","3").option("-m, --max-questions <n>","Max questions to harvest","100").option("-o, --output <dir>","Output directory","./paa-output").option("-f, --format <format>","Output format: json, csv, or both","both").option("--headless","Run browser in headless mode",!1).option("--profile <dir>","Persistent browser profile directory").option("--proxy <url>","Proxy server URL").option("--browser-api-key <key>","Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").addOption(new a("--\u006b\u0065\u0072\u006e\u0065\u006c-api-key <key>").hideHelp()).action(async e=>{try{let o=await t({query:e.query,location:e.location,gl:e.gl,hl:e.hl,depth:parseInt(e.depth,10),maxQuestions:parseInt(e.maxQuestions,10),outputDir:e.output,format:e.format,headless:e.headless,profileDir:e.profile,proxy:e.proxy,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.browserApiKey??e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey??r()});console.log(JSON.stringify({totalQuestions:o.totalQuestions,outputDir:o.stats.seed}))}catch(o){console.error(o instanceof Error?o.message:String(o)),process.exit(1)}});async function n(){await i.parseAsync()}n();

package/dist/{chunk-Q4A2KIPZ.js → chunk-3R5QLQT3.js}

@@ -1,4 +1,4 @@
-import{k as K}from"./chunk-5B42QVDC.js";import{a as X}from"./chunk-2GRE4BAO.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-7BEWBRMT.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
+import{k as K}from"./chunk-WTOKMWDS.js";import{a as X}from"./chunk-2GRE4BAO.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-7BEWBRMT.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
       CREATE TABLE IF NOT EXISTS gmail_selections (
         selection_id TEXT PRIMARY KEY,
         owner_identity TEXT NOT NULL,

package/dist/chunk-45Z77CM7.js

@@ -0,0 +1 @@
+var o="0.90.0";export{o as a};