mcp-scraper 0.99.1 → 0.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,30 @@ All notable changes to the public MCP Scraper client are documented here. This p
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.101.0] - 2026-10-22
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Discover and call Bug Reporting through npm, npx, CLI and desktop clients. Prepare a draft, review its redacted evidence and submit the same report with a stable receipt.
|
|
12
|
+
|
|
13
|
+
## [0.100.2] - 2026-10-21
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- Keep authenticated clients aligned with corrected hosted Maps details for unrated businesses.
|
|
18
|
+
|
|
19
|
+
## [0.100.1] - 2026-10-21
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- Keep desktop, CLI and stdio clients compatible with hosted Maps extraction improvements. Existing tool inputs and response contracts remain compatible.
|
|
24
|
+
|
|
25
|
+
## [0.100.0] - 2026-10-19
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
|
|
29
|
+
- Update desktop tool guidance for direct Reddit links and filtered discussion discovery. Hosted calls support the past-year window and expose verified post dates and captured comment coverage.
|
|
30
|
+
|
|
7
31
|
## [0.99.1] - 2026-10-17
|
|
8
32
|
|
|
9
33
|
### Changed
|
|
@@ -27,6 +51,12 @@ All notable changes to the public MCP Scraper client are documented here. This p
|
|
|
27
51
|
|
|
28
52
|
[0.98.0]: https://github.com/VilovietaSEO/mcp-scraper/releases/tag/v0.98.0
|
|
29
53
|
|
|
30
|
-
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.
|
|
54
|
+
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.100.2...HEAD
|
|
55
|
+
[0.100.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.99.1...client-v0.100.0
|
|
31
56
|
[0.99.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.98.0...client-v0.99.0
|
|
32
57
|
[0.99.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.99.0...client-v0.99.1
|
|
58
|
+
|
|
59
|
+
[0.100.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.99.1...client-v0.100.1
|
|
60
|
+
|
|
61
|
+
[0.100.2]: https://github.com/VilovietaSEO/mcp-scraper/compare/client-v0.100.1...client-v0.100.2
|
|
62
|
+
[0.101.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.100.2...v0.101.0
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{Command as Fe}from"commander";import{spawn as ye}from"node:child_process";import{mkdir as ze,writeFile as Ze}from"node:fs/promises";import{basename as Qe,join as ge}from"node:path";var v="0.
|
|
2
|
+
import{Command as Fe}from"commander";import{spawn as ye}from"node:child_process";import{mkdir as ze,writeFile as Ze}from"node:fs/promises";import{basename as Qe,join as ge}from"node:path";var v="0.101.0";import{createHash as Ce}from"node:crypto";var C="3.0.0",W="analytics_setup_site",Re="Install X-Ray",X="Install X-Ray and tell me what drives customers",P=`Call ${W} with { "startUrl": "<site URL>", "goal": "${X}" }; on every later call, reuse the returned setupId and revision.`,D=JSON.stringify({mcpServers:{"mcp-scraper":{command:"npx",args:["-y","--package","mcp-scraper@latest","mcp-scraper"],env:{MCP_SCRAPER_API_KEY:"${env:MCP_SCRAPER_API_KEY}"}}}},null,2),_e=JSON.stringify({mcpServers:{"mcp-scraper":{url:"https://mcpscraper.dev/mcp"}}},null,2),G={schema:"https://mcpscraper.dev/schemas/xray-install-contract-v1.json",protocolVersion:C,product:"Thorbit X-Ray",triggerPhrases:[Re,"Install X-Ray on this site and tell me what drives customers"],canonicalUrls:{markdown:"https://mcpscraper.dev/agent-install.md",json:"https://mcpscraper.dev/.well-known/xray-install.json",hostedMcp:"https://mcpscraper.dev/mcp"},facade:{tool:W,initialCall:{startUrl:"<site URL>",goal:X},continuationRule:"Treat setupId plus revision as the durable continuation handle. Reuse both after approval, runtime publication, verification, retry, or return."},topologies:{local_stdio:{label:"Local npm MCP process",authority:"The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration.",secretHandling:"Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it."},hosted_remote:{label:"Hosted authenticated MCP",authority:"The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path.",secretHandling:"Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat."}},hosts:[{id:"codex",label:"Codex",supportedTopologies:["local_stdio","hosted_remote"],mcpConfigMode:"toml",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:`[mcp_servers.mcp-scraper]
|
|
3
3
|
command = "npx"
|
|
4
4
|
args = ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"]
|
|
5
5
|
env_vars = ["MCP_SCRAPER_API_KEY"]`,continuation:P,rollback:"Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action."},{id:"claude-code",label:"Claude Code",supportedTopologies:["local_stdio","hosted_remote"],mcpConfigMode:"command",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:'claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" -- npx -y --package mcp-scraper@latest mcp-scraper',continuation:P,rollback:"Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action."},{id:"claude-desktop",label:"Claude Desktop",supportedTopologies:["local_stdio"],mcpConfigMode:"desktop_extension",workspaceRead:!1,workspaceWrite:!1,restartRequired:!0,exactConfig:"Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.",continuation:P,rollback:"Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action."},{id:"cursor",label:"Cursor",supportedTopologies:["local_stdio","hosted_remote"],mcpConfigMode:"json",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:D,continuation:P,rollback:"Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor."},{id:"windsurf",label:"Windsurf",supportedTopologies:["local_stdio"],mcpConfigMode:"json",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:D,continuation:P,rollback:"Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf."},{id:"cline",label:"Cline",supportedTopologies:["local_stdio"],mcpConfigMode:"json",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:D,continuation:P,rollback:"Remove mcp-scraper from Cline MCP Servers, then restart the extension host."},{id:"user-action-only",label:"User-action-only MCP host",supportedTopologies:["hosted_remote"],mcpConfigMode:"handoff_only",workspaceRead:!1,workspaceWrite:!1,restartRequired:!1,exactConfig:_e,continuation:P,rollback:"Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action."}],tagBootstrap:{requirement:"one_global_pixel_once",ongoingConfiguration:"xray_runtime_manifest",sourceAccessRequiredAfterInstall:!1,supportedInsertionPaths:["manual","tag_manager","cms_plugin","coding_host"]},workflow:["Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.","Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.","Infer the business model and customer outcome, then propose one privacy-safe measurement plan.","If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.","Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.","For SaaS, verify the measurement foundation: global tag coverage, ad-source capture, lifecycle events, currency and timezone, consent, and observed outcome events.","Optionally publish a structured post-purchase survey after approved disclosure evidence; reported influence remains separate from deterministic touches.","Compare observed attribution with customer-reported influence only with explicit weights totaling 1,000,000 micros and a named methodology version.","Keep view-through reporting unavailable until event-level provider evidence can join to an eligible X-Ray subject; aggregate impressions never qualify.","Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.","Require a separate exact-revision approval before any canary or real-visitor experiment activation.","Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.","On \u201Cimprove\u201D, propose one highest-value uncovered outcome, require approval when needed, and reverify."],safetyRules:["Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.","Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.","Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.","Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.","Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.","Never treat event-name presence as verification; require persisted definition ID and version evidence.","Never accept survey invitation tokens or exchange codes through MCP inputs or prompts.","Never treat customer-reported influence as identity, CRM mutation authority, ad-activation evidence, or a deterministic touch."],terminalStates:["complete","tag_required","input_required","approval_required","runtime_convergence_required","not_ready","error"]};function U(e){return Array.isArray(e)?`[${e.map(U).join(",")}]`:e&&typeof e=="object"?`{${Object.entries(e).sort(([r],[i])=>r.localeCompare(i)).map(([r,i])=>`${JSON.stringify(r)}:${U(i)}`).join(",")}}`:JSON.stringify(e)}function be(){return U(G)}var R=`sha256:${Ce("sha256").update(be(),"utf8").digest("hex")}`,q={...G,protocolDigest:R};function V(e){let r=q.hosts.find(i=>i.id===e);if(!r)throw new Error(`Unknown X-Ray host: ${e}`);return r}function F(){return["# Install X-Ray","",`Protocol: ${C} (${R})`,"",`Use ${W} as the single conversational setup facade. Ask for the target site URL if it is missing, then call it with the goal \u201C${X}\u201D.`,"","Continue the same setup with its setupId and revision. Confirm one global Pixel across all public rendering surfaces; present one measurement plan; publish it as a digest-bound runtime manifest after exact-revision approval; verify manifest and rule versions; then return first customer-journey answers or explicit readiness thresholds. For SaaS, verify the provider-neutral measurement foundation. Browser events never prove paid revenue. GitHub or repository access is not required after the tag is installed.","","Do not request pasted credentials, invent tracking code, silently activate measurement, call ad platforms directly, or claim customers/revenue from CTA clicks."].join(`
|
|
@@ -3,7 +3,7 @@ import{createInterface as w}from"node:readline";import{readFileSync as R}from"no
|
|
|
3
3
|
`));return _=[],s.id===e.id&&("result"in s||"error"in s)?s:(i?.(s),null)};try{for(;;){let{done:s,value:n}=await d.read();r+=a.decode(n,{stream:!s}),s&&r&&!r.endsWith(`
|
|
4
4
|
`)&&(r+=`
|
|
5
5
|
`);let l;for(;(l=r.indexOf(`
|
|
6
|
-
`))>=0;){let p=r.slice(0,l).replace(/\r$/,"");if(r=r.slice(l+1),p===""){let h=c();if(h)return h}else p.startsWith("data:")&&_.push(p.slice(5).replace(/^ /,""))}if(s){let p=c();if(p)return p;break}}}finally{await d.cancel().catch(()=>{}),d.releaseLock()}throw new Error("MCP Scraper event stream ended without a response")}async request(e,i){let t=await this.relay({jsonrpc:"2.0",id:++this.nextId,method:e,params:i});if(!t)throw new Error("MCP Scraper returned no response");if(t.error)throw new Error(typeof t.error.message=="string"?t.error.message:"MCP Scraper request failed");return t.result}callTool(e,i={}){return this.request("tools/call",{name:e,arguments:i})}};var f="0.
|
|
6
|
+
`))>=0;){let p=r.slice(0,l).replace(/\r$/,"");if(r=r.slice(l+1),p===""){let h=c();if(h)return h}else p.startsWith("data:")&&_.push(p.slice(5).replace(/^ /,""))}if(s){let p=c();if(p)return p;break}}}finally{await d.cancel().catch(()=>{}),d.releaseLock()}throw new Error("MCP Scraper event stream ended without a response")}async request(e,i){let t=await this.relay({jsonrpc:"2.0",id:++this.nextId,method:e,params:i});if(!t)throw new Error("MCP Scraper returned no response");if(t.error)throw new Error(typeof t.error.message=="string"?t.error.message:"MCP Scraper request failed");return t.result}callTool(e,i={}){return this.request("tools/call",{name:e,arguments:i})}};var f="0.101.0";function g(u){if(process.argv.includes("--help")||process.argv.includes("-h")||!process.argv.includes("--stdio")&&process.stdin.isTTY&&process.stdout.isTTY){process.stdout.write(`MCP Scraper ${f}
|
|
7
7
|
Authenticated hosted MCP client. Set MCP_SCRAPER_API_KEY and connect with --stdio.
|
|
8
8
|
`);return}let e=process.env.MCP_SCRAPER_API_KEY?.trim();if(!e){for(let a of[process.env.MCP_SCRAPER_KEY_PATH,P(v(),".mcp-scraper-key")])if(a)try{if(e=R(a,"utf8").trim(),e)break}catch{}}let i;try{i=new m({apiKey:e})}catch{process.stderr.write(`MCP_SCRAPER_API_KEY is required
|
|
9
9
|
`),process.exitCode=1;return}let t=new Map,o=a=>process.stdout.write(JSON.stringify(a)+`
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
var i={schemaVersion:1,ratePolicy:{version:"serp-page-2026-09-24",effectiveDate:"2026-09-24T00:00:00.000Z"},costEvidence:{id:"serp-page-rate-basis-2026-09",status:"owner-approved"},operationCosts:{serp:{credits:20,unit:"per page (ordinary search)",label:"SERP search",notes:"search_serp light mode returns one page of organic positions, URLs, titles, and descriptions by default. Unfiltered one-page full mode returns available same-page SERP features. Either mode requests two pages only when pages:2 is explicit. Unfiltered light searches cost 20 credits per delivered page, or 35 credits per delivered page if the backup supplies the result. One-page full searches cost 35 credits. Setting recency to week or month applies Google's date filter through a managed browser and returns organic results only. Date filters require one page and cost 35 credits. Two-page searches have no backup provider, and cannot use date filters or a saved browser identity. Unfiltered two-page full searches return organic results only for 40 credits, with rich features marked unsupported. If only page 1 is delivered, the search is marked partial and billed for one delivered page. A temporary 35-credit-per-requested-page hold is settled to the delivered rate. Location, language, device, and optional module fields do not change ordinary searches. Saved browser identity and capture_serp_snapshot searches retain their 60-credit base price; optional page snapshots add 1 credit per attempted URL."},paa:{credits:10,unit:"per question (+400 credit base)",label:"PAA harvest",notes:"Returns original-query PAA questions with answers and sources plus organic results and Google entity IDs. Optional same-page SERP modules are flag-controlled. Billed 400 credit base + 10 per question actually returned (unused estimate refunded)."},page_scrape:{credits:1,unit:"per page",label:"Page crawl / extract",notes:"Applies to both single-URL extraction and per-page site crawls."},listing_create:{credits:10,unit:"per new listing",label:"Directory or wiki listing creation",notes:"Fixed total price for a new Local Sourcebook directory listing or Transparent Commons wiki entity. Sourcebook acquisition is included."},listing_edit:{credits:2,unit:"per existing listing revision",label:"Directory or wiki listing edit",notes:"Fixed total price for refreshing an existing Local Sourcebook listing or revising an existing Transparent Commons entity."},url_map:{credits:5,unit:"per mapping operation",label:"Site URL mapping",notes:"Flat rate for the full /map-urls call regardless of URL count discovered."},yt_channel:{credits:2,unit:"per call",label:"YouTube search / channel harvest"},yt_transcription:{credits:200,unit:"base + 2/min of video",label:"YouTube transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_ad:{credits:7,unit:"per call",label:"Facebook search / ad lookup"},maps_search:{credits:5,unit:"per search",label:"Maps business search",notes:"Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."},maps_place:{credits:60,unit:"per business",label:"Maps business lookup",notes:"Maps place runs reserve the 60-credit base plus the declared review maximum before browsing. Completed runs settle to the base plus unique saved reviews; stopped attempts are refunded."},maps_review:{credits:1,unit:"per review card",label:"Maps review",notes:"One credit per unique saved review on a completed Maps place run. The declared maximum is held before browsing; unused credits are refunded."},fb_search:{credits:8,unit:"per search",label:"Facebook ad library search",notes:"Browser automation to search Facebook Ads Library by keyword."},google_ads_search:{credits:6,unit:"per search",label:"Google Ads Transparency search",notes:"Browser automation to find advertisers in Google Ads Transparency Center by domain or name."},google_ads_intel:{credits:2,unit:"per call",label:"Google Ads Transparency advertiser intel",notes:"Lists and hydrates an advertiser's creatives with image URLs and video references."},google_ads_transcribe:{credits:200,unit:"base + 2/min of video",label:"Google ad video transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_transcribe:{credits:200,unit:"base + 2/min of video",label:"Facebook video / ad transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_reels_inventory:{credits:4,unit:"per profile scan",label:"Facebook Reel URL inventory",notes:"Collects canonical Reel URLs from a Facebook profile. Logged-out scans are capped at 60 URLs; saved browser profiles can request larger inventories."},tiktok_transcribe:{credits:200,unit:"base + 2/min of video",label:"TikTok video transcription",notes:"Transcribes a public TikTok video from its playback media. Flat 200-credit base plus 2 credits per minute, reconciled after transcription."},instagram_profile:{credits:4,unit:"per profile scan",label:"Instagram profile content discovery",notes:"Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."},instagram_media:{credits:4,unit:"per post or reel",label:"Instagram media download",notes:"Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."},instagram_transcribe:{credits:200,unit:"base + 2/min of video",label:"Instagram media transcription",notes:"Whisper transcription of selected Instagram audio/video media. Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},browser_minute:{credits:120,unit:"per minute of use",label:"Interactive browser session",notes:"Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."},reddit_thread:{credits:30,unit:"per thread",label:"Reddit thread base lookup",notes:"Base lookup for the post itself. Comments billed separately per comment at reddit_comment rate. Refunded if the thread cannot be retrieved."},reddit_comment:{credits:2,unit:"per comment",label:"Reddit comment",notes:"Charged per comment actually extracted, billed down automatically if the balance runs out mid-thread."},video_analysis:{credits:6667,unit:"per 120 frames (max 480)",label:"Video breakdown (frame-by-frame + transcript)",notes:"Full video breakdown: samples frames across a video up to 60 minutes, analyzes visuals and audio, then produces a summary, pacing, topic outline, key points, hook, style, and replication guidance. Reserves 6667 Credits per requested 120-frame tier, up to 480 frames. Unused frame tiers and failed-run frame charges are refunded. Incurred analysis-model usage is billed separately, including usage before a failure; terminal status checks settle the original run."},trustpilot_reviews:{credits:5,unit:"per call",label:"Trustpilot review harvest",notes:"Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."},trustpilot_review:{credits:1,unit:"per review card",label:"Trustpilot review card",notes:"Charged per review actually extracted, billed down automatically if the balance runs out mid-page."},g2_reviews:{credits:5,unit:"per call",label:"G2 review harvest",notes:"Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."},g2_review:{credits:2,unit:"per review card",label:"G2 review card",notes:"Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."},diff_page:{credits:1,unit:"per check",label:"Page change check",notes:"Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."}},publicTiers:[{tier:"starter",label:"Starter",monthlyUsd:12,credits:8e4,concurrency:3},{tier:"growth",label:"Growth",monthlyUsd:40,credits:266667,concurrency:10},{tier:"scale",label:"Scale",monthlyUsd:100,credits:8e5,concurrency:20}],credits:{mcPerCredit:100,freeSignupCredits:0},inventory:{totalTools:347,scraperTools:233,memoryTools:114},concurrencyPack:{ratePolicyVersion:"2026-08-14.concurrency-pack-v1",monthlyUsd:5,currency:"usd",interval:"month",billingUnit:"pack",slotsPerPack:2},scaleCreditPack:{ratePolicyVersion:"2026-09-21.scale-credit-pack-v1",eligibleTier:"scale",incrementUsd:10,creditsPerIncrement:8e4,maxQuantity:100,currency:"usd",expiryMonths:3},addons:{memory:{monthlyUsd:19,quotaGb:5,freeQuotaGb:.01},connectedAccounts:{schemaVersion:2,ratePolicyVersion:"2026-08-03.connected-usage-v3",effectiveAt:"2026-08-03T00:00:00.000Z",entitlement:"paid_plan_add_on",connection:{billingMode:"flat_recurring_credits",amountCredits:15e3,interval:"month",unit:"active_connection",proration:"none"},usage:{billingMode:"credits",settlementMode:"authoritative_receipt",rounding:"aggregate_then_ceil_to_mc",functionRun:{credits:2,mc:200,unit:"per_function_run"},proxyRequest:{credits:2,mc:200,unit:"per_proxy_request"},compute:{creditsPerSecond:5,mcPerSecond:500,sourceUnit:"millisecond",unit:"per_compute_second",formula:"ceil(computeMilliseconds * mcPerSecond / 1000)"}}}}};var n={message:"Hosted search recovery handles failed requests more reliably while preserving saved results and safe retries."};var c={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(a,e,s){return s?`${c[e]}${a}${c.reset}`:a}function t(a,e,s){let o=a.padEnd(9," ");return` ${r(o,"muted",s)} ${e.join(r(" . ","muted",s))}`}function l(a){let e=a.color??!0,s=a.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",o=String.raw`
|
|
2
|
+
var i={schemaVersion:1,ratePolicy:{version:"serp-page-2026-09-24",effectiveDate:"2026-09-24T00:00:00.000Z"},costEvidence:{id:"serp-page-rate-basis-2026-09",status:"owner-approved"},operationCosts:{serp:{credits:20,unit:"per page (ordinary search)",label:"SERP search",notes:"search_serp light mode returns one page of organic positions, URLs, titles, and descriptions by default. Unfiltered one-page full mode returns available same-page SERP features. Either mode requests two pages only when pages:2 is explicit. Unfiltered light searches cost 20 credits per delivered page, or 35 credits per delivered page if the backup supplies the result. One-page full searches cost 35 credits. Setting recency to week or month applies Google's date filter through a managed browser and returns organic results only. Date filters require one page and cost 35 credits. Two-page searches have no backup provider, and cannot use date filters or a saved browser identity. Unfiltered two-page full searches return organic results only for 40 credits, with rich features marked unsupported. If only page 1 is delivered, the search is marked partial and billed for one delivered page. A temporary 35-credit-per-requested-page hold is settled to the delivered rate. Location, language, device, and optional module fields do not change ordinary searches. Saved browser identity and capture_serp_snapshot searches retain their 60-credit base price; optional page snapshots add 1 credit per attempted URL."},paa:{credits:10,unit:"per question (+400 credit base)",label:"PAA harvest",notes:"Returns original-query PAA questions with answers and sources plus organic results and Google entity IDs. Optional same-page SERP modules are flag-controlled. Billed 400 credit base + 10 per question actually returned (unused estimate refunded)."},page_scrape:{credits:1,unit:"per page",label:"Page crawl / extract",notes:"Applies to both single-URL extraction and per-page site crawls."},listing_create:{credits:10,unit:"per new listing",label:"Directory or wiki listing creation",notes:"Fixed total price for a new Local Sourcebook directory listing or Transparent Commons wiki entity. Sourcebook acquisition is included."},listing_edit:{credits:2,unit:"per existing listing revision",label:"Directory or wiki listing edit",notes:"Fixed total price for refreshing an existing Local Sourcebook listing or revising an existing Transparent Commons entity."},url_map:{credits:5,unit:"per mapping operation",label:"Site URL mapping",notes:"Flat rate for the full /map-urls call regardless of URL count discovered."},yt_channel:{credits:2,unit:"per call",label:"YouTube search / channel harvest"},yt_transcription:{credits:200,unit:"base + 2/min of video",label:"YouTube transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_ad:{credits:7,unit:"per call",label:"Facebook search / ad lookup"},maps_search:{credits:5,unit:"per search",label:"Maps business search",notes:"Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."},maps_place:{credits:60,unit:"per business",label:"Maps business lookup",notes:"Maps place runs reserve the 60-credit base plus the declared review maximum before browsing. Completed runs settle to the base plus unique saved reviews; stopped attempts are refunded."},maps_review:{credits:1,unit:"per review card",label:"Maps review",notes:"One credit per unique saved review on a completed Maps place run. The declared maximum is held before browsing; unused credits are refunded."},fb_search:{credits:8,unit:"per search",label:"Facebook ad library search",notes:"Browser automation to search Facebook Ads Library by keyword."},google_ads_search:{credits:6,unit:"per search",label:"Google Ads Transparency search",notes:"Browser automation to find advertisers in Google Ads Transparency Center by domain or name."},google_ads_intel:{credits:2,unit:"per call",label:"Google Ads Transparency advertiser intel",notes:"Lists and hydrates an advertiser's creatives with image URLs and video references."},google_ads_transcribe:{credits:200,unit:"base + 2/min of video",label:"Google ad video transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_transcribe:{credits:200,unit:"base + 2/min of video",label:"Facebook video / ad transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_reels_inventory:{credits:4,unit:"per profile scan",label:"Facebook Reel URL inventory",notes:"Collects canonical Reel URLs from a Facebook profile. Logged-out scans are capped at 60 URLs; saved browser profiles can request larger inventories."},tiktok_transcribe:{credits:200,unit:"base + 2/min of video",label:"TikTok video transcription",notes:"Transcribes a public TikTok video from its playback media. Flat 200-credit base plus 2 credits per minute, reconciled after transcription."},instagram_profile:{credits:4,unit:"per profile scan",label:"Instagram profile content discovery",notes:"Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."},instagram_media:{credits:4,unit:"per post or reel",label:"Instagram media download",notes:"Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."},instagram_transcribe:{credits:200,unit:"base + 2/min of video",label:"Instagram media transcription",notes:"Whisper transcription of selected Instagram audio/video media. Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},browser_minute:{credits:120,unit:"per minute of use",label:"Interactive browser session",notes:"Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."},reddit_thread:{credits:30,unit:"per thread",label:"Reddit thread base lookup",notes:"Base lookup for the post itself. Comments billed separately per comment at reddit_comment rate. Refunded if the thread cannot be retrieved."},reddit_comment:{credits:2,unit:"per comment",label:"Reddit comment",notes:"Charged per comment actually extracted, billed down automatically if the balance runs out mid-thread."},video_analysis:{credits:6667,unit:"per 120 frames (max 480)",label:"Video breakdown (frame-by-frame + transcript)",notes:"Full video breakdown: samples frames across a video up to 60 minutes, analyzes visuals and audio, then produces a summary, pacing, topic outline, key points, hook, style, and replication guidance. Reserves 6667 Credits per requested 120-frame tier, up to 480 frames. Unused frame tiers and failed-run frame charges are refunded. Incurred analysis-model usage is billed separately, including usage before a failure; terminal status checks settle the original run."},trustpilot_reviews:{credits:5,unit:"per call",label:"Trustpilot review harvest",notes:"Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."},trustpilot_review:{credits:1,unit:"per review card",label:"Trustpilot review card",notes:"Charged per review actually extracted, billed down automatically if the balance runs out mid-page."},g2_reviews:{credits:5,unit:"per call",label:"G2 review harvest",notes:"Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."},g2_review:{credits:2,unit:"per review card",label:"G2 review card",notes:"Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."},diff_page:{credits:1,unit:"per check",label:"Page change check",notes:"Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."}},publicTiers:[{tier:"starter",label:"Starter",monthlyUsd:12,credits:8e4,concurrency:3},{tier:"growth",label:"Growth",monthlyUsd:40,credits:266667,concurrency:10},{tier:"scale",label:"Scale",monthlyUsd:100,credits:8e5,concurrency:20}],credits:{mcPerCredit:100,freeSignupCredits:0},inventory:{totalTools:348,scraperTools:234,memoryTools:114},concurrencyPack:{ratePolicyVersion:"2026-08-14.concurrency-pack-v1",monthlyUsd:5,currency:"usd",interval:"month",billingUnit:"pack",slotsPerPack:2},scaleCreditPack:{ratePolicyVersion:"2026-09-21.scale-credit-pack-v1",eligibleTier:"scale",incrementUsd:10,creditsPerIncrement:8e4,maxQuantity:100,currency:"usd",expiryMonths:3},addons:{memory:{monthlyUsd:19,quotaGb:5,freeQuotaGb:.01},connectedAccounts:{schemaVersion:2,ratePolicyVersion:"2026-08-03.connected-usage-v3",effectiveAt:"2026-08-03T00:00:00.000Z",entitlement:"paid_plan_add_on",connection:{billingMode:"flat_recurring_credits",amountCredits:15e3,interval:"month",unit:"active_connection",proration:"none"},usage:{billingMode:"credits",settlementMode:"authoritative_receipt",rounding:"aggregate_then_ceil_to_mc",functionRun:{credits:2,mc:200,unit:"per_function_run"},proxyRequest:{credits:2,mc:200,unit:"per_proxy_request"},compute:{creditsPerSecond:5,mcPerSecond:500,sourceUnit:"millisecond",unit:"per_compute_second",formula:"ceil(computeMilliseconds * mcPerSecond / 1000)"}}}}};var n={message:"Bug Reporting captures a reviewed record of failed calls, errors and reproduction steps, then sends it to private support at your request. Includes a receipt and automatic credential redaction."};var c={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(a,e,o){return o?`${c[e]}${a}${c.reset}`:a}function t(a,e,o){let s=a.padEnd(9," ");return` ${r(s,"muted",o)} ${e.join(r(" . ","muted",o))}`}function l(a){let e=a.color??!0,o=a.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",s=String.raw`
|
|
3
3
|
__ __ ____ ____
|
|
4
4
|
| \/ |/ ___| _ \
|
|
5
5
|
| |\/| | | | |_) |
|
|
@@ -11,8 +11,8 @@ var i={schemaVersion:1,ratePolicy:{version:"serp-page-2026-09-24",effectiveDate:
|
|
|
11
11
|
\___ \| | | |_) | / _ \ | |_) | _| | |_) |
|
|
12
12
|
___) | |___| _ < / ___ \| __/| |___| _ <
|
|
13
13
|
|____/ \____|_| \_\/_/ \_\_| |_____|_| \_\
|
|
14
|
-
`,p=[`MCP_SCRAPER_API_KEY=${
|
|
15
|
-
`),u=["[mcp_servers.mcp-scraper]",'command = "npx"','args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',`env = { MCP_SCRAPER_API_KEY = "${
|
|
16
|
-
`);return[r(`mcp-scraper v${a.version}`,"bold",e),r("> mcp-scraper-install","muted",e),r(
|
|
17
|
-
`)}var d="0.
|
|
14
|
+
`,p=[`MCP_SCRAPER_API_KEY=${o} npx -y -p mcp-scraper@latest \\`," mcp-scraper-cli agent install claude --apply"].join(`
|
|
15
|
+
`),u=["[mcp_servers.mcp-scraper]",'command = "npx"','args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',`env = { MCP_SCRAPER_API_KEY = "${o}" }`].join(`
|
|
16
|
+
`);return[r(`mcp-scraper v${a.version}`,"bold",e),r("> mcp-scraper-install","muted",e),r(s,"amber",e),`${r("MCP Scraper Agent","cyan",e)} . v${a.version} . mcpscraper.dev`,"1/1 install surfaces ready",r(`Newest in v${a.version}: ${n.message}`,"lime",e),"",`${r("Tools","cyan",e)} ${r(`(${i.inventory.totalTools} MCP tools)`,"muted",e)}`,t("search",["harvest_paa","search_serp","maps_search","maps_place_intel"],e),t("extract",["extract_url","map_site_urls","extract_site","audit_site","directory_workflow"],e),t("build",["create_editorial_reading_room","rank_tracker_workflow","portable HTML"],e),t("media",["youtube_harvest","youtube_transcribe","facebook_reels_inventory","facebook_ad_search","facebook_page_intel","facebook_ad_transcribe","facebook_video_transcribe","tiktok_video_transcribe","instagram_profile_content","instagram_media_download","reddit_thread"],e),t("browser",["serp_identity_create","serp_identity_list","browser_open","browser_profile_connect","browser_profile_list","browser_close","browser_screenshot","browser_read","browser_locate","browser_replay_mark","browser_replay_annotate"],e),t("connect",["list_service_connections","describe_service_connection_tool","import_service_connection_to_memory","export_connected_service_data","renew_connected_data_download","read_service_connection","call_service_connection_action"],e),t("commons",["commons_search_entities","commons_get_entity_linkset","commons_prepare_entity","commons_submit_entity","commons_prepare_publication","commons_claim_publication","commons_publish_editorial","commons_get_publication"],e),t("account",["credits_info","reports","MCP resources"],e),t("memory",["memory-put","memory-get","memory-search","list-vaults","record-fact","list_artifact_templates"],e),`${r("Workflows","cyan",e)} ${r("(MCP + CLI + API)","muted",e)}`,t("route",["workflow_list","workflow_suggest","workflow_run","workflow_step","workflow_status","workflow_artifact_read"],e),t("seo",["directory","agent-packet","competitive audit","map/serp comparison","PAA/AIO briefs"],e),"",r("Usage tips:","amber",e),"Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.","Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.","Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install","Hosted browser sessions use direct/no-proxy egress by default.","Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.","Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.","Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.","Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.","Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.","For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.","If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.","",`${r("Ready.","lime",e)} Install the combined MCP server with one command:`,"",r("Setup doctor","amber",e),"npx -y -p mcp-scraper@latest mcp-scraper-cli doctor","",r("Hosted profile setup","amber",e),'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',"Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.","",r("Claude Code one-command setup","amber",e),p,"Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list","",r("Codex config","amber",e),u,"",r("Claude Desktop Extension","amber",e),"Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb","",r("Safety note:","muted",e),"mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.","Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",""].join(`
|
|
17
|
+
`)}var d="0.101.0";var m=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY,b=process.argv.includes("--help")||process.argv.includes("-h");b&&(process.stdout.write(["Usage: mcp-scraper-install [--no-color]","","Prints the branded MCP Scraper terminal install card and copyable install commands.","mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.",""].join(`
|
|
18
18
|
`)),process.exit(0));process.stdout.write(l({version:d,color:!m,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()}));
|
|
@@ -3,7 +3,7 @@ import{createInterface as P}from"node:readline";import{readFileSync as w}from"no
|
|
|
3
3
|
`));return u=[],s.id===e.id&&("result"in s||"error"in s)?s:(i?.(s),null)};try{for(;;){let{done:s,value:o}=await d.read();r+=c.decode(o,{stream:!s}),s&&r&&!r.endsWith(`
|
|
4
4
|
`)&&(r+=`
|
|
5
5
|
`);let l;for(;(l=r.indexOf(`
|
|
6
|
-
`))>=0;){let p=r.slice(0,l).replace(/\r$/,"");if(r=r.slice(l+1),p===""){let m=a();if(m)return m}else p.startsWith("data:")&&u.push(p.slice(5).replace(/^ /,""))}if(s){let p=a();if(p)return p;break}}}finally{await d.cancel().catch(()=>{}),d.releaseLock()}throw new Error("MCP Scraper event stream ended without a response")}async request(e,i){let t=await this.relay({jsonrpc:"2.0",id:++this.nextId,method:e,params:i});if(!t)throw new Error("MCP Scraper returned no response");if(t.error)throw new Error(typeof t.error.message=="string"?t.error.message:"MCP Scraper request failed");return t.result}callTool(e,i={}){return this.request("tools/call",{name:e,arguments:i})}};var g="0.
|
|
6
|
+
`))>=0;){let p=r.slice(0,l).replace(/\r$/,"");if(r=r.slice(l+1),p===""){let m=a();if(m)return m}else p.startsWith("data:")&&u.push(p.slice(5).replace(/^ /,""))}if(s){let p=a();if(p)return p;break}}}finally{await d.cancel().catch(()=>{}),d.releaseLock()}throw new Error("MCP Scraper event stream ended without a response")}async request(e,i){let t=await this.relay({jsonrpc:"2.0",id:++this.nextId,method:e,params:i});if(!t)throw new Error("MCP Scraper returned no response");if(t.error)throw new Error(typeof t.error.message=="string"?t.error.message:"MCP Scraper request failed");return t.result}callTool(e,i={}){return this.request("tools/call",{name:e,arguments:i})}};var g="0.101.0";function y(h){if(process.argv.includes("--help")||process.argv.includes("-h")||!process.argv.includes("--stdio")&&process.stdin.isTTY&&process.stdout.isTTY){process.stdout.write(`MCP Scraper ${g}
|
|
7
7
|
Authenticated hosted MCP client. Set MCP_SCRAPER_API_KEY and connect with --stdio.
|
|
8
8
|
`);return}let e=process.env.MCP_SCRAPER_API_KEY?.trim();if(!e){for(let c of[process.env.MCP_SCRAPER_KEY_PATH,C(R(),".mcp-scraper-key")])if(c)try{if(e=w(c,"utf8").trim(),e)break}catch{}}let i;try{i=new f({apiKey:e})}catch{process.stderr.write(`MCP_SCRAPER_API_KEY is required
|
|
9
9
|
`),process.exitCode=1;return}let t=new Map,n=c=>process.stdout.write(JSON.stringify(c)+`
|