mcp-scraper 0.93.3 → 0.94.0

package/dist/{chunk-BS4V4CVX.js → chunk-2GEVQBI6.js}

@@ -1,4 +1,4 @@
-import{A as f,Ga as T,Ia as h,Ja as w,Ka as _,La as y,Ma as C,ga as g,ia as S,va as R,wa as E,ya as v,z as u}from"./chunk-4FGLXXZ2.js";import{a as P}from"./chunk-DFJT2YX6.js";import{a as m}from"./chunk-SWCAKYBP.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
+import{A as f,Ga as T,Ia as h,Ja as w,Ka as _,La as y,Ma as C,ga as g,ia as S,va as R,wa as E,ya as v,z as u}from"./chunk-ASP7YHUY.js";import{a as P}from"./chunk-C5MWTMRD.js";import{a as m}from"./chunk-4SNU2TWB.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
 `).filter(o=>o.startsWith("data:")).map(o=>o.slice(5).trim()).filter(Boolean);if(!s.length)return JSON.parse(r);for(let o of s){let e=JSON.parse(o);if(e.result||e.error)return e}throw new Error("hosted MCP returned no JSON-RPC result")}function a(r){return{content:[{type:"text",text:JSON.stringify({ok:!1,error:r})}],isError:!0}}var c=class{baseUrl;apiKey;constructor(s,o){this.baseUrl=s.replace(/\/$/,""),this.apiKey=o}async callMemoryTool(s,o){let e=x.get(s);if(!e)return a(`unknown memory tool: ${s}`);try{let n=await fetch(`${this.baseUrl}/mcp`,{method:"POST",headers:{"content-type":"application/json",accept:"application/json, text/event-stream","x-api-key":this.apiKey},body:JSON.stringify({jsonrpc:"2.0",id:`memory:${e}`,method:"tools/call",params:{name:e,arguments:o}})}),l=await n.text();if(!n.ok)return a(`hosted memory ${e} failed (HTTP ${n.status})`);let t=O(l);return t.error?a(t.error.message??`hosted memory ${e} failed`):t.result??a(`hosted memory ${e} returned no result`)}catch(n){return a(n instanceof Error?n.message:`hosted memory ${e} call failed`)}}};function M(r,s){if(s.length===0)throw new Error("Restricted MCP tool allowlist must contain at least one tool name");let o=new Set;for(let t of s){if(typeof t!="string"||t.trim()!==t||t.length===0)throw new Error("Restricted MCP tool allowlist contains a malformed tool name");if(o.has(t))throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${t}`);o.add(t)}let e=new Set,n=r,l=n.registerTool.bind(r);return n.registerTool=(t,i,p)=>{if(o.has(t))return e.add(t),l(t,i,p)},{assertComplete(){let t=[...o].filter(i=>!e.has(i));if(t.length>0)throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${t.join(", ")}`)},registeredToolNames(){return[...e]}}}var L=new Set(["paa","serp","browser-agent","scheduled-results","memory"]);function $(){let s=[process.env.MCP_SCRAPER_KEY_PATH?.trim(),A(I(),".mcp-scraper-key")].filter(Boolean);for(let o of s)try{let e=N(o,"utf8").trim();if(e)return e}catch{}}function Y(r,s={}){let o=s.toolsets??L,e=process.env.MCP_SCRAPER_BASE_URL?.trim()||process.env.MCP_BASE_URL?.trim()||"https://mcpscraper.dev",n=g(),l=S({deploymentProfile:n,transportProfile:"stdio",baseUrl:e,explicitlyEnabled:process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK==="1"}),t=process.env.BROWSER_AGENT_CONSOLE_URL?.trim()||e,i=new k({name:"mcp-scraper",version:m},{instructions:u,cacheHints:{"server/discover":{ttlMs:3e5,cacheScope:"private"},"tools/list":{ttlMs:3e5,cacheScope:"private"},"prompts/list":{ttlMs:3e5,cacheScope:"private"},"resources/list":{ttlMs:3e5,cacheScope:"private"},"resources/templates/list":{ttlMs:3e5,cacheScope:"private"},"resources/read":{ttlMs:6e4,cacheScope:"private"}}});f(i);let p=s.allowedToolNames?M(i,s.allowedToolNames):void 0,d=s.httpExecutor??new T(e,r,{localNetworkAccess:l});return o.has("paa")&&v(i,d,{ownerId:R(r),deploymentProfile:n,transportProfile:"stdio",baseUrl:e,localNetworkAccess:l,taskHandleSecret:r}),o.has("serp")&&E(i,d,{exposeDevelopmentDiagnostics:n==="development"||n==="test"}),o.has("browser-agent")&&h(i,{baseUrl:e,apiKey:r,consoleBaseUrl:t}),o.has("scheduled-results")&&C(i,new y(e,r)),o.has("memory")&&_(i,new c(e,r)),p?.assertComplete(),i}function se(r={}){let s=process.argv.includes("--stdio")||process.env.MCP_SCRAPER_FORCE_STDIO==="1",o=!!(process.stdin.isTTY&&process.stdout.isTTY),e=process.argv.includes("--help")||process.argv.includes("-h");if(!s&&(o||e)){let l=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY;process.stdout.write(P({version:m,color:!l,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()})),process.exit(0)}let n=(process.env.MCP_SCRAPER_API_KEY??process.env.MCP_SCRAPER_KEY??process.env.MCP_API_KEY??$())?.trim();n||(process.stderr.write(`MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required
 `),process.exit(1)),b(()=>Y(n,r),{legacy:"serve",onerror(l){process.stderr.write(`${l.message}
 `)}})}export{se as a};

package/dist/chunk-4SNU2TWB.js

@@ -0,0 +1 @@
+var o="0.94.0";export{o as a};

package/dist/{chunk-LI7WHOII.js → chunk-56C243PX.js}

@@ -1,4 +1,4 @@
-import{a as Q,d as Se}from"./chunk-QTDLZTQ7.js";import{createHash as pe,createHmac as Yt,randomBytes as P,randomUUID as y}from"crypto";import{Pool as Wt}from"pg";var ct=[[/perplexity/i,"llm","Perplexity"],[/chatgpt|chat\.openai|openai/i,"llm","ChatGPT"],[/claude|anthropic/i,"llm","Claude"],[/\bgrok\b|xai/i,"llm","Grok"],[/ai[ _-]?overview|google[ _-]?ai|\bsge\b|gemini/i,"llm","Google AI Overview"],[/instagram/i,"social","Instagram"],[/facebook|\bfb\b|meta/i,"social","Facebook"],[/linkedin/i,"social","LinkedIn"],[/tiktok/i,"social","TikTok"],[/youtube|youtu\.be/i,"social","YouTube"],[/twitter|x\.com/i,"social","X"],[/\bg2\b|g2\.com/i,"review","G2"],[/trustpilot/i,"review","Trustpilot"],[/\bbbb\b|better business bureau/i,"review","BBB"],[/capterra/i,"review","Capterra"],[/yelp/i,"review","Yelp"],[/google|bing|duckduckgo|yahoo/i,"search","Search"]];function q(e,t=240){return typeof e=="string"&&e.trim()?e.trim().slice(0,t):null}function Ce(e){try{return e?new URL(e).hostname.replace(/^www\./,""):""}catch{return""}}function Re(e){let t=e.properties??{},i=q(e.source,180)||Ce(e.referrer)||"(direct)",n=q(e.medium,180)||(i==="(direct)"?"(none)":"referral"),s=e.clickIds??{},a=`${i} ${n} ${Ce(e.referrer)}`,r=i==="(direct)"?"direct":/email|newsletter/i.test(n)?"email":"referral",o=i==="(direct)"?"Direct":i;for(let[d,_,u]of ct)if(d.test(a)){r=_,o=u;break}return s.fbclid?(r="social",o=/instagram/i.test(a)?"Instagram":"Facebook"):s.ttclid?(r="social",o="TikTok"):s.rdt_cid?(r="social",o="Reddit"):s.gclid||s.gbraid||s.wbraid?(r="search",o="Google"):s.msclkid&&(r="search",o="Microsoft Ads"),{channelFamily:r,platform:o,source:i,medium:n,campaign:q(e.campaign),campaignId:q(t.campaign_id??t.utm_id),adSetId:q(t.adset_id??t.ad_set_id??t.adgroup_id??t.ad_group_id),adId:q(t.ad_id),creativeId:q(t.creative_id),placement:q(t.placement),term:q(t.utm_term),content:q(t.utm_content)}}import{createHash as yt,randomUUID as ee}from"crypto";import{createCipheriv as _t,createDecipheriv as ut,createHmac as fe,randomBytes as Et,scryptSync as lt,timingSafeEqual as Nt}from"crypto";var Z="xray1";function mt(){return process.env.ANALYTICS_RESTRICTED_DATA_SECRET?.trim()||Q()}function _e(e){return lt(mt(),`xray-restricted:${e}:v1`,32)}function De(e,t){let i=t.trim().slice(0,80);if(!i)throw new Error("analytics encryption purpose required");let n=Et(12),s=_t("aes-256-gcm",_e(i),n);s.setAAD(Buffer.from(`${Z}:${i}`,"utf8"));let a=Buffer.concat([s.update(e,"utf8"),s.final()]),r=s.getAuthTag();return[Z,Buffer.from(i,"utf8").toString("base64url"),n.toString("base64url"),r.toString("base64url"),a.toString("base64url")].join(".")}function Ue(e,t){try{let[i,n,s,a,r]=e.split(".");if(i!==Z||!n||!s||!a||!r)return null;let o=Buffer.from(n,"base64url").toString("utf8");if(o!==t)return null;let d=ut("aes-256-gcm",_e(o),Buffer.from(s,"base64url"));return d.setAAD(Buffer.from(`${Z}:${o}`,"utf8")),d.setAuthTag(Buffer.from(a,"base64url")),Buffer.concat([d.update(Buffer.from(r,"base64url")),d.final()]).toString("utf8")}catch{return null}}function bi(e,t){return fe("sha256",_e(`identity:${e}`)).update(t.trim().toLowerCase()).digest("hex")}function hi(e,t,i,n){return fe("sha256",n).update(`${e}.${t}.${i}`).digest("hex")}function xi(e,t){if(!e||e.length!==t.length)return!1;try{return Nt(Buffer.from(e),Buffer.from(t))}catch{return!1}}var we="enhanced_matching",pt=90,Tt=10,H=class extends Error{constructor(i,n,s){super(n);this.code=i;this.status=s;this.name="AnalyticsIdentityProfileError"}code;status};function te(e,t){return e?.trim().replace(/\s+/g," ").slice(0,t)||null}function Lt(e){if(!e)return null;let[t,i]=e.split("@");return!t||!i?null:`${t.slice(0,1)}${"*".repeat(Math.min(6,Math.max(2,t.length-1)))}@${i}`}function At(e){if(!e)return null;let t=e.replace(/\D/g,"");return t.length<4?null:`***-***-${t.slice(-4)}`}function ue(e){return yt("sha256").update(e).digest("hex")}async function ie(e,t){if(!t.signals.length)return;let i=t.signals.map(a=>a.kind),n=t.signals.map(a=>a.valueHmac);if((await e.query(`SELECT 1 FROM analytics_identity_tombstones
+import{a as Q,d as Se}from"./chunk-L552AQQY.js";import{createHash as pe,createHmac as Yt,randomBytes as P,randomUUID as y}from"crypto";import{Pool as Wt}from"pg";var ct=[[/perplexity/i,"llm","Perplexity"],[/chatgpt|chat\.openai|openai/i,"llm","ChatGPT"],[/claude|anthropic/i,"llm","Claude"],[/\bgrok\b|xai/i,"llm","Grok"],[/ai[ _-]?overview|google[ _-]?ai|\bsge\b|gemini/i,"llm","Google AI Overview"],[/instagram/i,"social","Instagram"],[/facebook|\bfb\b|meta/i,"social","Facebook"],[/linkedin/i,"social","LinkedIn"],[/tiktok/i,"social","TikTok"],[/youtube|youtu\.be/i,"social","YouTube"],[/twitter|x\.com/i,"social","X"],[/\bg2\b|g2\.com/i,"review","G2"],[/trustpilot/i,"review","Trustpilot"],[/\bbbb\b|better business bureau/i,"review","BBB"],[/capterra/i,"review","Capterra"],[/yelp/i,"review","Yelp"],[/google|bing|duckduckgo|yahoo/i,"search","Search"]];function q(e,t=240){return typeof e=="string"&&e.trim()?e.trim().slice(0,t):null}function Ce(e){try{return e?new URL(e).hostname.replace(/^www\./,""):""}catch{return""}}function Re(e){let t=e.properties??{},i=q(e.source,180)||Ce(e.referrer)||"(direct)",n=q(e.medium,180)||(i==="(direct)"?"(none)":"referral"),s=e.clickIds??{},a=`${i} ${n} ${Ce(e.referrer)}`,r=i==="(direct)"?"direct":/email|newsletter/i.test(n)?"email":"referral",o=i==="(direct)"?"Direct":i;for(let[d,_,u]of ct)if(d.test(a)){r=_,o=u;break}return s.fbclid?(r="social",o=/instagram/i.test(a)?"Instagram":"Facebook"):s.ttclid?(r="social",o="TikTok"):s.rdt_cid?(r="social",o="Reddit"):s.gclid||s.gbraid||s.wbraid?(r="search",o="Google"):s.msclkid&&(r="search",o="Microsoft Ads"),{channelFamily:r,platform:o,source:i,medium:n,campaign:q(e.campaign),campaignId:q(t.campaign_id??t.utm_id),adSetId:q(t.adset_id??t.ad_set_id??t.adgroup_id??t.ad_group_id),adId:q(t.ad_id),creativeId:q(t.creative_id),placement:q(t.placement),term:q(t.utm_term),content:q(t.utm_content)}}import{createHash as yt,randomUUID as ee}from"crypto";import{createCipheriv as _t,createDecipheriv as ut,createHmac as fe,randomBytes as Et,scryptSync as lt,timingSafeEqual as Nt}from"crypto";var Z="xray1";function mt(){return process.env.ANALYTICS_RESTRICTED_DATA_SECRET?.trim()||Q()}function _e(e){return lt(mt(),`xray-restricted:${e}:v1`,32)}function De(e,t){let i=t.trim().slice(0,80);if(!i)throw new Error("analytics encryption purpose required");let n=Et(12),s=_t("aes-256-gcm",_e(i),n);s.setAAD(Buffer.from(`${Z}:${i}`,"utf8"));let a=Buffer.concat([s.update(e,"utf8"),s.final()]),r=s.getAuthTag();return[Z,Buffer.from(i,"utf8").toString("base64url"),n.toString("base64url"),r.toString("base64url"),a.toString("base64url")].join(".")}function Ue(e,t){try{let[i,n,s,a,r]=e.split(".");if(i!==Z||!n||!s||!a||!r)return null;let o=Buffer.from(n,"base64url").toString("utf8");if(o!==t)return null;let d=ut("aes-256-gcm",_e(o),Buffer.from(s,"base64url"));return d.setAAD(Buffer.from(`${Z}:${o}`,"utf8")),d.setAuthTag(Buffer.from(a,"base64url")),Buffer.concat([d.update(Buffer.from(r,"base64url")),d.final()]).toString("utf8")}catch{return null}}function bi(e,t){return fe("sha256",_e(`identity:${e}`)).update(t.trim().toLowerCase()).digest("hex")}function hi(e,t,i,n){return fe("sha256",n).update(`${e}.${t}.${i}`).digest("hex")}function xi(e,t){if(!e||e.length!==t.length)return!1;try{return Nt(Buffer.from(e),Buffer.from(t))}catch{return!1}}var we="enhanced_matching",pt=90,Tt=10,H=class extends Error{constructor(i,n,s){super(n);this.code=i;this.status=s;this.name="AnalyticsIdentityProfileError"}code;status};function te(e,t){return e?.trim().replace(/\s+/g," ").slice(0,t)||null}function Lt(e){if(!e)return null;let[t,i]=e.split("@");return!t||!i?null:`${t.slice(0,1)}${"*".repeat(Math.min(6,Math.max(2,t.length-1)))}@${i}`}function At(e){if(!e)return null;let t=e.replace(/\D/g,"");return t.length<4?null:`***-***-${t.slice(-4)}`}function ue(e){return yt("sha256").update(e).digest("hex")}async function ie(e,t){if(!t.signals.length)return;let i=t.signals.map(a=>a.kind),n=t.signals.map(a=>a.valueHmac);if((await e.query(`SELECT 1 FROM analytics_identity_tombstones
       WHERE site_id=$1 AND (kind,value_hmac) IN (
         SELECT * FROM unnest($2::text[],$3::text[])
       ) LIMIT 1`,[t.siteId,i,n])).rowCount)throw new H("analytics_identity_deleted","This identity cannot be automatically recreated.",410)}async function ne(e,t){let i=te(t.displayName,160),n=te(t.email,320)?.toLowerCase()??null,s=te(t.phone,40);if(!i&&!n&&!s)return!1;let r=(await e.query(`SELECT p.public_ref FROM analytics_people p

package/dist/{chunk-NLKA6SHC.js → chunk-72MWPIMR.js}

@@ -1,4 +1,4 @@
-import{k as K}from"./chunk-IJF4VXJ7.js";import{a as X}from"./chunk-QTDLZTQ7.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-WJ4XFLS4.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
+import{k as K}from"./chunk-BRNTVOCW.js";import{a as X}from"./chunk-L552AQQY.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-ZPK5LYEN.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
       CREATE TABLE IF NOT EXISTS gmail_selections (
         selection_id TEXT PRIMARY KEY,
         owner_identity TEXT NOT NULL,