mcp-scraper 0.95.2 → 0.96.0

package/dist/chunk-GPSPSJBJ.js

@@ -1 +0,0 @@
-function o(e,_=!1,n="light",r=1,R=!1,t=!1){let E=_&&(R||t)?24e4:_&&n==="full"?r>=2?75e3:45e3:_?4e4:4e5,S=Math.min(E+5e3,795e3);return{serverMs:E,clientMs:S}}function M(){return(process.env.BROWSER_SERVICE_API_KEY??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_API_KEY)?.trim()||void 0}function u(){return(process.env.BROWSER_AGENT_PROFILE_NAME??process.env.BROWSER_SERVICE_PROFILE_NAME??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_PROFILE_NAME??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_PROFILE_NAME)?.trim()||void 0}function i(){let e=(process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES??process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_PROFILE_SAVE_CHANGES??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();if(e){if(["1","true","yes","on"].includes(e))return!0;if(["0","false","no","off"].includes(e))return!1}}export{o as a,M as b,u as c,i as d};

package/dist/chunk-HE45FFBU.js

@@ -1 +0,0 @@
-import{isIP as I}from"net";import{lookup as T}from"dns/promises";import p from"ipaddr.js";function P(n){return n.startsWith("[")&&n.endsWith("]")?n.slice(1,-1):n}function b(n){let t=P(n);if(!I(t))return!1;try{let e=p.parse(t),s=o=>[["192.0.0.0",24],["192.0.2.0",24],["198.18.0.0",15],["198.51.100.0",24],["203.0.113.0",24]].some(([m,u])=>o.match(p.parse(m),u));if(e instanceof p.IPv6&&e.isIPv4MappedAddress()){let o=e.toIPv4Address();return o.range()!=="unicast"||s(o)}return e instanceof p.IPv4&&s(e)?!0:e.range()!=="unicast"}catch{return!0}}async function N(n){let t=P(n.toLowerCase());if(t==="localhost"||t.endsWith(".localhost")||t.endsWith(".local")||b(t))return!0;try{let e=await T(t,{all:!0,verbatim:!0});return e.length===0||e.some(s=>b(s.address))}catch{return!0}}async function S(n,t){let e;try{e=new URL(n.trim())}catch{return{error:`Invalid ${t.field}`}}return(t.requireHttps?["https:"]:["http:","https:"]).includes(e.protocol)?await N(e.hostname)?{error:`${t.field} must resolve to a public internet host`}:{parsed:e}:{error:t.requireHttps?`${t.field} must use https`:`${t.field} must use http or https`}}import{load as C}from"cheerio";var _="script,style,noscript,svg,iframe,canvas,nav,header,footer,aside",k=4096;function d(n){return(n??"").replace(/\s+/g," ").trim()}function L(n){let t=n.replace(/[\x00-\x09\x0b\x0c\x0e-\x1f]/g," ").trim();if(!t)return null;try{return JSON.parse(t)}catch(e){let s=/at position (\d+)/.exec(e instanceof Error?e.message:"")?.[1];if(!s)return null;try{return JSON.parse(t.slice(0,Number(s)))}catch{return null}}}function g(n,t,e={value:2e3}){if(e.value--<=0||n==null)return;if(Array.isArray(n)){for(let l of n)g(l,t,e);return}if(typeof n!="object")return;let s=n,o=s["@type"];if(typeof o=="string"&&o&&t.add(o),Array.isArray(o))for(let l of o)typeof l=="string"&&l&&t.add(l);Array.isArray(s["@graph"])&&g(s["@graph"],t,e)}function H(n){let t=n("main").first();if(t.length)return t;let e=n("article").first();if(e.length)return e;let s=n("body").first();return s.length?s:n.root()}function w(n,t){let e=Number(n.attr(t)??0);return Number.isFinite(e)&&e>0?e:null}function W(n){let t=C(n),e={};t("meta").each((f,i)=>{let r=t(i),a=d(r.attr("name")??r.attr("property")).toLowerCase(),c=d(r.attr("content"));a&&c&&(e[a]=c)});let s=[];t("h1,h2,h3,h4,h5,h6").each((f,i)=>{let r=t(i),a=d(r.text());a&&s.push({level:Number(i.tagName.slice(1)),text:a})});let o=[],l=new Set;t('script[type="application/ld+json"]').each((f,i)=>{let r=L(t(i).text());r!=null&&(Array.isArray(r)?o.push(...r):o.push(r),g(r,l))});let m=new Set;t("[itemtype]").each((f,i)=>{let r=t(i).attr("itemtype")??"";for(let a of r.split(/\s+/))try{let c=new URL(a);if(c.hostname==="schema.org"||c.hostname==="www.schema.org"){let h=c.pathname.replace(/^\/+/,"");h&&m.add(h)}}catch{}});let u=H(t).clone();u.find(_).remove(),u.find("[href],[src],[srcset],[data-src],[data-srcset]").each((f,i)=>{let r=t(i);for(let a of["href","src","srcset","data-src","data-srcset"]){let c=r.attr(a);c&&c.length>k&&r.removeAttr(a)}}),u.find("img").each((f,i)=>{let r=t(i),a=r.attr("src");if(!a||a.startsWith("data:")){let c=r.attr("data-src")??r.attr("data-lazy-src")??r.attr("data-original");c&&r.attr("src",c)}});let A=u.html()??"",v=d(u.text()),y=[];t("a[href]").each((f,i)=>{let r=t(i),a=d(r.attr("href"));a&&y.push({href:a,text:d(r.text()),rel:d(r.attr("rel")).toLowerCase()||null})});let x=[];return t("img").each((f,i)=>{let r=t(i),a=Object.fromEntries(Object.entries(i.attribs??{}).map(([c,h])=>[c.toLowerCase(),h]));x.push({src:r.attr("src")??r.attr("data-src")??r.attr("data-lazy-src")??r.attr("data-original")??null,srcset:r.attr("srcset")??r.attr("data-srcset")??null,alt:r.attr("alt")??null,width:w(r,"width"),height:w(r,"height"),attributes:a})}),{$:t,sourceHtml:n,title:d(t("title").first().text())||null,meta:e,headings:s,schema:o,schemaTypes:[...l],microdataTypes:[...m],canonicalUrl:d(t('link[rel~="canonical"]').first().attr("href"))||null,mainHtml:A,mainText:v,bodyClass:t("body").first().attr("class")??"",links:y,images:x}}export{b as a,S as b,W as c};

package/dist/chunk-HUV2WTRW.js

@@ -1 +0,0 @@
-import{createHash as B,randomUUID as N}from"crypto";import{mkdir as O,readFile as T,stat as v,writeFile as U}from"fs/promises";import{homedir as k}from"os";import{dirname as S,join as A}from"path";var d="connected-data-exports/",b=10080*60*1e3,R=900*1e3;function p(){return process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function D(){return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim()||A(k(),"Downloads","mcp-scraper")}function g(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function x(t){return(t.toLowerCase().replace(/[^a-z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||"connected-data-export").slice(0,120)}function C(t){if(!t.startsWith(d))return null;let e=(t.split("/").at(-1)??"").match(/^(\d{13})-/);if(!e)return null;let r=Number(e[1]);return Number.isFinite(r)?r:null}function _(t){return t.startsWith(d)&&t.slice(d.length).split("/")[0]||null}function P(t){let n=C(t);return n===null?null:new Date(n+b)}function I(t,n=Date.now()){let e=C(t);return e===null||e+b<=n}async function h(t,n){let e=p();if(!e)return null;let r=Math.min(Date.now()+R,n.getTime());if(r<=Date.now())return null;let{issueSignedToken:a,presignUrl:i}=await import("@vercel/blob"),s=await a({token:e,pathname:t,operations:["get"],validUntil:r}),{presignedUrl:o}=await i(s,{access:"private",operation:"get",pathname:t,validUntil:r});return{url:o,expiresAt:new Date(r).toISOString()}}async function H(t){return L({...t,contentType:"application/x-ndjson"})}async function L(t){let n=t.stableCreatedAt?Date.parse(t.stableCreatedAt):Number.NaN,e=Number.isFinite(n)?n:Date.now(),r=x(t.filename),a=r.includes(".")?`.${r.split(".").at(-1)}`:".bin",i=t.stableKey?x(t.stableKey):N(),s=`${d}${t.ownerId}/${e}-${x(t.exportId)}-${i}--${r}`,o=(typeof t.content=="string",Buffer.from(t.content)),u=o.byteLength,c=B("sha256").update(o).digest("hex"),l=new Date(e+b),f=p(),m=s;if(f){let{put:w}=await import("@vercel/blob");m=(await w(s,o,{access:"private",token:f,contentType:t.contentType,addRandomSuffix:!t.stableKey,...t.stableKey?{allowOverwrite:!0}:{},cacheControlMaxAge:60,multipart:u>100*1024*1024})).pathname}else{if(g())throw new Error("connected_data_private_blob_not_configured");let w=A(D(),"blobs",s);await O(S(w),{recursive:!0}),await U(w,o)}let y=await h(m,l);return{artifactId:m,filename:r,contentType:t.contentType,bytes:u,sha256:c,createdAt:new Date(e).toISOString(),expiresAt:l.toISOString(),downloadUrl:y?.url??null,downloadUrlExpiresAt:y?.expiresAt??null,readback:{tool:"report_artifact_read",arguments:{artifactId:m,offset:0,maxBytes:2e4},continuation:"Repeat with offset set to the previous result nextOffset until nextOffset is null."}}}async function V(t){if(_(t.artifactId)!==t.ownerId)return null;let n=P(t.artifactId);if(!n||n.getTime()<=Date.now())return null;let e=await h(t.artifactId,n);return e?{downloadUrl:e.url,downloadUrlExpiresAt:e.expiresAt,expiresAt:n.toISOString()}:null}async function E(t,n=Number.POSITIVE_INFINITY){let e=t.getReader(),r=[],a=0;try{for(;;){let{done:i,value:s}=await e.read();if(i)break;if(s){let o=Buffer.from(s);if(a+=o.byteLength,a>n)return await e.cancel("artifact_too_large").catch(()=>{}),null;r.push(o)}}}finally{e.releaseLock()}return Buffer.concat(r)}async function j(t,n,e){if(I(t))return null;let r=p(),a;if(r){let{get:u}=await import("@vercel/blob"),c=await u(t,{access:"private",token:r,useCache:!1});if(!c||c.statusCode!==200)return null;let l=await E(c.stream);if(!l)return null;a=l}else{if(g())return null;try{a=await T(A(D(),"blobs",t))}catch{return null}}let i=a.length,s=Math.max(0,n),o=Math.min(i,s+e);return{text:a.subarray(s,o).toString("utf8"),totalBytes:i,nextOffset:o<i?o:null}}async function q(t){if(_(t.artifactId)!==t.ownerId||I(t.artifactId))return null;let n=p(),e;if(n){let{get:r}=await import("@vercel/blob"),a=await r(t.artifactId,{access:"private",token:n,useCache:!1});if(!a||a.statusCode!==200)return null;let i=await E(a.stream,t.maxBytes??25*1024*1024);if(!i)return null;e=i}else{if(g())return null;try{let r=A(D(),"blobs",t.artifactId);if((await v(r)).size>(t.maxBytes??25*1024*1024))return null;e=await T(r)}catch{return null}}return e.byteLength>(t.maxBytes??25*1024*1024)?null:e}async function z(t={}){let n=t.now??new Date,e=p();if(!e)return{deleted:0,store:g()?"none":"local"};if(!t.force&&!(n.getUTCHours()===3&&n.getUTCMinutes()===17))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=n.getTime()-b,{list:a,del:i}=await import("@vercel/blob"),s,o=0;for(let u=0;u<20;u++){let c=await a({prefix:d,token:e,limit:1e3,cursor:s}),l=c.blobs.filter(f=>new Date(f.uploadedAt).getTime()<=r);if(l.length>0&&(await i(l.map(f=>f.pathname),{token:e}),o+=l.length),!c.hasMore||!c.cursor)break;s=c.cursor}return{deleted:o,store:"private-vercel-blob"}}export{d as a,_ as b,H as c,L as d,V as e,j as f,q as g,z as h};