mcp-scraper 0.95.1 → 0.96.0

package/dist/chunk-B3JRL45T.js

@@ -0,0 +1 @@
+import{isIP as c}from"net";import{lookup as d}from"dns/promises";import o from"ipaddr.js";function a(e){return e.startsWith("[")&&e.endsWith("]")?e.slice(1,-1):e}function i(e){let r=a(e);if(!c(r))return!1;try{let t=o.parse(r),s=n=>[["192.0.0.0",24],["192.0.2.0",24],["198.18.0.0",15],["198.51.100.0",24],["203.0.113.0",24]].some(([l,u])=>n.match(o.parse(l),u));if(t instanceof o.IPv6&&t.isIPv4MappedAddress()){let n=t.toIPv4Address();return n.range()!=="unicast"||s(n)}return t instanceof o.IPv4&&s(t)?!0:t.range()!=="unicast"}catch{return!0}}async function p(e){let r=a(e.toLowerCase());if(r==="localhost"||r.endsWith(".localhost")||r.endsWith(".local")||i(r))return!0;try{let t=await d(r,{all:!0,verbatim:!0});return t.length===0||t.some(s=>i(s.address))}catch{return!0}}async function I(e,r){let t;try{t=new URL(e.trim())}catch{return{error:`Invalid ${r.field}`}}return(r.requireHttps?["https:"]:["http:","https:"]).includes(t.protocol)?await p(t.hostname)?{error:`${r.field} must resolve to a public internet host`}:{parsed:t}:{error:r.requireHttps?`${r.field} must use https`:`${r.field} must use http or https`}}export{i as a,I as b};

package/dist/chunk-B4ZKPLSP.js

@@ -0,0 +1 @@
+function S(){return(process.env.BROWSER_SERVICE_API_KEY??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_API_KEY)?.trim()||void 0}function i(){return(process.env.BROWSER_AGENT_PROFILE_NAME??process.env.BROWSER_SERVICE_PROFILE_NAME??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_PROFILE_NAME??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_PROFILE_NAME)?.trim()||void 0}function u(){let e=(process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES??process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_PROFILE_SAVE_CHANGES??process.env.\u004b\u0045\u0052\u004e\u0045\u004c_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();if(e){if(["1","true","yes","on"].includes(e))return!0;if(["0","false","no","off"].includes(e))return!1}}var A="none";function a(e){switch(e){case"day":return"qdr:d";case"week":return"qdr:w";case"month":return"qdr:m";case"year":return"qdr:y";default:return}}function M(e,r=!1,n="light",t=1,_=!1,E=!1){let o=r&&(_||E)?24e4:r&&n==="full"?t>=2?75e3:45e3:r?4e4:4e5,R=Math.min(o+5e3,795e3);return{serverMs:o,clientMs:R}}function s(e){return e.reason??new DOMException("The operation was aborted.","AbortError")}async function f(e,r){if(!r)return e;if(r.aborted)throw s(r);let n,t=new Promise((_,E)=>{n=()=>E(s(r)),r.addEventListener("abort",n,{once:!0})});e.catch(()=>{});try{return await Promise.race([e,t])}finally{n&&r.removeEventListener("abort",n)}}function O(e,r){let n=AbortSignal.timeout(e);return r?AbortSignal.any([r,n]):n}export{A as a,a as b,M as c,S as d,i as e,u as f,s as g,f as h,O as i};

package/dist/chunk-C7MGJUTU.js

@@ -0,0 +1 @@
+var k="Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.",m="This operation is temporarily unavailable. Please retry later.",b=["request_aborted","captcha_exhausted","captcha_or_blocked","location_mismatch","proxy_tunnel_failed","proxy_unavailable","harvest_timeout","mcp_request_timeout","mcp_http_error","extraction_failed","concurrency_limit_exceeded","unauthorized","forbidden","invalid_request","rate_limited","upstream_rate_limited","insufficient_balance","service_unavailable","response_lost","vendor_unavailable","page_not_found","page_forbidden","page_rate_limited","page_server_error","page_http_error","bot_check_unresolved","page_too_large","page_unreachable","browser_session_interrupted","site_export_not_found","directory_job_not_found","site_export_format_unavailable","site_export_read_failed","site_export_image_not_found","image_url_invalid","image_url_not_https","image_host_unresolvable","image_host_blocked","image_redirect_invalid","image_source_unavailable","image_too_large","image_redirect_limit","image_base64_invalid","image_source_missing","image_source_ambiguous","image_source_type_invalid","image_artifact_not_found","image_page_too_large","page_image_not_found","page_image_candidates_unusable","image_format_unsupported","image_storage_unconfigured","featured_image_unhostable"],h=["request_aborted","captcha","location_mismatch","connection","timeout","extraction","concurrency_limit","unauthorized","forbidden","invalid_request","not_found","rate_limit","billing","service_unavailable"];function S(e){return e?JSON.stringify(e):null}function R(e){if(e==null)return null;try{let r=typeof e=="string"?JSON.parse(e):e;if(!r||typeof r!="object"||Array.isArray(r))return null;let t=r;if(typeof t.error_code!="string"||!p.has(t.error_code))return null;let a=C({errorCode:t.error_code,errorType:t.error_type,retryable:t.retryable,retryAfterSeconds:t.retry_after_seconds,chargeStatus:t.charge_status,details:t.details});if(typeof t.message=="string"){let n=g(t.message).trim().slice(0,1e3);n&&!/https?:\/\//i.test(n)&&(a.message=n)}return a}catch{return null}}var p=new Set(b),f=new Set(h),y=new Set(["active","limit","operation","upgrade_url","topup_url","upgrade_command","slots_per_pack","pack_price_usd","balance_credits","required_credits","retry_guidance","http_status"]),v=12,T=500;function o(e){return typeof e=="string"&&p.has(e)?e:"service_unavailable"}function x(e){if(!e||typeof e!="object"||Array.isArray(e))return;let r={};for(let[t,a]of Object.entries(e)){if(Object.keys(r).length>=v)break;if(y.has(t))if(typeof a=="string"){if((t==="upgrade_url"||t==="topup_url")&&!E(a))continue;let n=g(a).slice(0,T);n&&(r[t]=n)}else(typeof a=="number"&&Number.isFinite(a)||typeof a=="boolean"||a===null)&&(r[t]=a)}return Object.keys(r).length>0?r:void 0}function E(e){if(e.startsWith("/")&&!e.startsWith("//"))return!0;try{let r=new URL(e);return r.protocol==="https:"&&(r.hostname==="mcpscraper.dev"||r.hostname.endsWith(".mcpscraper.dev"))}catch{return!1}}function P(e){let r=o(e);switch(r){case"captcha_exhausted":case"captcha_or_blocked":return"captcha";case"proxy_tunnel_failed":case"proxy_unavailable":case"response_lost":return"connection";case"harvest_timeout":case"mcp_request_timeout":return"timeout";case"concurrency_limit_exceeded":return"concurrency_limit";case"rate_limited":case"upstream_rate_limited":return"rate_limit";case"extraction_failed":case"page_not_found":case"page_forbidden":case"page_rate_limited":case"page_server_error":case"page_http_error":case"bot_check_unresolved":case"page_too_large":case"page_unreachable":case"browser_session_interrupted":case"site_export_not_found":case"site_export_format_unavailable":case"site_export_read_failed":case"site_export_image_not_found":return"extraction";case"image_url_invalid":case"image_url_not_https":case"image_host_unresolvable":case"image_host_blocked":case"image_redirect_invalid":case"image_source_unavailable":case"image_too_large":case"image_redirect_limit":case"image_base64_invalid":case"image_source_missing":case"image_source_ambiguous":case"image_source_type_invalid":case"image_artifact_not_found":case"image_page_too_large":case"page_image_not_found":case"page_image_candidates_unusable":case"image_format_unsupported":case"image_storage_unconfigured":case"featured_image_unhostable":return"invalid_request";case"insufficient_balance":return"billing";case"vendor_unavailable":case"mcp_http_error":return"service_unavailable";case"directory_job_not_found":return"not_found";default:return r}}function w(e){switch(o(e)){case"request_aborted":return"The request was cancelled before it completed.";case"captcha_exhausted":case"captcha_or_blocked":return"The target site returned a verification challenge. Heavy traffic can temporarily increase these challenges; retry in a few minutes.";case"location_mismatch":return"The target site returned results for a different location. Please retry.";case"proxy_tunnel_failed":case"proxy_unavailable":return"The web request could not establish a working connection. Please retry later.";case"harvest_timeout":return"The request timed out before it completed. Please retry with a smaller request.";case"mcp_request_timeout":return"The MCP caller stopped waiting before a final response arrived. Durable work may still be running; continue with the returned Task or job handle instead of restarting it.";case"concurrency_limit_exceeded":return"The service is available, but this account has reached its parallel-operation limit. Wait for an active operation to finish or add capacity.";case"unauthorized":return"Authentication is required to perform this operation.";case"forbidden":return"This account does not have permission to perform this operation.";case"invalid_request":return"The request is invalid. Check the supplied inputs and try again.";case"rate_limited":return"This account is sending requests too quickly. Wait briefly and retry.";case"upstream_rate_limited":return"The connected provider is temporarily rate-limiting requests. Wait briefly and retry.";case"mcp_http_error":return"The API returned an unexpected HTTP response.";case"insufficient_balance":return"This account does not have enough Credits for the operation.";case"extraction_failed":return"The target could not be extracted. Check retryable before attempting it again.";case"directory_job_not_found":return"This directory job was not found for the current account.";case"response_lost":return"The connection was lost before the response arrived. The operation may have already completed and been billed on our side \u2014 check credits_info before retrying blind.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"page_not_found":return"The page could not be found. It may have been moved or deleted.";case"page_forbidden":return"The site refused the request.";case"page_rate_limited":return"The site is rate-limiting requests right now. Retrying after a short wait usually works.";case"page_server_error":return"The page's own server returned an error. This is on the target site's side, not something a retry here can fix.";case"page_http_error":return"The page returned an unexpected HTTP response.";case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. MCP Scraper will retry once with a fresh session; retry the request if this persists.";case"site_export_not_found":return"The site export or requested page was not found, expired, or does not belong to this account.";case"site_export_format_unavailable":return"That representation was not retained for this export page. Read the export manifest and request one of the formats marked available.";case"site_export_read_failed":return"The stored site export could not be read or did not pass its integrity check.";case"site_export_image_not_found":return"The requested site-export image was not found, expired, or does not belong to this account.";case"image_url_invalid":return"The image source is not a valid URL. Use a stable public HTTPS URL or inline image bytes.";case"image_url_not_https":return"Commons image sources must use a public HTTPS URL or inline image bytes.";case"image_host_unresolvable":return"The image host could not be resolved. Use a different stable public URL or inline image bytes.";case"image_host_blocked":return"The image source resolves to a network address Commons cannot fetch safely.";case"image_redirect_invalid":return"The image source redirected to an invalid or unsafe destination.";case"image_source_unavailable":return"The image source could not be fetched. Host a different stable public URL or provide inline image bytes.";case"image_too_large":return"The image exceeds the Commons image size limit.";case"image_redirect_limit":return"The image source exceeded the Commons redirect limit.";case"image_base64_invalid":return"The supplied inline image bytes are invalid.";case"image_source_missing":return"Provide exactly one of a public image or webpage URL, inline image bytes, or an owner-scoped image artifact.";case"image_source_ambiguous":return"Provide only one image source: sourceUrl, imageBase64, or artifactId.";case"image_source_type_invalid":return"Use sourceType only with inline image bytes or an owner-scoped image artifact.";case"image_artifact_not_found":return"The image artifact was not found, expired, or does not belong to this account.";case"image_page_too_large":return"The webpage is too large for bounded Commons image discovery.";case"page_image_not_found":return"The webpage did not expose a usable featured or content image.";case"page_image_candidates_unusable":return"The webpage exposed image candidates, but none returned supported public image bytes.";case"image_format_unsupported":return"Commons accepts JPEG, PNG, GIF, or WebP image bytes.";case"image_storage_unconfigured":return"Commons image storage is temporarily unavailable.";case"featured_image_unhostable":return"The required featured image could not be hosted for Commons publication.";default:return m}}function C(e){let r=o(e.errorCode),t=typeof e.retryAfterSeconds=="number"&&Number.isFinite(e.retryAfterSeconds)?Math.max(1,Math.min(86400,Math.trunc(e.retryAfterSeconds))):void 0,a=e.chargeStatus==="not_charged"||e.chargeStatus==="refund_pending"||e.chargeStatus==="refunded"||e.chargeStatus==="charged"?e.chargeStatus:void 0,n=typeof e.errorType=="string"&&f.has(e.errorType)?e.errorType:void 0,s=x(e.details);return{error_code:r,error_type:n??P(r),message:w(r),retryable:e.retryable===!0,...t?{retry_after_seconds:t}:{},...a?{charge_status:a}:{},...s?{details:s}:{}}}function g(e){return e.replace((new RegExp("https?:\\/\\/[^\\s]*(?:\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061|on\u006b\u0065\u0072\u006e\u0065\u006c|\u006b\u0065\u0072\u006e\u0065\u006c\\.sh)[^\\s]*", "gi")),"the service").replace(/bright\s*data/gi,"the service").replace((new RegExp("\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061", "gi")),"the service").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\.sh\\s+sessions?", "gi")),"sessions").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\.sh\\s+session", "gi")),"this session").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\.sh", "gi")),"the service").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\s+sessions?", "gi")),"sessions").replace((new RegExp("\u006b\u0065\u0072\u006e\u0065\u006c\\s+session", "gi")),"this session").replace(/\b(?:sub|cus|sess|proxy)_[A-Za-z0-9_-]+\b/g,"[redacted]").replace((new RegExp("\\b(?:\u004b\u0045\u0052\u004e\u0045\u004c|\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041|\u0042\u0052\u0049\u0047\u0048\u0054_\u0044\u0041\u0054\u0041)_[A-Z0-9_]*API_KEY\\b", "g")),"BROWSER_SERVICE_API_KEY").replace((new RegExp("\\b\u006b\u0065\u0072\u006e\u0065\u006c\\b", "gi")),"the service").replace(/  +/g," ").trim()}var i=class extends Error{constructor(t){super(`CAPTCHA detected. ${t}`);this.instructions=t}instructions;name="CaptchaError"},c=class extends Error{constructor(t,a){super(t);this.cause=a}cause;name="ExtractionError"},u=class extends Error{constructor(t){super(`Google Maps search did not reach a complete result state (${t})`);this.pageState=t}pageState;name="MapsSearchIncompletePageError"},l=class extends Error{name="RequestAbortedError";constructor(r="Request aborted before harvest completed"){super(r)}},_=class extends Error{name="PaaInteractionInterruptedError"},d=class extends Error{name="LocationMismatchError";constructor(r="Google returned results for a different location than requested"){super(r)}};export{k as a,m as b,b as c,S as d,R as e,o as f,w as g,C as h,g as i,i as j,c as k,u as l,l as m,_ as n,d as o};

package/dist/{chunk-W2BVJ7S2.js → chunk-CNSVZTTT.js}

@@ -1,13 +1,13 @@
-import{b as y}from"./chunk-HE45FFBU.js";var $=100*1024,L=new Set(["webp","avif","svg","svg+xml"]);function w(s){return s==null?null:s<1024?`${s} B`:s<1024*1024?`${(s/1024).toFixed(1)} KB`:`${(s/(1024*1024)).toFixed(2)} MB`}var R=s=>s.replace(/&amp;/g,"&").replace(/&#0?38;/g,"&").replace(/&#x26;/gi,"&").trim(),M=s=>s.replace(/^https?:\/\//i,"//").replace(/\/$/,""),E=(s,r)=>{if(s){let c=s.split(";")[0].trim().toLowerCase();if(c.startsWith("image/"))return c.replace("image/","")}return(r.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1]||"unknown").toLowerCase()};function T(s){let r=new Map;for(let c of s)for(let n of c.imageLinks||[]){let e=R(n);if(!/^https?:\/\//i.test(e)||/[{}]|%7[bd]/i.test(e))continue;let a=M(e),i=r.get(a);(!i||/^https:/i.test(e)&&/^http:/i.test(i))&&r.set(a,e)}return[...r.values()]}async function D(s,r){let c=new AbortController,n=setTimeout(()=>c.abort(),r),e=i=>({len:i.headers.get("content-length"),cr:i.headers.get("content-range"),ct:i.headers.get("content-type"),status:i.status}),a=async i=>{let g=s;for(let p=0;p<=5;p++){let m=await y(g,{field:"image URL"});if(m.error||!m.parsed)throw new Error(m.error??"Image URL was rejected");let u=await fetch(m.parsed.href,{method:i,...i==="GET"?{headers:{Range:"bytes=0-0"}}:{},redirect:"manual",signal:c.signal});if(u.status>=300&&u.status<400){let t=u.headers.get("location");if(!t)throw new Error(`HTTP ${u.status} redirect did not include Location`);g=new URL(t,m.parsed.href).href;continue}return u}throw new Error("Image request exceeded five redirects")};try{let i=null,g=null,p=null;try{let m=e(await a("HEAD"));p=m.status,g=m.ct,m.len&&(i=Number(m.len))}catch{}if(i==null){let m=await a("GET"),u=e(m);if(p=p??u.status,g=g??u.ct,u.cr&&u.cr.includes("/")){let t=u.cr.split("/")[1];t&&t!=="*"&&(i=Number(t))}else u.len&&(i=Number(u.len));try{await m.body?.cancel()}catch{}}return{url:s,status:p,bytes:Number.isFinite(i)?i:null,contentType:g}}catch(i){return{url:s,status:null,bytes:null,contentType:null,error:String(i.message||i)}}finally{clearTimeout(n)}}async function v(s,r,c){let n=new Array(s.length),e=0;return await Promise.all(Array.from({length:Math.min(r,s.length)},async()=>{for(;e<s.length;){let a=e++;n[a]=await c(s[a])}})),n}async function S(s,r={}){return A(T(s),r)}async function A(s,r={}){let c=r.concurrency??12,n=r.timeoutMs??12e3,e=r.max??5e3,a=T([{imageLinks:s}]).slice(0,e),g=(await v(a,c,t=>D(t,n))).map(t=>{let f=E(t.contentType,t.url);return{...t,size:w(t.bytes),format:f,over100kb:t.bytes!=null&&t.bytes>$,legacyFormat:f!=="unknown"&&!L.has(f)}}),p=g.filter(t=>t.bytes!=null),m=p.reduce((t,f)=>t+f.bytes,0),u={};for(let t of g)u[t.format]=(u[t.format]||0)+1;return{rows:g.sort((t,f)=>(f.bytes??0)-(t.bytes??0)),summary:{unique:g.length,sized:p.length,totalBytes:m,totalSize:w(m)??"0 B",avgSize:w(Math.round(m/(p.length||1)))??"0 B",over100kb:g.filter(t=>t.over100kb).length,legacyFormat:g.filter(t=>t.legacyFormat).length,formatCounts:u,...r.sampleTruncated?{sampleTruncated:!0,sampleCap:e}:{}}}}function _(s){let r=s.summary,c=Object.entries(r.formatCounts).sort((a,i)=>i[1]-a[1]).map(([a,i])=>`${a}:${i}`).join("  "),n=s.rows.filter(a=>a.over100kb).slice(0,15),e=["## Images",`**${r.unique} unique images** \xB7 ${r.totalSize} total \xB7 ${r.avgSize} avg \xB7 ${r.over100kb} over 100 KB \xB7 ${r.legacyFormat} legacy format`,`Formats: ${c}`];if(n.length){e.push("","| Size | Format | URL |","|------|--------|-----|");for(let a of n)e.push(`| ${a.size} | ${a.format} | ${a.url} |`)}return e.join(`
-`)}function I(s){let r=s.replace(/^www\./,"").toLowerCase(),c=r.split(".");return c.length<=2?r:c.slice(-2).join(".")}function U(s,r,c){let n="";try{n=I(new URL(c).hostname)}catch{n=""}let e=s.filter(l=>l.internal),a=r.length,i=r.filter(l=>l.orphan).length,g=e.filter(l=>l.targetStatus!=null&&l.targetStatus>=400).length,p=r.reduce((l,h)=>l+h.inlinks,0),m=r.reduce((l,h)=>l+h.outlinksInternal+h.outlinksExternal,0),u={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0};for(let l of r)l.inlinks===0?u.zero++:l.inlinks<=2?u.oneToTwo++:l.inlinks<=10?u.threeToTen++:u.elevenPlus++;let t=[...r].sort((l,h)=>h.inlinks-l.inlinks).slice(0,20).map(l=>({url:l.url,inlinks:l.inlinks,outlinksInternal:l.outlinksInternal,outlinksExternal:l.outlinksExternal})),f=new Map,d=0;for(let l of s){let h;try{h=I(new URL(l.to).hostname)}catch{continue}if(!h||h===n)continue;d++;let k=f.get(h)??{links:0,nofollow:0,pages:new Set};k.links++,l.nofollow&&k.nofollow++,k.pages.add(l.from),f.set(h,k)}let o=[...f.entries()].map(([l,h])=>({domain:l,links:h.links,nofollow:h.nofollow,pages:h.pages.size})).sort((l,h)=>h.links-l.links),b=l=>Math.round(l*10)/10;return{summary:{internal:{totalLinks:e.length,pages:a,orphans:i,brokenInternal:g,avgInlinks:a?b(p/a):0,avgOutlinks:a?b(m/a):0,distribution:u,topByInlinks:t},external:{totalLinks:d,uniqueDomains:o.length,topDomains:o.slice(0,20)}},externalDomains:o}}function X(s){let r=s.summary;return["## Link analysis",`**Internal:** ${r.internal.totalLinks} links \xB7 ${r.internal.pages} pages \xB7 avg ${r.internal.avgInlinks} inlinks/page \xB7 ${r.internal.orphans} orphans \xB7 ${r.internal.brokenInternal} broken`,`**External:** ${r.external.totalLinks} links to ${r.external.uniqueDomains} domains`,"","### Inlink distribution",`- 0 inlinks (orphans): ${r.internal.distribution.zero}`,`- 1\u20132: ${r.internal.distribution.oneToTwo}`,`- 3\u201310: ${r.internal.distribution.threeToTen}`,`- 11+: ${r.internal.distribution.elevenPlus}`,"","### Top 20 internal pages by inlinks","| inlinks | out (int/ext) | URL |","|---|---|---|",...r.internal.topByInlinks.map(n=>`| ${n.inlinks} | ${n.outlinksInternal}/${n.outlinksExternal} | ${n.url} |`),"","### Top 20 external domains by links","| links | nofollow | from pages | domain |","|---|---|---|---|",...r.external.topDomains.map(n=>`| ${n.links} | ${n.nofollow} | ${n.pages} | ${n.domain} |`)].join(`
-`)}function x(s,r){let c=new Map;for(let n of s){let e=r(n);if(e==null||e==="")continue;let a=String(e);c.has(a)||c.set(a,[]),c.get(a).push(n.url)}return[...c.values()].filter(n=>n.length>1).flat()}function B(s,r,c){let n=t=>({count:t.length,urls:t.slice(0,200)}),e=t=>s.filter(t).map(f=>f.url),a=t=>t.extractionStatus!=="failed",i=t=>a(t)&&t.status===200,g=new Set(s.filter(a).map(t=>t.url)),p=t=>{try{return new URL(t).pathname}catch{return""}},m={"title.missing":n(e(t=>i(t)&&!t.title)),"title.duplicate":n(x(s.filter(i),t=>t.title)),"title.tooLong":n(e(t=>(t.titleLength??0)>60||(t.titlePixels??0)>561)),"title.tooShort":n(e(t=>t.title!=null&&(t.titleLength??0)<30)),"title.sameAsH1":n(e(t=>!!t.title&&!!t.h1&&t.title.trim()===t.h1.trim())),"meta.missing":n(e(t=>i(t)&&!t.metaDescription)),"meta.duplicate":n(x(s.filter(i),t=>t.metaDescription)),"meta.tooLong":n(e(t=>(t.metaDescLength??0)>155)),"meta.tooShort":n(e(t=>t.metaDescription!=null&&(t.metaDescLength??0)<70)),"h1.missing":n(e(t=>i(t)&&!t.h1)),"h1.multiple":n(e(t=>!!t.h1_2)),"h1.duplicate":n(x(s.filter(i),t=>t.h1)),"h1.tooLong":n(e(t=>(t.h1?.length??0)>70)),"h2.missing":n(e(t=>i(t)&&t.h2Count===0)),"crawl.extractionFailed":n(e(t=>!a(t))),"indexability.nonIndexable":n(e(t=>a(t)&&!t.indexable)),"indexability.noindex":n(e(t=>t.indexabilityReason==="noindex")),"canonical.missing":n(e(t=>i(t)&&!t.canonicalUrl)),"canonical.canonicalised":n(e(t=>t.indexabilityReason==="canonicalised")),"response.broken4xx":n(e(t=>(t.status??0)>=400&&(t.status??0)<500)),"response.error5xx":n(e(t=>(t.status??0)>=500)),"response.redirect3xx":n(e(t=>(t.status??0)>=300&&(t.status??0)<400)),"response.noResponse":n(e(t=>t.status===null)),"content.thin":n(e(t=>i(t)&&t.wordCount>0&&t.wordCount<200)),"content.exactDuplicate":n(x(s.filter(t=>i(t)&&t.contentHash),t=>t.contentHash)),"images.missingAlt":n(e(t=>a(t)&&t.imagesMissingAlt>0)),"schema.missing":n(e(t=>i(t)&&t.schemaTypes.length===0)),"url.tooLong":n(e(t=>t.url.length>115)),"url.uppercase":n(e(t=>/[A-Z]/.test(p(t.url)))),"url.underscores":n(e(t=>p(t.url).includes("_"))),"links.orphan":n([...r.values()].filter(t=>t.orphan&&g.has(t.url)).map(t=>t.url))},u=c;if(!u){let t=d=>{try{let o=new URL(d);return o.hash="",(o.origin+o.pathname).replace(/\/+$/,"")+o.search}catch{return d.replace(/#.*$/,"").replace(/\/+$/,"")}},f=new Map(s.map(d=>[t(d.url),d.status]));u=new Set;for(let d of s)for(let o of d.outlinks??[]){if(!o.internal)continue;let b=f.get(t(o.href));b!=null&&b>=400&&u.add(d.url)}}return m["links.brokenInternal"]=n([...u]),m}function C(s,r,c,n){let e=r.length,a=r.filter(o=>o.extractionStatus!=="failed").length,i=e-a,g=o=>o.startsWith("response.broken")||o.startsWith("response.error")||o.startsWith("links.broken")?"\u{1F534}":o.startsWith("title.missing")||o.startsWith("h1.missing")||o.startsWith("indexability")||o.startsWith("canonical.missing")?"\u{1F7E0}":"\u{1F7E1}",p=Object.entries(c).filter(([,o])=>o.count>0).sort((o,b)=>b[1].count-o[1].count).map(([o,b])=>`| ${g(o)} | \`${o}\` | ${b.count} | ${b.urls.slice(0,3).join(" \xB7 ")}${b.urls.length>3?" \u2026":""} |`),m=[...n.values()].map(o=>o.crawlDepth).filter(o=>o!=null),u=m.length?Math.max(...m):0,t=new Set(r.filter(o=>o.extractionStatus!=="failed").map(o=>o.url)),f=[...n.values()].filter(o=>o.orphan&&t.has(o.url)).length,d=[...n.values()].sort((o,b)=>b.inlinks-o.inlinks).slice(0,5);return[`# SEO Crawl Report: ${s}`,`**${e} pages attempted** \xB7 ${a} extracted \xB7 ${i} failed \xB7 max crawl depth ${u} \xB7 ${f} orphan page(s)`,`
+import{b as y}from"./chunk-B3JRL45T.js";var $=100*1024,L=new Set(["webp","avif","svg","svg+xml"]);function x(s){return s==null?null:s<1024?`${s} B`:s<1024*1024?`${(s/1024).toFixed(1)} KB`:`${(s/(1024*1024)).toFixed(2)} MB`}var R=s=>s.replace(/&amp;/g,"&").replace(/&#0?38;/g,"&").replace(/&#x26;/gi,"&").trim(),M=s=>s.replace(/^https?:\/\//i,"//").replace(/\/$/,""),E=(s,r)=>{if(s){let c=s.split(";")[0].trim().toLowerCase();if(c.startsWith("image/"))return c.replace("image/","")}return(r.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1]||"unknown").toLowerCase()};function T(s){let r=new Map;for(let c of s)for(let n of c.imageLinks||[]){let e=R(n);if(!/^https?:\/\//i.test(e)||/[{}]|%7[bd]/i.test(e))continue;let a=M(e),o=r.get(a);(!o||/^https:/i.test(e)&&/^http:/i.test(o))&&r.set(a,e)}return[...r.values()]}async function D(s,r){let c=new AbortController,n=setTimeout(()=>c.abort(),r),e=o=>({len:o.headers.get("content-length"),cr:o.headers.get("content-range"),ct:o.headers.get("content-type"),status:o.status}),a=async o=>{let g=s;for(let p=0;p<=5;p++){let m=await y(g,{field:"image URL"});if(m.error||!m.parsed)throw new Error(m.error??"Image URL was rejected");let u=await fetch(m.parsed.href,{method:o,...o==="GET"?{headers:{Range:"bytes=0-0"}}:{},redirect:"manual",signal:c.signal});if(u.status>=300&&u.status<400){let t=u.headers.get("location");if(!t)throw new Error(`HTTP ${u.status} redirect did not include Location`);g=new URL(t,m.parsed.href).href;continue}return u}throw new Error("Image request exceeded five redirects")};try{let o=null,g=null,p=null;try{let m=e(await a("HEAD"));p=m.status,g=m.ct,m.len&&(o=Number(m.len))}catch{}if(o==null){let m=await a("GET"),u=e(m);if(p=p??u.status,g=g??u.ct,u.cr&&u.cr.includes("/")){let t=u.cr.split("/")[1];t&&t!=="*"&&(o=Number(t))}else u.len&&(o=Number(u.len));try{await m.body?.cancel()}catch{}}return{url:s,status:p,bytes:Number.isFinite(o)?o:null,contentType:g}}catch(o){return{url:s,status:null,bytes:null,contentType:null,error:String(o.message||o)}}finally{clearTimeout(n)}}async function v(s,r,c){let n=new Array(s.length),e=0;return await Promise.all(Array.from({length:Math.min(r,s.length)},async()=>{for(;e<s.length;){let a=e++;n[a]=await c(s[a])}})),n}async function S(s,r={}){return A(T(s),r)}async function A(s,r={}){let c=r.concurrency??12,n=r.timeoutMs??12e3,e=r.max??5e3,a=T([{imageLinks:s}]).slice(0,e),g=(await v(a,c,t=>D(t,n))).map(t=>{let f=E(t.contentType,t.url);return{...t,size:x(t.bytes),format:f,over100kb:t.bytes!=null&&t.bytes>$,legacyFormat:f!=="unknown"&&!L.has(f)}}),p=g.filter(t=>t.bytes!=null),m=p.reduce((t,f)=>t+f.bytes,0),u={};for(let t of g)u[t.format]=(u[t.format]||0)+1;return{rows:g.sort((t,f)=>(f.bytes??0)-(t.bytes??0)),summary:{unique:g.length,sized:p.length,totalBytes:m,totalSize:x(m)??"0 B",avgSize:x(Math.round(m/(p.length||1)))??"0 B",over100kb:g.filter(t=>t.over100kb).length,legacyFormat:g.filter(t=>t.legacyFormat).length,formatCounts:u,...r.sampleTruncated?{sampleTruncated:!0,sampleCap:e}:{}}}}function _(s){let r=s.summary,c=Object.entries(r.formatCounts).sort((a,o)=>o[1]-a[1]).map(([a,o])=>`${a}:${o}`).join("  "),n=s.rows.filter(a=>a.over100kb).slice(0,15),e=["## Images",`**${r.unique} unique images** \xB7 ${r.totalSize} total \xB7 ${r.avgSize} avg \xB7 ${r.over100kb} over 100 KB \xB7 ${r.legacyFormat} legacy format`,`Formats: ${c}`];if(n.length){e.push("","| Size | Format | URL |","|------|--------|-----|");for(let a of n)e.push(`| ${a.size} | ${a.format} | ${a.url} |`)}return e.join(`
+`)}function I(s){let r=s.replace(/^www\./,"").toLowerCase(),c=r.split(".");return c.length<=2?r:c.slice(-2).join(".")}function U(s,r,c){let n="";try{n=I(new URL(c).hostname)}catch{n=""}let e=s.filter(l=>l.internal),a=r.length,o=r.filter(l=>l.orphan).length,g=e.filter(l=>l.targetStatus!=null&&l.targetStatus>=400).length,p=r.reduce((l,h)=>l+h.inlinks,0),m=r.reduce((l,h)=>l+h.outlinksInternal+h.outlinksExternal,0),u={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0};for(let l of r)l.inlinks===0?u.zero++:l.inlinks<=2?u.oneToTwo++:l.inlinks<=10?u.threeToTen++:u.elevenPlus++;let t=[...r].sort((l,h)=>h.inlinks-l.inlinks).slice(0,20).map(l=>({url:l.url,inlinks:l.inlinks,outlinksInternal:l.outlinksInternal,outlinksExternal:l.outlinksExternal})),f=new Map,d=0;for(let l of s){let h;try{h=I(new URL(l.to).hostname)}catch{continue}if(!h||h===n)continue;d++;let k=f.get(h)??{links:0,nofollow:0,pages:new Set};k.links++,l.nofollow&&k.nofollow++,k.pages.add(l.from),f.set(h,k)}let i=[...f.entries()].map(([l,h])=>({domain:l,links:h.links,nofollow:h.nofollow,pages:h.pages.size})).sort((l,h)=>h.links-l.links),b=l=>Math.round(l*10)/10;return{summary:{internal:{totalLinks:e.length,pages:a,orphans:o,brokenInternal:g,avgInlinks:a?b(p/a):0,avgOutlinks:a?b(m/a):0,distribution:u,topByInlinks:t},external:{totalLinks:d,uniqueDomains:i.length,topDomains:i.slice(0,20)}},externalDomains:i}}function X(s){let r=s.summary;return["## Link analysis",`**Internal:** ${r.internal.totalLinks} links \xB7 ${r.internal.pages} pages \xB7 avg ${r.internal.avgInlinks} inlinks/page \xB7 ${r.internal.orphans} orphans \xB7 ${r.internal.brokenInternal} broken`,`**External:** ${r.external.totalLinks} links to ${r.external.uniqueDomains} domains`,"","### Inlink distribution",`- 0 inlinks (orphans): ${r.internal.distribution.zero}`,`- 1\u20132: ${r.internal.distribution.oneToTwo}`,`- 3\u201310: ${r.internal.distribution.threeToTen}`,`- 11+: ${r.internal.distribution.elevenPlus}`,"","### Top 20 internal pages by inlinks","| inlinks | out (int/ext) | URL |","|---|---|---|",...r.internal.topByInlinks.map(n=>`| ${n.inlinks} | ${n.outlinksInternal}/${n.outlinksExternal} | ${n.url} |`),"","### Top 20 external domains by links","| links | nofollow | from pages | domain |","|---|---|---|---|",...r.external.topDomains.map(n=>`| ${n.links} | ${n.nofollow} | ${n.pages} | ${n.domain} |`)].join(`
+`)}function w(s,r){let c=new Map;for(let n of s){let e=r(n);if(e==null||e==="")continue;let a=String(e);c.has(a)||c.set(a,[]),c.get(a).push(n.url)}return[...c.values()].filter(n=>n.length>1).flat()}function B(s,r,c){let n=t=>({count:t.length,urls:t.slice(0,200)}),e=t=>s.filter(t).map(f=>f.url),a=t=>t.extractionStatus!=="failed",o=t=>a(t)&&t.status===200,g=new Set(s.filter(a).map(t=>t.url)),p=t=>{try{return new URL(t).pathname}catch{return""}},m={"title.missing":n(e(t=>o(t)&&!t.title)),"title.duplicate":n(w(s.filter(o),t=>t.title)),"title.tooLong":n(e(t=>(t.titleLength??0)>60||(t.titlePixels??0)>561)),"title.tooShort":n(e(t=>t.title!=null&&(t.titleLength??0)<30)),"title.sameAsH1":n(e(t=>!!t.title&&!!t.h1&&t.title.trim()===t.h1.trim())),"meta.missing":n(e(t=>o(t)&&!t.metaDescription)),"meta.duplicate":n(w(s.filter(o),t=>t.metaDescription)),"meta.tooLong":n(e(t=>(t.metaDescLength??0)>155)),"meta.tooShort":n(e(t=>t.metaDescription!=null&&(t.metaDescLength??0)<70)),"h1.missing":n(e(t=>o(t)&&!t.h1)),"h1.multiple":n(e(t=>!!t.h1_2)),"h1.duplicate":n(w(s.filter(o),t=>t.h1)),"h1.tooLong":n(e(t=>(t.h1?.length??0)>70)),"h2.missing":n(e(t=>o(t)&&t.h2Count===0)),"crawl.extractionFailed":n(e(t=>!a(t))),"indexability.nonIndexable":n(e(t=>a(t)&&!t.indexable)),"indexability.noindex":n(e(t=>t.indexabilityReason==="noindex")),"canonical.missing":n(e(t=>o(t)&&!t.canonicalUrl)),"canonical.canonicalised":n(e(t=>t.indexabilityReason==="canonicalised")),"response.broken4xx":n(e(t=>(t.status??0)>=400&&(t.status??0)<500)),"response.error5xx":n(e(t=>(t.status??0)>=500)),"response.redirect3xx":n(e(t=>(t.status??0)>=300&&(t.status??0)<400)),"response.noResponse":n(e(t=>t.status===null)),"content.thin":n(e(t=>o(t)&&t.wordCount>0&&t.wordCount<200)),"content.exactDuplicate":n(w(s.filter(t=>o(t)&&t.contentHash),t=>t.contentHash)),"images.missingAlt":n(e(t=>a(t)&&t.imagesMissingAlt>0)),"schema.missing":n(e(t=>o(t)&&t.schemaTypes.length===0)),"url.tooLong":n(e(t=>t.url.length>115)),"url.uppercase":n(e(t=>/[A-Z]/.test(p(t.url)))),"url.underscores":n(e(t=>p(t.url).includes("_"))),"links.orphan":n([...r.values()].filter(t=>t.orphan&&g.has(t.url)).map(t=>t.url))},u=c;if(!u){let t=d=>{try{let i=new URL(d);return i.hash="",(i.origin+i.pathname).replace(/\/+$/,"")+i.search}catch{return d.replace(/#.*$/,"").replace(/\/+$/,"")}},f=new Map(s.map(d=>[t(d.url),d.status]));u=new Set;for(let d of s)for(let i of d.outlinks??[]){if(!i.internal)continue;let b=f.get(t(i.href));b!=null&&b>=400&&u.add(d.url)}}return m["links.brokenInternal"]=n([...u]),m}function C(s,r,c,n){let e=r.length,a=r.filter(i=>i.extractionStatus!=="failed").length,o=e-a,g=i=>i.startsWith("response.broken")||i.startsWith("response.error")||i.startsWith("links.broken")?"\u{1F534}":i.startsWith("title.missing")||i.startsWith("h1.missing")||i.startsWith("indexability")||i.startsWith("canonical.missing")?"\u{1F7E0}":"\u{1F7E1}",p=Object.entries(c).filter(([,i])=>i.count>0).sort((i,b)=>b[1].count-i[1].count).map(([i,b])=>`| ${g(i)} | \`${i}\` | ${b.count} | ${b.urls.slice(0,3).join(" \xB7 ")}${b.urls.length>3?" \u2026":""} |`),m=[...n.values()].map(i=>i.crawlDepth).filter(i=>i!=null),u=m.length?Math.max(...m):0,t=new Set(r.filter(i=>i.extractionStatus!=="failed").map(i=>i.url)),f=[...n.values()].filter(i=>i.orphan&&t.has(i.url)).length,d=[...n.values()].sort((i,b)=>b.inlinks-i.inlinks).slice(0,5);return[`# SEO Crawl Report: ${s}`,`**${e} pages attempted** \xB7 ${a} extracted \xB7 ${o} failed \xB7 max crawl depth ${u} \xB7 ${f} orphan page(s)`,`
 ## Issues
 | | Issue | Count | Examples |
 |---|-------|-------|----------|
 ${p.join(`
 `)||"| \u2705 | none | 0 | \u2014 |"}`,`
 ## Most-linked pages
-${d.map(o=>`- ${o.inlinks} inlinks \xB7 depth ${o.crawlDepth??"\u2014"} \xB7 ${o.url}`).join(`
+${d.map(i=>`- ${i.inlinks} inlinks \xB7 depth ${i.crawlDepth??"\u2014"} \xB7 ${i.url}`).join(`
 `)}`,`
 _Thresholds: title 60ch/561px, meta 155ch, H1 70ch, thin <200 words, URL 115ch. Pixel widths are estimates._`].join(`
 `)}export{S as a,A as b,_ as c,U as d,X as e,B as f,C as g};

package/dist/chunk-DGAW7FD7.js

@@ -0,0 +1,12 @@
+import{a as D,e as A,f as T,g as C,h as w,i as p}from"./chunk-R5LJL6IR.js";import{B as k,C as x,D as v,b as E,g,x as I}from"./chunk-6SNOUSSZ.js";import{a as P,b as S}from"./chunk-ZPQNY5TI.js";import{a as _}from"./chunk-WNUXMPVX.js";import{randomUUID as O}from"crypto";var L=class{constructor(t,r,i){this.userId=t;this.sourceSurface=r;this.fence=i;if(i&&i.userId!==t)throw Error("Billing scope principal differs from execution claim")}userId;sourceSurface;fence;identity=`debit:${O()}`;charges=new Map;pending=new Map;compensating=new Set;closing=!1;async debit(t,r,i,c,a){if(this.closing||this.compensating.has(t))throw Error("Billing scope is compensating; new charges are forbidden");let s=this.charges.get(t);if(s?.outcome==="pending")throw Error("Named debit outcome is pending; reconcile before retrying");if(s?.outcome==="funded"&&s.amountMc!==r)throw Error("An applied debit amount cannot change");let o={key:s?.key??`${this.identity}:${t}`,amountMc:r,operation:i,description:c,refundOperation:a,outcome:"pending"};if(s&&(s.operation!==i||s.description!==c||s.refundOperation!==a))throw Error("Named charge identity cannot change");this.charges.set(t,o);let u=(async()=>{if(this.fence){let f=await A(this.fence,{label:t,amountMc:r,operation:i,description:c,refundOperation:a,sourceSurface:this.sourceSurface});if(o.key=f.key,this.closing||this.compensating.has(t))throw Error("Billing scope began compensating while planning")}return this.fence?w(this.userId,r,i,c,o.key,this.fence):w(this.userId,r,i,c,o.key)})();this.pending.set(t,u);try{let f=await u;return o.outcome=f.ok?"funded":"unfunded",{...f,billingKey:o.key}}finally{this.pending.delete(t)}}async compensate(t){this.compensating.add(t),await this.pending.get(t)?.catch(()=>{});try{this.fence&&await this.loadDurablePlans();let r=this.charges.get(t);if(!r)return"not_charged";let i=await C(this.userId,r.key);return i?i.amountMc!==r.amountMc||i.operation!==r.operation||i.description!==r.description?"refund_pending":i.status==="claimed"?this.fence?.mode==="compensate"||r.outcome==="unfunded"?"not_charged":"refund_pending":i.status!=="applied"?"refund_pending":(this.fence?await p(this.userId,r.key,0,r.refundOperation,`${this.sourceSurface} failed`,this.sourceSurface,this.fence):await p(this.userId,r.key,0,r.refundOperation,`${this.sourceSurface} failed`,this.sourceSurface)).final_amount_mc===0?"refunded":"refund_pending":this.fence?.mode==="compensate"||r.outcome==="unfunded"?"not_charged":"refund_pending"}catch{return"refund_pending"}}async compensateAll(){if(this.closing=!0,await Promise.allSettled([...this.pending.values()]),this.fence)try{await this.loadDurablePlans()}catch{return"refund_pending"}let t=await Promise.all([...this.charges.keys()].map(r=>this.compensate(r)));return t.includes("refund_pending")?"refund_pending":t.includes("refunded")?"refunded":"not_charged"}async loadDurablePlans(){if(this.fence)for(let t of await T(this.fence)){let r=this.charges.get(t.label);if(t.sourceSurface!==this.sourceSurface)throw Error("Persisted billing source differs from scope");this.charges.set(t.label,{key:t.key,amountMc:t.amountMc,operation:t.operation,description:t.description,refundOperation:t.refundOperation,outcome:r?.key===t.key?r.outcome:"pending"})}}};async function H(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0||!Number.isSafeInteger(e.refundMc)||e.refundMc<0)throw Error("invalid_legacy_refund");await _().batch([e.admission,{sql:`INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
+            SELECT ?, ?, ?, ?, ? WHERE changes() = 1 AND ? > 0 AND datetime(?) > datetime('now')`,args:[e.userId,e.refundMc,e.refundMc,e.operation,e.expiresAt,e.refundMc,e.expiresAt]},{sql:"UPDATE users SET balance_mc = balance_mc + ? WHERE id = ? AND ? > 0 AND changes() = 1",args:[e.refundMc,e.userId,e.refundMc]},{sql:`INSERT INTO ledger (user_id, amount_mc, operation, description)
+            SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,args:[e.userId,e.refundMc,e.operation,e.description,e.refundMc]},e.acknowledgment],"write")}function J(e){return e.status==="complete"||e.status==="partial"||e.status==="failed"}function y(e,t,r){return e.userId===t&&(e.executionGeneration??0)===r&&J(e)}function W(e){let t=e.options.xraySetupReceipt;return e.options.heldMc===0&&e.options.debitKey==null&&t?.billingClass===E&&t.consumesMcpScraperCredits===!1&&t.heldMc===0&&t.billedMc===0}function b(e,t){if(!e||!y(e,t.userId,t.generation)||JSON.stringify(e.settlementPlan)!==JSON.stringify(t))return null;let r=e.settlementReceipt;return!r||r.jobId!==t.jobId||r.userId!==t.userId||r.generation!==t.generation||e.billedMc!==r.netChargeMc?null:r}async function Q(e,t,r,i,c,a=0){let s=await g(e);if(!s||!y(s,t,a))throw Error("extract_settlement_identity_or_terminal_conflict");if(s.billedMc!=null)return s.settlementReceipt??null;let o=Number(s.options.heldMc??0);if(![o,r,i].every(d=>Number.isSafeInteger(d)&&d>=0)||r+i!==o)throw Error("extract_settlement_amount_conflict");let u=String(s.options.billingClass??"credits");if(u===E&&!W(s))throw Error("xray_extract_zero_charge_finalization_rejected");let f={version:1,jobId:e,userId:t,generation:a,status:s.status,heldMc:o,debitKey:typeof s.options.debitKey=="string"?s.options.debitKey:null,billingClass:u,netChargeMc:i,refundMc:r,reference:c},l;try{l=await k(f)}catch(d){if(l=await g(e).catch(()=>null),!l?.settlementPlan||!y(l,t,a))throw d}if(!l||!y(l,t,a)||!l.settlementPlan)throw Error("extract_settlement_admission_conflict");let n=l.settlementPlan;if(n.jobId!==e||n.userId!==t||n.generation!==a||n.status!==l.status||n.heldMc!==o||n.debitKey!==f.debitKey||n.billingClass!==u)throw Error("extract_settlement_plan_conflict");let N=b(l,n);if(N)return N;let m={version:1,jobId:e,userId:t,generation:a,netChargeMc:n.netChargeMc,refundMc:0,expiredRefundMc:0};try{if(u===E)await x(n,m);else if(n.debitKey){let d=await p(t,n.debitKey,n.netChargeMc,S.EXTRACT_SITE_REFUND,n.reference,"extract_site");m.netChargeMc=d.final_amount_mc,m.refundMc=d.refund_mc,m.expiredRefundMc=d.expired_refund_mc??0,await x(n,m)}else{let h=await _().execute({sql:"SELECT datetime(created_at, ?) AS expires_at FROM site_extract_jobs WHERE id = ?",args:[D,e]}),R=[e,t,a,n.status,JSON.stringify(n)];await H({userId:t,refundMc:n.refundMc,expiresAt:String(h.rows[0]?.expires_at),operation:S.EXTRACT_SITE_REFUND,description:n.reference,admission:{sql:`UPDATE site_extract_jobs SET billed_mc = -1
+          WHERE id = ? AND user_id = ? AND execution_generation = ? AND status = ?
+            AND settlement_plan_json = ? AND billed_mc IS NULL`,args:R},acknowledgment:{sql:`UPDATE site_extract_jobs SET billed_mc = ?,
+          settlement_receipt_json = json_set(?, '$.refundMc', CASE WHEN changes() = 1 THEN ? ELSE 0 END,
+            '$.expiredRefundMc', CASE WHEN changes() = 1 THEN 0 ELSE ? END),
+          public_error_json = CASE WHEN json_valid(public_error_json) THEN json_set(public_error_json, '$.charge_status',
+            CASE WHEN ? > 0 THEN 'charged' WHEN changes() = 1 AND ? > 0 THEN 'refunded' ELSE 'not_charged' END) ELSE public_error_json END,
+          updated_at = datetime('now')
+          WHERE id = ? AND user_id = ? AND execution_generation = ? AND status = ?
+            AND settlement_plan_json = ? AND billed_mc = -1`,args:[n.netChargeMc,JSON.stringify(m),n.refundMc,n.refundMc,n.netChargeMc,n.refundMc,...R]}})}}catch(d){let h=b(await g(e).catch(()=>null),n);if(h)return h;throw d}let M=b(await g(e),n);if(!M)throw Error("extract_settlement_acknowledgment_unknown");return M}async function V(e){if(e.userId==null||e.billedMc!=null)return e.settlementReceipt??null;if(!J(e))throw Error("extract_settlement_requires_terminal_job");let t=Number(e.options.heldMc??0),r=await I(e.id),i=Math.min(r*P.page_scrape,t),c="site";try{c=new URL(e.startUrl).hostname}catch{}return await v(e.id,e.userId,t-i,i,c,e.executionGeneration??0)}export{L as a,Q as b,V as c};

package/dist/chunk-FRXKMOXI.js

@@ -0,0 +1,41 @@
+import{a as ot}from"./chunk-U4WIJZCX.js";import{F as fe,a as Qe,c as Ve}from"./chunk-6SNOUSSZ.js";import{g as Ke,h as Ze}from"./chunk-KXJCGV3H.js";import{c as st}from"./chunk-JMZIDHYQ.js";import{c as tt,e as nt,f as rt,g as it}from"./chunk-CNSVZTTT.js";import{c as et}from"./chunk-5BKSKLHX.js";import{h as Ge}from"./chunk-C7MGJUTU.js";import{a as se}from"./chunk-WNUXMPVX.js";import{createWriteStream as ae,mkdtempSync as Yt,mkdirSync as le,readFileSync as Wt,readdirSync as Gt,rmSync as Kt,statSync as Zt,writeFileSync as y}from"fs";import{createHash as W}from"crypto";import{join as l}from"path";import{tmpdir as Qt}from"os";import{createGzip as Vt}from"zlib";import{once as G}from"events";import{finished as ue}from"stream/promises";import{ZipFile as en}from"yazl";import tn from"p-limit";var vt=new Set(["browser_navigation_blocked","browser_navigation_failed","browser_timeout","browser_session_failed","browser_result_missing"]);function Mt(e){return e instanceof Error?e.code:void 0}function At(e){return e instanceof Error?e.message:String(e)}function at(e){let r=e?.match(/^browser_http_(\d+)$/);return r?Number(r[1]):null}function It(e){return e===404?"page_not_found":e===403?"page_forbidden":e===429?"page_rate_limited":e>=500&&e<=599?"page_server_error":"page_http_error"}function Rt(e){let r=At(e),i=Mt(e);if(ot(e,r))return{errorCode:"vendor_unavailable",httpStatus:null,retryable:!0};let a=at(i)??at(r.match(/^(browser_http_\d+)/)?.[1]);return a!=null?{errorCode:It(a),httpStatus:a,retryable:a===429||a>=500}:i==="browser_challenge_unresolved"||/^browser_challenge_unresolved/.test(r)?{errorCode:"bot_check_unresolved",httpStatus:null,retryable:!0}:i==="browser_response_too_large"||/exceeds \d+ byte limit/i.test(r)?{errorCode:"page_too_large",httpStatus:null,retryable:!1}:/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(r)?{errorCode:"browser_session_interrupted",httpStatus:null,retryable:!0}:vt.has(i??"")||/timeout|navigation_failed|ERR_/i.test(r)?{errorCode:"page_unreachable",httpStatus:null,retryable:!0}:{errorCode:"extraction_failed",httpStatus:null,retryable:!1}}function lt(e,r){switch(e){case"page_not_found":return`The page could not be found (HTTP ${r??404}). It may have been moved or deleted.`;case"page_forbidden":return`The site refused the request (HTTP ${r??403}).`;case"page_rate_limited":return"The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";case"page_server_error":return`The page's own server returned an error (HTTP ${r}). This is on the target site's side, not something a retry here can fix.`;case"page_http_error":return`The page returned an unexpected HTTP ${r} response.`;case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. A retry uses a fresh browser session.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"extraction_failed":return"The page could not be extracted. Check retryable before attempting it again; contact support if this persists for the same URL."}}function on(e,r={}){let i=lt(e.errorCode,e.httpStatus);return{...Ge({errorCode:e.errorCode,retryable:e.retryable,chargeStatus:r.chargeStatus,details:{...r.details??{},...e.httpStatus!=null?{http_status:e.httpStatus}:{}}}),message:i,error:i}}function an(e){return e.httpStatus===404?404:e.httpStatus===403?403:e.httpStatus===429?429:e.httpStatus!=null&&e.httpStatus>=500&&e.httpStatus<=599?500:e.httpStatus!=null?502:e.errorCode==="vendor_unavailable"?503:502}var Dt="wayback_capture_missing";function Me(e,r){if(e==null)return{failureCode:null,failureReason:null};if(e===Dt)return{failureCode:e,failureReason:r??null};let i=new Error(r??e);i.code=e;let a=Rt(i);return{failureCode:a.errorCode,failureReason:lt(a.errorCode,a.httpStatus)}}import{createHash as Ie}from"crypto";import{neon as $t}from"@neondatabase/serverless";import{waitUntil as Ut}from"@vercel/functions";var jt="jina",ye=null,ut=!1;function be(){return(process.env.JINA_EMBED_MODEL??"jina-embeddings-v5-omni-small").trim()}function _e(){return Number((process.env.JINA_EMBED_DIM??"1024").trim())}function Ae(){return!!(process.env.JINA_API_KEY?.trim()&&process.env.MEMORY_DATABASE_URL?.trim())}function oe(){if(ye)return ye;let e=process.env.MEMORY_DATABASE_URL?.trim();if(!e)throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");return ye=$t(e),ye}async function mt(){if(ut)return;let e=_e();await oe().query("CREATE EXTENSION IF NOT EXISTS vector"),await oe().query(`
+    CREATE TABLE IF NOT EXISTS commons_index_vectors (
+      document_id   TEXT PRIMARY KEY,
+      entity_id     TEXT NOT NULL,
+      document_type TEXT NOT NULL,
+      title         TEXT NOT NULL,
+      embedding     vector(${e}) NOT NULL,
+      model         TEXT NOT NULL,
+      updated_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+    )
+  `),await oe().query("CREATE INDEX IF NOT EXISTS commons_index_vectors_entity ON commons_index_vectors(entity_id)"),ut=!0}async function we(e){let r=process.env.JINA_API_KEY?.trim();if(!r)throw new Error("JINA_API_KEY is not set; Commons semantic search cannot embed.");if(!e.length)return[];let i=await fetch("https://api.jina.ai/v1/embeddings",{method:"POST",headers:{authorization:`Bearer ${r}`,"content-type":"application/json"},body:JSON.stringify({model:be(),dimensions:_e(),input:e.map(p=>({text:p.slice(0,8e3)}))}),signal:AbortSignal.timeout(6e4)});if(!i.ok)throw new Error(`Jina embedding request failed with HTTP ${i.status}: ${(await i.text()).slice(0,200)}`);let s=((await i.json()).data??[]).map(p=>p.embedding);if(s.length!==e.length)throw new Error(`Jina returned ${s.length} embeddings for ${e.length} inputs.`);return s}var Nt=60*6e4;async function Lt(e=50){if(!Ae())return{claimed:0,embedded:0,failed:0,remaining:0};await mt();let r=Math.max(1,Math.min(200,Math.floor(e))),a=(await se().execute({sql:`SELECT id, entity_id, document_type, title, text FROM commons_index_documents
+          WHERE embedding_status = 'queued' OR (embedding_status = 'failed' AND updated_at < ?)
+          ORDER BY updated_at ASC LIMIT ?`,args:[new Date(Date.now()-Nt).toISOString(),r]})).rows;if(!a.length)return{claimed:0,embedded:0,failed:0,remaining:await ct()};let s=0,p=0,M=be();try{let h=await we(a.map(S=>`${S.title}
+
+${S.text}`));for(let[S,_]of a.entries()){let C=`[${h[S].join(",")}]`;await oe().query(`INSERT INTO commons_index_vectors (document_id, entity_id, document_type, title, embedding, model, updated_at)
+         VALUES ($1, $2, $3, $4, $5::vector, $6, now())
+         ON CONFLICT (document_id) DO UPDATE SET entity_id = EXCLUDED.entity_id, document_type = EXCLUDED.document_type,
+           title = EXCLUDED.title, embedding = EXCLUDED.embedding, model = EXCLUDED.model, updated_at = now()`,[_.id,_.entity_id,_.document_type,_.title,C,M]),await se().execute({sql:`UPDATE commons_index_documents SET embedding_status = 'indexed', embedding_provider = ?, embedding_model = ?,
+              vector_ref = ?, indexed_at = ?, error = NULL WHERE id = ?`,args:[jt,M,_.id,new Date().toISOString(),_.id]}),s+=1}}catch(h){p=a.length-s;let S=(h instanceof Error?h.message:String(h)).slice(0,500);for(let _ of a.slice(s))await se().execute({sql:"UPDATE commons_index_documents SET embedding_status = 'failed', error = ?, updated_at = ? WHERE id = ?",args:[S,new Date().toISOString(),_.id]}).catch(()=>{})}return{claimed:a.length,embedded:s,failed:p,remaining:await ct()}}function dn(){Ae()&&Ut(Lt(10).catch(e=>{console.error("[commons/embeddings] immediate embed failed; the scheduled drain will retry",e instanceof Error?e.message:String(e))}))}async function ct(){let e=await se().execute("SELECT COUNT(*) AS n FROM commons_index_documents WHERE embedding_status IN ('queued', 'failed')");return Number(e.rows[0]?.n??0)}async function pn(e,r=40){let i=new Map;if(!Ae()||!e.trim())return i;await mt();let[a]=await we([e]);if(!a)return i;let s=await oe().query(`SELECT entity_id, MAX(1 - (embedding <=> $1::vector)) AS score
+     FROM commons_index_vectors GROUP BY entity_id ORDER BY score DESC LIMIT $2`,[`[${a.join(",")}]`,Math.max(1,Math.min(100,r))]);for(let p of s)i.set(String(p.entity_id),Number(p.score));return i}var Ot=500,Bt=.9,qt=1e4,Ft=5e4;function Se(e,r){let i=10**r;return Math.round(e*i)/i}function Jt(e,r){if(e.length===0||e.length!==r.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let i=0,a=0,s=0;for(let p=0;p<e.length;p++)i+=e[p]*r[p],a+=e[p]*e[p],s+=r[p]*r[p];return a===0||s===0?0:Math.max(-1,Math.min(1,i/(Math.sqrt(a)*Math.sqrt(s))))}function Ht(e){let r=Ie("sha256");for(let i of[...e].sort((a,s)=>a.url.localeCompare(s.url)))r.update(i.url),r.update("\0"),r.update(i.contentHash),r.update("\0");return r.digest("hex")}function dt(e){let r=e.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return r.length>=12?r:null}function zt(e){let r=e.map(h=>h.bodyMarkdown.split(/\n{2,}/).map(S=>S.trim()).filter(Boolean)),i=e.length>=4?Math.max(3,Math.ceil(e.length*.5)):null,a=new Map;for(let h of r){let S=new Set(h.map(dt).filter(_=>!!_));for(let _ of S)a.set(_,(a.get(_)??0)+1)}let s=new Set;if(i!=null)for(let[h,S]of a)S>=i&&s.add(h);let p=0;return{texts:e.map((h,S)=>{let _=[];for(let P of r[S]){let I=dt(P);if(I&&s.has(I)){p+=P.length;continue}_.push(P)}let C=_.join(`
+
+`).trim(),v=C.length>=60?C:h.bodyMarkdown;return`${h.title?.trim()||"Untitled page"}
+
+${v}`}),minimumPageCount:i,removedBlockSignatures:s.size,removedCharacters:p}}var Re=class{parent;constructor(r){this.parent=Array.from({length:r},(i,a)=>a)}find(r){let i=this.parent[r];return i!==r&&(this.parent[r]=this.find(i)),this.parent[r]}union(r,i){let a=this.find(r),s=this.find(i);a!==s&&(this.parent[s]=a)}};async function pt(e,r={}){let i=e.filter(o=>o.bodyMarkdown.trim().length>0).slice(0,Ot),a=Number.isFinite(r.threshold)?r.threshold:Bt,s=Number.isFinite(r.maxPairs)?r.maxPairs:qt,p=Math.max(0,Math.min(1,a)),M=Math.max(1,Math.min(Ft,Math.floor(s))),h=r.model??be(),S=r.dimensions??_e(),_=r.embedTexts??we,C=zt(i),v=[],P=new Map,I=[];for(let[o,f]of i.entries()){let w=f.contentHash||Ie("sha256").update(f.bodyMarkdown).digest("hex"),k=P.get(w);k==null&&(k=v.length,P.set(w,k),v.push(C.texts[o])),I.push(k)}let D=[];for(let o=0;o<v.length;o+=64)D.push(...await _(v.slice(o,o+64)));if(D.length!==v.length)throw new Error(`Embedding provider returned ${D.length} vectors for ${v.length} unique page bodies.`);let K=I.map(o=>D[o]),ce=new Re(i.length),L=[],O=[];for(let o=0;o<i.length;o++)for(let f=o+1;f<i.length;f++){let w=Jt(K[o],K[f]);L.push(w),!(w<p)&&(O.push({source:o,target:f,score:w}),ce.union(o,f))}O.sort((o,f)=>f.score-o.score||i[o.source].url.localeCompare(i[f.source].url)||i[o.target].url.localeCompare(i[f.target].url));let R=[...L].sort((o,f)=>o-f),U=o=>R.length?Se(R[Math.floor((R.length-1)*o)],6):null,j=new Map;for(let o=0;o<i.length;o++){let f=ce.find(o),w=j.get(f)??[];w.push(o),j.set(f,w)}let B=new Map,J=[...j.values()].filter(o=>o.length>1).sort((o,f)=>f.length-o.length||i[o[0]].url.localeCompare(i[f[0]].url)).map((o,f)=>{let w=`cluster-${String(f+1).padStart(3,"0")}`;for(let k of o)B.set(k,w);return{clusterId:w,pageCount:o.length,urls:o.map(k=>i[k].url).sort()}}),H=O.slice(0,M).map((o,f)=>{let w=i[o.source],k=i[o.target],q=Se(o.score,6);return{sourceUrl:w.url,sourceTitle:w.title,targetUrl:k.url,targetTitle:k.title,similarity:q,similarityPercent:Se(q*100,2),corpusPercentile:R.length<=1?100:Se((1-f/(R.length-1))*100,2),exactContentDuplicate:!!(w.contentHash&&w.contentHash===k.contentHash),sourceWordCount:w.wordCount,targetWordCount:k.wordCount,clusterId:B.get(o.source)??null}}),z=Ht(i),me=Ie("sha256").update(JSON.stringify({corpusSha256:z,model:h,dimensions:S,threshold:p,maxPairs:M,boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:C.minimumPageCount,removedBlockSignatures:C.removedBlockSignatures}})).digest("hex");return{generatedAt:new Date().toISOString(),provider:"jina",model:h,dimensions:S,threshold:p,requestedMaxPairs:M,comparedPages:i.length,possiblePairs:i.length*Math.max(0,i.length-1)/2,qualifyingPairs:O.length,returnedPairs:H.length,pairsTruncated:O.length>H.length,corpusSha256:z,analysisSha256:me,scoreDistribution:{min:U(0),p25:U(.25),median:U(.5),p75:U(.75),p90:U(.9),max:U(1)},boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:C.minimumPageCount,removedBlockSignatures:C.removedBlockSignatures,removedCharacters:C.removedCharacters},rows:H,clusters:J}}import{basename as Xt}from"path";function Y(e){return e.split("#")[0].replace(/\/$/,"")}function De(e){let r=e.replace(/^www\./,"").toLowerCase(),i=r.split(".");return i.length<=2?r:i.slice(-2).join(".")}function gt(e,r){try{let i=new URL(e);return Xt(i.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`image-${r}`}catch{return`image-${r}`}}function ht(){let e=new Map;return r=>{let i=r.replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)||"page",a=e.get(i)??0;return e.set(i,a+1),a?`${i}-${a}`:i}}function ft(e){let r=e==null?"":String(e);return/[",\r\n]/.test(r)?`"${r.replace(/"/g,'""')}"`:r}function yt(e){return{source_url:e.sourceUrl,source_title:e.sourceTitle,target_url:e.targetUrl,target_title:e.targetTitle,similarity:e.similarity,similarity_percent:e.similarityPercent,corpus_percentile:e.corpusPercentile,exact_content_duplicate:e.exactContentDuplicate,source_word_count:e.sourceWordCount,target_word_count:e.targetWordCount,cluster_id:e.clusterId}}var $e=20,Ue=500,nn=8,bt=10*1024*1024,_t=100*1024*1024,je=25e4;async function Fn(e,r={}){let i=e.options.downloadImages===!0;if(e.userId==null)throw new Error("site extract artifact owner is missing");let a=Qe(String(e.userId)),s=Yt(l(Qt(),"extract-bundle-"));le(s,{recursive:!0}),i&&le(l(s,"images"),{recursive:!0});let p=e.options.captureRenderedDom===!0,M=e.options.semanticSimilarity===!0;p&&le(l(s,"rendered-dom"),{recursive:!0});try{let h=[],S=[],_=[],C=ht(),v=[],P=e.options.waybackTimeline,I=new Map;for await(let n of fe(e.id))for(let t of n){I.set(Y(t.url),t.status),h.push({url:t.url,status:t.status,title:t.title,titleLength:t.titleLength,titlePixels:t.titlePixels,metaDescription:t.metaDescription,metaDescLength:t.metaDescLength,h1:t.h1,h1_2:t.h1_2,h2Count:t.h2Count,extractionStatus:t.extractionStatus,indexable:t.indexable,indexabilityReason:t.indexabilityReason,canonicalUrl:t.canonicalUrl?t.url:null,wordCount:t.wordCount,contentHash:t.contentHash,imagesMissingAlt:t.imagesMissingAlt,schemaTypes:t.schemaTypes.length>0?["present"]:[],outlinks:[]});let u=t.pageId??W("sha256").update(t.archivedUrl??t.url).digest("hex"),E=l(s,"pages",u);le(E,{recursive:!0});let d=t.contentRef?await a(t.contentRef):null,g=d?.html??null,c=d?.markdown??t.bodyMarkdown??null;if(M&&t.extractionStatus==="successful"&&c?.trim()&&S.push({url:t.url,title:t.title,bodyMarkdown:c,contentHash:t.contentHash,wordCount:t.wordCount}),p&&g!=null){let We=`rendered-dom/${C(t.url)}.html.txt`;y(l(s,We),g),_.push({url:t.url,path:We,bytes:Buffer.byteLength(g),truncated:t.renderedDomTruncated===!0,sanitized:t.renderedDomSanitized===!0})}let m=g==null?null:`pages/${u}/page.html`,b=c==null?null:`pages/${u}/page.md`,x=c==null||!t.archiveRequestedMonth?null:`pages/${t.archiveRequestedMonth}/${u}.md`;g!=null&&m&&y(l(s,m),g),c!=null&&b&&y(l(s,b),c),c!=null&&x&&(le(l(s,"pages",t.archiveRequestedMonth),{recursive:!0}),y(l(s,x),c));let{bodyMarkdown:A,contentRef:ne,discoveryLinks:re,...Ct}=t,ie={version:"site-export-page.v1",...Ct,pageId:u,content:{htmlPath:m,markdownPath:b,htmlBytes:g==null?0:Buffer.byteLength(g),markdownBytes:c==null?0:Buffer.byteLength(c),htmlSha256:g==null?null:W("sha256").update(g).digest("hex"),markdownSha256:c==null?null:W("sha256").update(c).digest("hex")}},Xe=JSON.stringify(ie,null,2),Ye=`pages/${u}/page.json`;y(l(s,Ye),Xe),v.push({pageId:u,url:t.archivedUrl??t.url,jsonPath:Ye,htmlPath:m,markdownPath:b,legacyMarkdownPath:x,jsonSha256:W("sha256").update(Xe).digest("hex"),htmlSha256:ie.content.htmlSha256,markdownSha256:ie.content.markdownSha256,htmlBytes:ie.content.htmlBytes,markdownBytes:ie.content.markdownBytes})}let D=Vt(),K=ae(l(s,"links.jsonl.gz"));D.pipe(K);let ce=async n=>{D.write(n)||await G(D,"drain")},L=ae(l(s,"schema.jsonl")),O=async n=>{L.write(n)||await G(L,"drain")},R=ae(l(s,"failures.jsonl")),U=async n=>{R.write(n)||await G(R,"drain")},j=new Map,B=new Map,J=new Map,H=new Map,z=new Map,me=new Set,o=0,f=0,w=0,k=0,q=0,xe=!1,Ee="";try{Ee=De(new URL(e.startUrl).hostname)}catch{Ee=""}let wt=tn(nn),Ne=[],Le=[],Oe=[],Z=0,de=0,ke=0,St=n=>ke+n>_t?!1:(ke+=n,!0);for await(let n of fe(e.id))for(let t of n){if(t.schema?.length&&await O(JSON.stringify({url:t.url,schema:t.schema})+`
+`),t.extractionStatus==="failed"){let{failureCode:d,failureReason:g}=Me(t.failureCode,t.failureReason);await U(JSON.stringify({url:t.url,status:t.status,via:t.via,failureCode:d,failureReason:g,fetchAttempts:t.fetchAttempts??[]})+`
+`)}if(t.imageLinks?.length)for(let[d,g]of t.imageLinks.entries()){let c=W("sha256").update(`${t.url}\0${g}`).digest("hex"),m={imageId:c,sourcePage:t.url,sourceUrl:g,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Le.push(m),!i)continue;if(d>=$e){m.status="skipped_page_limit",m.failureCode="page_image_limit",m.failureReason=`Only the first ${$e} image candidates are downloaded per page.`;continue}if(Z>=Ue){m.status="skipped_site_limit",m.failureCode="site_image_limit",m.failureReason=`Only the first ${Ue} image candidates are downloaded per site.`;continue}Z++;let b=`${c}-${gt(g,d)}`,x=l(s,"images");Ne.push(wt(()=>st(g,x,b,{expectedType:"image",maxBytes:bt,consumeBytes:St})).then(async A=>{let ne=Wt(A.savedPath),re=await Ze({ownerId:String(e.userId),jobId:e.id,imageId:c,filename:b,contentType:A.mimeType??"application/octet-stream",content:ne,sourceUrl:g,sourcePage:t.url});Oe.push(re),m.status="downloaded",m.path=`images/${b}`,m.artifactId=re.key,m.mimeType=A.mimeType,m.bytes=A.sizeBytes,m.sha256=W("sha256").update(ne).digest("hex")},A=>{de++,m.status="failed",m.failureCode="image_download_failed",m.failureReason=(A instanceof Error?A.message:String(A)).replace(/\s+/g," ").slice(0,500)}))}let u=Y(t.url),E={int:0,ext:0};for(let d of t.outlinks??[]){let g=(d.rel??"").split(/\s+/).includes("nofollow"),c=Y(d.href),m=I.get(c)??null;if(await ce(JSON.stringify({from:t.url,to:d.href,anchor:d.anchor,rel:d.rel,internal:d.internal,nofollow:g,targetStatus:m})+`
+`),k++,q>=je){xe=!0;continue}if(q++,d.internal){if(E.int++,o++,j.has(c)||j.set(c,new Set),j.get(c).add(u),d.anchor){B.has(c)||B.set(c,new Map);let b=B.get(c);b.set(d.anchor,(b.get(d.anchor)??0)+1)}!g&&(I.get(c)===200||!I.has(c))&&(J.has(u)||J.set(u,new Set),J.get(u).add(c)),m!=null&&m>=400&&(w++,me.add(t.url))}else{E.ext++;let b="";try{b=De(new URL(d.href).hostname)}catch{continue}if(!b||b===Ee)continue;f++;let x=z.get(b)??{links:0,nofollow:0,pages:new Set};x.links++,g&&x.nofollow++,x.pages.add(t.url),z.set(b,x)}}H.set(t.url,E)}let xt=Promise.all([ue(K),ue(L),ue(R)]);D.end(),L.end(),R.end(),await xt,await Promise.all(Ne);let X=new Map,Pe=Y(e.startUrl);X.set(Pe,0);let Te=[Pe];for(;Te.length>0;){let n=Te.shift(),t=X.get(n);for(let u of J.get(n)??[])X.has(u)||(X.set(u,t+1),Te.push(u))}let Q=new Map;for(let n of h){let t=Y(n.url),u=j.get(t)??new Set,E=B.get(t),d=E?[...E.entries()].sort((c,m)=>m[1]-c[1]).slice(0,3).map(c=>c[0]):[],g=H.get(n.url)??{int:0,ext:0};Q.set(n.url,{url:n.url,inlinks:u.size,uniqueInlinks:u.size,outlinksInternal:g.int,outlinksExternal:g.ext,crawlDepth:X.has(t)?X.get(t):null,orphan:u.size===0&&t!==Pe,topAnchors:d})}let Be=rt(h,Q,me),V=it(e.startUrl,h,Be,Q),ee=Ve(e),F={jobId:e.id,startUrl:e.startUrl,status:e.successfulUrls===0?"failed":e.failedUrls>0||e.remainingUrls>0||ee.creditTruncated?"partial":"complete",...ee,discovered:e.totalUrls,attempted:e.attemptedUrls,successful:e.successfulUrls,failed:e.failedUrls,remaining:e.remainingUrls,linkAnalysis:{truncated:xe,analyzedEdges:q,streamedEdges:k,edgeCap:je},generatedAt:new Date().toISOString()};V=["## Crawl delivery",`- Discovered: ${F.discovered}`,`- Attempted: ${F.attempted}`,`- Successful: ${F.successful}`,`- Failed: ${F.failed}`,`- Remaining: ${F.remaining}`,`- Requested page cap: ${ee.requestedMaxPages}`,`- Funded page cap: ${ee.effectiveMaxPages}`,`- Credit-truncated: ${ee.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${xe?` (capped at ${je})`:""}`,"",V].join(`
+`),r.imageAudit&&(V+=`
+
+${tt(r.imageAudit)}`);let pe=[...Q.values()],qe=n=>Math.round(n*10)/10,te={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},Fe=0,Je=0;for(let n of pe)Fe+=n.inlinks,Je+=n.outlinksInternal+n.outlinksExternal,n.inlinks===0?te.zero++:n.inlinks<=2?te.oneToTwo++:n.inlinks<=10?te.threeToTen++:te.elevenPlus++;let Ce=[...z.entries()].map(([n,t])=>({domain:n,links:t.links,nofollow:t.nofollow,pages:t.pages.size})).sort((n,t)=>t.links-n.links),ve={summary:{internal:{totalLinks:o,pages:h.length,orphans:pe.filter(n=>n.orphan).length,brokenInternal:w,avgInlinks:h.length?qe(Fe/h.length):0,avgOutlinks:h.length?qe(Je/h.length):0,distribution:te,topByInlinks:[...pe].sort((n,t)=>t.inlinks-n.inlinks).slice(0,20).map(n=>({url:n.url,inlinks:n.inlinks,outlinksInternal:n.outlinksInternal,outlinksExternal:n.outlinksExternal}))},external:{totalLinks:f,uniqueDomains:Ce.length,topDomains:Ce.slice(0,20)}},externalDomains:Ce},ge=ae(l(s,"pages.jsonl")),Et=new Map(v.map(n=>[n.pageId,n])),$=P?ae(l(s,"capture-matrix.jsonl")):null,He=new Set,ze=(n,t)=>{let u=t;try{u=new URL(t).href}catch{}return`${n??""}\0${u}`};for await(let n of fe(e.id))for(let t of n){let u=t.extractionStatus==="failed"?Me(t.failureCode,t.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:E,schema:d,outlinks:g,contentRef:c,...m}=t,b=Q.get(t.url),x=t.pageId?Et.get(t.pageId):void 0,A={...m,...t.extractionStatus==="failed"?u:{},representations:x?{jsonPath:x.jsonPath,htmlPath:x.htmlPath,markdownPath:x.markdownPath,jsonSha256:x.jsonSha256,htmlSha256:x.htmlSha256,markdownSha256:x.markdownSha256,htmlBytes:x.htmlBytes,markdownBytes:x.markdownBytes}:null,inlinks:b?.inlinks??0,crawlDepth:b?.crawlDepth??null,orphan:b?.orphan??!1};if(ge.write(JSON.stringify(A)+`
+`)||await G(ge,"drain"),$){let ne=ze(t.archiveRequestedMonth,t.originalUrl??t.url);He.add(ne);let re={requestedMonth:t.archiveRequestedMonth??null,captureTimestamp:t.archiveTimestamp??null,originalUrl:t.originalUrl??t.url,archivedUrl:t.archivedUrl??null,status:t.extractionStatus==="successful"?"captured":"failed",httpStatus:t.status,contentHash:t.contentHash||null,wordCount:t.wordCount,failureCode:u.failureCode,failureReason:u.failureReason};$.write(JSON.stringify(re)+`
+`)||await G($,"drain")}}if($&&P?.timeline?.urls){let n=P.timeline,t=n.urls,u=(()=>{if(n.months?.length)return[...new Set(n.months)].sort();if(!n.from||!n.to)return[];let E=[],d=new Date(`${n.from}-01T00:00:00Z`),g=new Date(`${n.to}-01T00:00:00Z`);for(;d<=g&&E.length<60;)E.push(`${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,"0")}`),d.setUTCMonth(d.getUTCMonth()+(n.intervalMonths??1));return E})();for(let E of u)for(let d of t)He.has(ze(E,d))||$.write(JSON.stringify({requestedMonth:E,captureTimestamp:null,originalUrl:d,archivedUrl:null,status:"missing",httpStatus:null,contentHash:null,wordCount:0,failureCode:"wayback_capture_missing",failureReason:"No successful HTML capture was available in the requested month."})+`
+`)||await G($,"drain")}let kt=ue(ge),Pt=$?ue($):Promise.resolve();if(ge.end(),$?.end(),await Promise.all([kt,Pt]),p&&y(l(s,"rendered-dom","manifest.jsonl"),_.map(n=>JSON.stringify(n)).join(`
+`)),M){let n=await pt(S,{threshold:Number(e.options.similarityThreshold??void 0),maxPairs:Number(e.options.similarityMaxPairs??void 0),embedTexts:r.embedSimilarityTexts}),t=n.rows.map(yt),u=t.length>0?Object.keys(t[0]):["source_url","source_title","target_url","target_title","similarity","similarity_percent","corpus_percentile","exact_content_duplicate","source_word_count","target_word_count","cluster_id"],E=[u.join(","),...t.map(m=>u.map(b=>ft(m[b]??null)).join(","))].join(`
+`)+`
+`,{rows:d,clusters:g,...c}=n;y(l(s,"similarity-table.csv"),E),y(l(s,"similarity.jsonl"),t.map(m=>JSON.stringify(m)).join(`
+`)),y(l(s,"similarity-table-schema.json"),JSON.stringify({tableNameSuggestion:`site_similarity_${new URL(e.startUrl).hostname.replace(/[^a-z0-9]+/gi,"_").replace(/^_+|_+$/g,"").toLowerCase()}`,defaultSort:[{column:"similarity",direction:"desc"}],columns:{source_url:"text",source_title:"text",target_url:"text",target_title:"text",similarity:"number",similarity_percent:"number",corpus_percentile:"number",exact_content_duplicate:"boolean",source_word_count:"integer",target_word_count:"integer",cluster_id:"text"}},null,2)),y(l(s,"similarity-summary.json"),JSON.stringify(c,null,2)),y(l(s,"content-clusters.json"),JSON.stringify(n.clusters,null,2)),V+=["","## Rendered content similarity",`- Compared pages: ${n.comparedPages}`,`- Raw cosine threshold: ${n.threshold}`,`- Corpus score distribution: min ${n.scoreDistribution.min??"n/a"} / median ${n.scoreDistribution.median??"n/a"} / p90 ${n.scoreDistribution.p90??"n/a"} / max ${n.scoreDistribution.max??"n/a"}`,`- Qualifying pairs: ${n.qualifyingPairs}`,`- Returned table rows: ${n.returnedPairs}${n.pairsTruncated?" (capped)":""}`,`- Multi-page clusters: ${n.clusters.length}`,`- Embedding model: ${n.model} (${n.dimensions} dimensions)`,`- Corpus boilerplate removed: ${n.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${n.boilerplateRemoval.removedCharacters} characters`,`- Corpus SHA-256: ${n.corpusSha256}`].join(`
+`)}y(l(s,"report.md"),V),y(l(s,"crawl-summary.json"),JSON.stringify(F,null,2)),y(l(s,"images-manifest.jsonl"),Le.map(n=>JSON.stringify(n)).join(`
+`)),y(l(s,"manifest.json"),JSON.stringify({version:"site-export.v1",jobId:e.id,startUrl:e.startUrl,generatedAt:F.generatedAt,outputProfile:{json:!0,html:!0,markdown:!0,imageManifest:!0,imageBytes:i},pages:[...v].sort((n,t)=>n.pageId.localeCompare(t.pageId)),imagesManifestPath:"images-manifest.jsonl",download:"This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."},null,2)),P&&y(l(s,"wayback-manifest.json"),JSON.stringify({rootUrl:P.rootUrl??e.startUrl,requested:P.timeline??null,maxPagesPerSnapshot:P.maxPagesPerSnapshot??null,captureMatrix:"capture-matrix.jsonl"},null,2)),y(l(s,"issues.json"),JSON.stringify(Be,null,2)),y(l(s,"link-metrics.jsonl"),pe.map(n=>JSON.stringify(n)).join(`
+`)),y(l(s,"link-report.md"),nt(ve)),y(l(s,"links-summary.json"),JSON.stringify(ve.summary,null,2)),y(l(s,"external-domains.json"),JSON.stringify(ve.externalDomains,null,2)),r.imageAudit&&(y(l(s,"images.jsonl"),r.imageAudit.rows.map(n=>JSON.stringify(n)).join(`
+`)),y(l(s,"images-summary.json"),JSON.stringify(r.imageAudit.summary,null,2))),r.seoAudit&&y(l(s,"seo-audit.json"),JSON.stringify(r.seoAudit,null,2)),r.branding&&y(l(s,"branding.json"),JSON.stringify(r.branding,null,2)),i&&y(l(s,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-de,failed:de,downloadedBytes:ke,perPageCap:$e,perSiteCap:Ue,perFileByteCap:bt,totalByteCap:_t},null,2));let T=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(p){T.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let n of _)T.push({name:n.path,type:"text/plain"})}M&&(T.push({name:"similarity-table.csv",type:"text/csv"}),T.push({name:"similarity.jsonl",type:"application/x-ndjson"}),T.push({name:"similarity-table-schema.json",type:"application/json"}),T.push({name:"similarity-summary.json",type:"application/json"}),T.push({name:"content-clusters.json",type:"application/json"})),P&&(T.push({name:"wayback-manifest.json",type:"application/json"}),T.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),r.imageAudit&&(T.push({name:"images.jsonl",type:"application/x-ndjson"}),T.push({name:"images-summary.json",type:"application/json"})),r.seoAudit&&T.push({name:"seo-audit.json",type:"application/json"}),r.branding&&T.push({name:"branding.json",type:"application/json"}),i&&T.push({name:"images-download-summary.json",type:"application/json"});let N=new en,Tt=Ke({ownerId:String(e.userId),jobId:e.id,createdAt:new Date,content:N.outputStream});for(let n of T)N.addFile(l(s,n.name),n.name);let he=[...v].sort((n,t)=>n.pageId.localeCompare(t.pageId));for(let n of he)n.markdownPath&&N.addFile(l(s,n.markdownPath),n.markdownPath);for(let n of he)n.legacyMarkdownPath&&N.addFile(l(s,n.legacyMarkdownPath),n.legacyMarkdownPath);for(let n of he)n.htmlPath&&N.addFile(l(s,n.htmlPath),n.htmlPath);for(let n of he)N.addFile(l(s,n.jsonPath),n.jsonPath);if(i&&Z>de){let n=l(s,"images");for(let t of Gt(n,{recursive:!0})){let u=l(n,t);Zt(u).isFile()&&N.addFile(u,`images/${t.split("\\").join("/")}`)}}return N.end(),[await et("export_packaging",()=>Tt),...Oe]}finally{Kt(s,{recursive:!0,force:!0})}}export{Rt as a,on as b,an as c,Me as d,Ae as e,Lt as f,dn as g,ct as h,pn as i,je as j,Fn as k};

package/dist/chunk-GVYGAZV7.js

@@ -0,0 +1 @@
+import{g as c,j as d,k as p,l as w,m}from"./chunk-6SNOUSSZ.js";import{i as u}from"./chunk-C7MGJUTU.js";import{Inngest as x}from"inngest";var h=new x({id:"mcp-scraper"});function b(r){return Math.min(3600,Math.max(15,15*2**Math.min(8,Math.max(0,r-1))))}async function f(r){let t={claimed:r.rows.length,dispatched:0,failed:0},e=()=>{t.unknown=(t.unknown??0)+1},o=async a=>{let i=await r.read(a).catch(()=>null);return i?.id===a.id?i:null};for(let a of r.rows){let i=a.claimToken;if(!i||!await r.isCurrent(a,i).catch(()=>!1)){e();continue}try{await r.send(a)}catch(n){let l=(n instanceof Error?n.message:String(n)).slice(0,2e3);if(await r.retainRetry(a,i,l,b(a.attempts)).catch(async()=>{let s=await o(a);return s?.status==="failed"&&s.lastError===l}))t.failed+=1;else{let s=await o(a);s?.status==="dispatched"&&s.dispatchedAt?t.dispatched+=1:e()}continue}if(await r.acknowledge(a,i).catch(()=>!1))t.dispatched+=1;else{let n=await o(a);n?.status==="dispatched"&&n.dispatchedAt?t.dispatched+=1:e()}}return t}async function k(r){return f({rows:r,isCurrent:(t,e)=>p(t.id,e),send:async t=>{try{return await h.send({id:t.id,name:"mcp-scraper/extract.requested",data:{jobId:t.id}})}catch(e){throw Error(u(e instanceof Error?e.message:String(e)).slice(0,2e3))}},acknowledge:(t,e)=>w(t.id,e),retainRetry:(t,e,o,a)=>m(t.id,e,o,a),read:async t=>{let e=await c(t.id);return e?{id:e.id,status:e.dispatchedAt?"dispatched":e.dispatchClaimToken?"dispatching":e.dispatchError?"failed":"pending",dispatchedAt:e.dispatchedAt??null,lastError:e.dispatchError??null}:null}})}async function A(r){if((await c(r))?.dispatchedAt)return{claimed:0,dispatched:1,failed:0};let e=await d({jobId:r});return e.length?k(e):{claimed:0,dispatched:0,failed:0,unknown:1}}async function S(r=25){let t=await k(await d({limit:r}));return{checked:t.claimed,dispatched:t.dispatched,failed:t.failed,...t.unknown?{unknown:t.unknown}:{}}}export{h as a,f as b,A as c,S as d};

package/dist/chunk-HCK6PLM3.js

@@ -0,0 +1,3 @@
+import{a as w}from"./chunk-JMZIDHYQ.js";import{e as v,j as T}from"./chunk-SEZETSB3.js";import{i as I}from"./chunk-SDQW7IJN.js";import{b as y}from"./chunk-B3JRL45T.js";import{c as E}from"./chunk-OILOHQNL.js";import{createHash as L,randomBytes as N}from"crypto";import{basename as D,extname as O,join as j}from"path";import k from"p-limit";import{ZipFile as $}from"yazl";var P="page-media/",z=10080*60*1e3,W=900*1e3,_=10*1024*1024,G=45*1024*1024,X=15e5,K=25e5,V=6,B=5;function S(){return process.env.PAGE_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function C(){return{prefix:P,artifactTtlMs:z,downloadTtlMs:W,token:S()}}function Y(e,r){if(e.length>=3&&e[0]===255&&e[1]===216&&e[2]===255)return{mimeType:"image/jpeg",extension:"jpg"};if(e.length>=8&&e[0]===137&&e[1]===80&&e[2]===78&&e[3]===71)return{mimeType:"image/png",extension:"png"};if(e.length>=12&&String.fromCharCode(...e.slice(0,4))==="RIFF"&&String.fromCharCode(...e.slice(8,12))==="WEBP")return{mimeType:"image/webp",extension:"webp"};if(e.length>=6&&["GIF87a","GIF89a"].includes(String.fromCharCode(...e.slice(0,6))))return{mimeType:"image/gif",extension:"gif"};if(e.length>=12&&String.fromCharCode(...e.slice(4,12)).match(/^ftyp(?:avif|avis)$/))return{mimeType:"image/avif",extension:"avif"};let t=Buffer.from(e.slice(0,Math.min(e.length,1024))).toString("utf8").replace(/^\uFEFF/,"").trimStart();return(r==="image/svg+xml"||t.startsWith("<svg")||/^<\?xml[\s\S]*?<svg/i.test(t))&&/<svg[\s>]/i.test(t)?{mimeType:"image/svg+xml",extension:"svg"}:null}function H(e,r,t){if(t==="image")return Y(e,r);if(e.length>=12&&String.fromCharCode(...e.slice(4,8))==="ftyp")return{mimeType:"video/mp4",extension:"mp4"};if(e.length>=4&&e[0]===26&&e[1]===69&&e[2]===223&&e[3]===163)return{mimeType:"video/webm",extension:"webm"};if(e.length>=4&&String.fromCharCode(...e.slice(0,4))==="OggS")return t==="audio"?{mimeType:"audio/ogg",extension:"ogg"}:{mimeType:"video/ogg",extension:"ogv"};if(t==="audio"&&e.length>=3&&String.fromCharCode(...e.slice(0,3))==="ID3")return{mimeType:"audio/mpeg",extension:"mp3"};if(r&&w(r)===t){let i=r.split("/")[1]?.replace("jpeg","jpg").replace("mpeg","mp3").replace("svg+xml","svg");if(i&&/^[a-z0-9]+$/i.test(i))return{mimeType:r,extension:i}}return null}function J(e,r){let t=Buffer.from(e);if(r==="image/png"&&t.length>=24)return{width:t.readUInt32BE(16),height:t.readUInt32BE(20)};if(r==="image/gif"&&t.length>=10)return{width:t.readUInt16LE(6),height:t.readUInt16LE(8)};if(r==="image/webp"&&t.length>=30){let i=t.toString("ascii",12,16);if(i==="VP8X")return{width:1+t.readUIntLE(24,3),height:1+t.readUIntLE(27,3)};if(i==="VP8L"&&t[20]===47){let a=t.readUInt32LE(21);return{width:(a&16383)+1,height:(a>>14&16383)+1}}}if(r==="image/jpeg"){let i=2;for(;i+9<t.length;){if(t[i]!==255){i+=1;continue}let a=t[i+1];if(a===216||a===217){i+=2;continue}let o=t.readUInt16BE(i+2);if(o<2||i+o+2>t.length)break;if([192,193,194,195,197,198,199,201,202,203,205,206,207].includes(a))return{height:t.readUInt16BE(i+5),width:t.readUInt16BE(i+7)};i+=o+2}}if(r==="image/svg+xml"){let a=t.toString("utf8",0,Math.min(t.length,65536)).match(/<svg\b[^>]*>/i)?.[0]??"",o=Number(a.match(/\bwidth=["']?([\d.]+)/i)?.[1]??0),d=Number(a.match(/\bheight=["']?([\d.]+)/i)?.[1]??0);if(o>0&&d>0)return{width:Math.round(o),height:Math.round(d)};let c=a.match(/\bviewBox=["']([^"']+)["']/i)?.[1]?.trim().split(/[\s,]+/).map(Number);if(c?.length===4&&c[2]>0&&c[3]>0)return{width:Math.round(c[2]),height:Math.round(c[3])}}return null}async function Z(e){let r=Number(e.headers.get("content-length")??"");if(Number.isFinite(r)&&r>_)throw new Error("media_too_large");if(!e.body){let o=Buffer.from(await e.arrayBuffer());if(o.length>_)throw new Error("media_too_large");return o}let t=e.body.getReader(),i=[],a=0;try{for(;;){let{done:o,value:d}=await t.read();if(o)break;if(d){if(a+=d.byteLength,a>_)throw new Error("media_too_large");i.push(Buffer.from(d))}}}finally{t.releaseLock()}return Buffer.concat(i)}async function q(e,r){let t=e;for(let i=0;i<=B;i+=1){let a=await y(t,{field:"website media URL"});if(a.error||!a.parsed)throw new Error("media_url_rejected");let o=await fetch(a.parsed.href,{redirect:"manual",headers:{Accept:r==="image"?"image/*":`${r}/*`},signal:AbortSignal.timeout(15e3)});if([301,302,303,307,308].includes(o.status)){let s=o.headers.get("location");if(!s||i===B)throw new Error("media_redirect_rejected");t=new URL(s,a.parsed).href;continue}if(!o.ok)throw new Error(`media_fetch_${o.status}`);let d=await Z(o),c=o.headers.get("content-type")?.split(";")[0].trim().toLowerCase()??null,m=H(d,c,r);if(!m||w(m.mimeType)!==r)throw new Error("media_content_invalid");let f=r==="image"?J(d,m.mimeType):null;return{bytes:d,...m,finalUrl:a.parsed.href,width:f?.width??null,height:f?.height??null,sha256:L("sha256").update(d).digest("hex")}}throw new Error("media_redirect_rejected")}function Q(e){return new Promise((r,t)=>{let i=new $,a=[];i.outputStream.on("data",o=>a.push(Buffer.from(o))),i.outputStream.once("error",t),i.outputStream.once("end",()=>r(Buffer.concat(a)));for(let o of e)i.addBuffer(o.content,o.path);i.end()})}function M(e){return e.toLowerCase().replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,70)||"website"}function ee(e){try{return D(new URL(e).pathname).replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)}catch{return""}}function te(e){return I(e,P)}async function me(e){return te(e.artifactId)!==e.ownerId?null:T({policy:C(),artifactId:e.artifactId,maxBytes:60*1024*1024})}async function fe(e){let r=[],t=k(V),i=new Map,a=0,o=0,d=0;await Promise.all(e.media.assets.map((n,h)=>t(async()=>{try{let l=await q(n.url,n.type);n.finalUrl=l.finalUrl,n.mimeType=l.mimeType,n.sizeBytes=l.bytes.length,n.sha256=l.sha256,n.width=l.width??n.width,n.height=l.height??n.height;let g=i.get(l.sha256);if(g){n.filename=g,n.duplicateOf=g,n.downloadStatus="downloaded",n.downloadError=null;return}if(a+l.bytes.length>G)throw new Error("archive_byte_limit_reached");a+=l.bytes.length;let u=ee(l.finalUrl),A=O(u),F=u?A?u.slice(0,-A.length):u:n.type,p=`media/${String(h+1).padStart(4,"0")}-${M(F)}.${l.extension}`;i.set(l.sha256,p),r.push({path:p,content:l.bytes}),n.filename=p,n.duplicateOf=null,n.downloadStatus="downloaded",n.downloadError=null,n.type==="image"&&d<e.maxInlineImages&&l.bytes.length<=X&&o+l.bytes.length<=K&&(n.inlinePreview={data:l.bytes.toString("base64"),mimeType:l.mimeType},o+=l.bytes.length,d+=1)}catch(l){n.finalUrl=null,n.mimeType=null,n.sizeBytes=null,n.sha256=null,n.savedPath=null,n.downloadStatus="failed",n.downloadError=l instanceof Error?l.message:"media_download_failed",n.inlinePreview=null}})));let c=e.media.assets.filter(n=>n.downloadStatus==="downloaded").length,m=e.media.assets.length-c;if(m>0&&e.media.warnings.push(`${m} media asset(s) could not be downloaded; media.jsonl contains bounded error codes.`),c===0)return{media:e.media,artifact:null};let f=e.media.assets.map(({inlinePreview:n,...h})=>h),s={pageUrl:e.pageUrl,title:e.title,extractedAt:e.extractedAt,completeness:e.media.completeness,exhausted:e.media.exhausted,stopReason:e.media.stopReason,scrollRounds:e.media.scrollRounds,staticFound:e.media.staticFound,renderedFound:e.media.renderedFound,totalFound:e.media.totalFound,retainedAfterFilteringAndVariantCollapse:e.media.assets.length,downloaded:c,failed:m,notes:["The inventory unions static-source and rendered-page discovery.","complete means rendered scrolling reached three stable document-height checks; it does not claim every interactive control was opened.","Media records preserve discovery evidence but do not assert ownership, rights, identity, or publishability."]};r.push({path:"summary.json",content:Buffer.from(JSON.stringify(s,null,2))}),r.push({path:"media.jsonl",content:Buffer.from(f.map(n=>JSON.stringify(n)).join(`
+`)+`
+`)});let R=await Q(r),U=N(6).toString("hex"),x=await v({policy:C(),ownerId:e.ownerId,artifactKey:`${U}.zip`,createdAt:new Date(e.extractedAt),filename:`${M(e.title||new URL(e.pageUrl).hostname)}-website-media.zip`,contentType:"application/zip",content:R}),b=S()?null:j(E(),"blobs",x.artifactId);return{media:e.media,artifact:{...x,localPath:b}}}export{P as a,z as b,C as c,te as d,me as e,fe as f};