mcp-scraper 0.89.6 → 0.89.8

package/dist/{extract-bundle-BTZQJTYV.js → extract-bundle-DI7OVLT5.js}

@@ -1,4 +1,4 @@
-import{c as Je,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-3H7HU5JI.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-WXOLIRFJ.js";import"./chunk-YQZGZBB4.js";import"./chunk-QPWPR5XG.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-MZN4U5BL.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-YQ5U5C2X.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
+import{c as We,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-JVU2THSY.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-VT6SLBOU.js";import"./chunk-XG6GCEUE.js";import"./chunk-M5QHXNFZ.js";import{c as Je,e as ze,f as Fe,g as He}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-E4ME7J6G.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-7BEWBRMT.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
 
 `).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
 
@@ -8,11 +8,11 @@ ${C}`}),minimumPageCount:n,removedBlockSignatures:b.size,removedCharacters:w}}va
             ORDER BY rowid
             LIMIT ${Rt}`,args:[r,n]});if(!c.rows.length)return;if(n=Number(c.rows.at(-1)?.cursor_id??n),!Number.isSafeInteger(n)||n<=0)throw new Error("site extract page cursor is invalid");yield c.rows.map(b=>JSON.parse(String(b.page)))}}function Et(r){let s=r==null?"":String(r);return/[",\r\n]/.test(s)?`"${s.replace(/"/g,'""')}"`:s}function $t(r){return{source_url:r.sourceUrl,source_title:r.sourceTitle,target_url:r.targetUrl,target_title:r.targetTitle,similarity:r.similarity,similarity_percent:r.similarityPercent,corpus_percentile:r.corpusPercentile,exact_content_duplicate:r.exactContentDuplicate,source_word_count:r.sourceWordCount,target_word_count:r.targetWordCount,cluster_id:r.clusterId}}async function rn(r,s={}){let n=a(Mt(),`extract-bundle-${r.id}-${Date.now()}`),c=r.options.downloadImages===!0;if(r.userId==null)throw new Error("site extract artifact owner is missing");let b=Xe(String(r.userId));ie(n,{recursive:!0}),c&&ie(a(n,"images"),{recursive:!0});let w=r.options.captureRenderedDom===!0,$=r.options.semanticSimilarity===!0;w&&ie(a(n,"rendered-dom"),{recursive:!0});try{let f=[],M=[],I=[],A=jt(),C=[],_=r.options.waybackTimeline,T=new Map;for await(let t of Ae(r.id))for(let e of t){T.set(oe(e.url),e.status),f.push({url:e.url,status:e.status,title:e.title,titleLength:e.titleLength,titlePixels:e.titlePixels,metaDescription:e.metaDescription,metaDescLength:e.metaDescLength,h1:e.h1,h1_2:e.h1_2,h2Count:e.h2Count,extractionStatus:e.extractionStatus,indexable:e.indexable,indexabilityReason:e.indexabilityReason,canonicalUrl:e.canonicalUrl?e.url:null,wordCount:e.wordCount,contentHash:e.contentHash,imagesMissingAlt:e.imagesMissingAlt,schemaTypes:e.schemaTypes.length>0?["present"]:[],outlinks:[]});let o=e.pageId??G("sha256").update(e.archivedUrl??e.url).digest("hex"),x=a(n,"pages",o);ie(x,{recursive:!0});let m=e.contentRef?await b(e.contentRef):null,d=m?.html??null,l=m?.markdown??e.bodyMarkdown??null;if($&&e.extractionStatus==="successful"&&l?.trim()&&M.push({url:e.url,title:e.title,bodyMarkdown:l,contentHash:e.contentHash,wordCount:e.wordCount}),w&&d!=null){let Ne=`rendered-dom/${A(e.url)}.html.txt`;g(a(n,Ne),d),I.push({url:e.url,path:Ne,bytes:Buffer.byteLength(d),truncated:e.renderedDomTruncated===!0,sanitized:e.renderedDomSanitized===!0})}let u=d==null?null:`pages/${o}/page.html`,h=l==null?null:`pages/${o}/page.md`,S=l==null||!e.archiveRequestedMonth?null:`pages/${e.archiveRequestedMonth}/${o}.md`;d!=null&&u&&g(a(n,u),d),l!=null&&h&&g(a(n,h),l),l!=null&&S&&(ie(a(n,"pages",e.archiveRequestedMonth),{recursive:!0}),g(a(n,S),l));let{bodyMarkdown:R,contentRef:te,discoveryLinks:ne,...dt}=e,re={version:"site-export-page.v1",...dt,pageId:o,content:{htmlPath:u,markdownPath:h,htmlBytes:d==null?0:Buffer.byteLength(d),markdownBytes:l==null?0:Buffer.byteLength(l),htmlSha256:d==null?null:G("sha256").update(d).digest("hex"),markdownSha256:l==null?null:G("sha256").update(l).digest("hex")}},Le=JSON.stringify(re,null,2),Oe=`pages/${o}/page.json`;g(a(n,Oe),Le),C.push({pageId:o,url:e.archivedUrl??e.url,jsonPath:Oe,htmlPath:u,markdownPath:h,legacyMarkdownPath:S,jsonSha256:G("sha256").update(Le).digest("hex"),htmlSha256:re.content.htmlSha256,markdownSha256:re.content.markdownSha256,htmlBytes:re.content.htmlBytes,markdownBytes:re.content.markdownBytes})}let j=It(),X=se(a(n,"links.jsonl.gz"));j.pipe(X);let le=async t=>{j.write(t)||await Y(j,"drain")},L=se(a(n,"schema.jsonl")),O=async t=>{L.write(t)||await Y(L,"drain")},v=se(a(n,"failures.jsonl")),U=async t=>{v.write(t)||await Y(v,"drain")},D=new Map,N=new Map,z=new Map,F=new Map,H=new Map,ue=new Set,i=0,p=0,y=0,k=0,q=0,he=!1,fe="";try{fe=st(new URL(r.startUrl).hostname)}catch{fe=""}let it=At(Tt),Re=[],Te=[],ve=[],Z=0,ce=0,ye=0,at=t=>ye+t>rt?!1:(ye+=t,!0);for await(let t of Ae(r.id))for(let e of t){if(e.schema?.length&&await O(JSON.stringify({url:e.url,schema:e.schema})+`
 `),e.extractionStatus==="failed"){let{failureCode:m,failureReason:d}=ke(e.failureCode,e.failureReason);await U(JSON.stringify({url:e.url,status:e.status,via:e.via,failureCode:m,failureReason:d,fetchAttempts:e.fetchAttempts??[]})+`
-`)}if(e.imageLinks?.length)for(let[m,d]of e.imageLinks.entries()){let l=G("sha256").update(`${e.url}\0${d}`).digest("hex"),u={imageId:l,sourcePage:e.url,sourceUrl:d,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Te.push(u),!c)continue;if(m>=Me){u.status="skipped_page_limit",u.failureCode="page_image_limit",u.failureReason=`Only the first ${Me} image candidates are downloaded per page.`;continue}if(Z>=Ie){u.status="skipped_site_limit",u.failureCode="site_image_limit",u.failureReason=`Only the first ${Ie} image candidates are downloaded per site.`;continue}Z++;let h=`${l}-${vt(d,m)}`,S=a(n,"images");Re.push(it(()=>Je(d,S,h,{expectedType:"image",maxBytes:nt,consumeBytes:at})).then(async R=>{let te=bt(R.savedPath),ne=await Ye({ownerId:String(r.userId),jobId:r.id,imageId:l,filename:h,contentType:R.mimeType??"application/octet-stream",content:te,sourceUrl:d,sourcePage:e.url});ve.push(ne),u.status="downloaded",u.path=`images/${h}`,u.artifactId=ne.key,u.mimeType=R.mimeType,u.bytes=R.sizeBytes,u.sha256=G("sha256").update(te).digest("hex")},R=>{ce++,u.status="failed",u.failureCode="image_download_failed",u.failureReason=(R instanceof Error?R.message:String(R)).replace(/\s+/g," ").slice(0,500)}))}let o=oe(e.url),x={int:0,ext:0};for(let m of e.outlinks??[]){let d=(m.rel??"").split(/\s+/).includes("nofollow"),l=oe(m.href),u=T.get(l)??null;if(await le(JSON.stringify({from:e.url,to:m.href,anchor:m.anchor,rel:m.rel,internal:m.internal,nofollow:d,targetStatus:u})+`
-`),k++,q>=Ce){he=!0;continue}if(q++,m.internal){if(x.int++,i++,D.has(l)||D.set(l,new Set),D.get(l).add(o),m.anchor){N.has(l)||N.set(l,new Map);let h=N.get(l);h.set(m.anchor,(h.get(m.anchor)??0)+1)}!d&&(T.get(l)===200||!T.has(l))&&(z.has(o)||z.set(o,new Set),z.get(o).add(l)),u!=null&&u>=400&&(y++,ue.add(e.url))}else{x.ext++;let h="";try{h=st(new URL(m.href).hostname)}catch{continue}if(!h||h===fe)continue;p++;let S=H.get(h)??{links:0,nofollow:0,pages:new Set};S.links++,d&&S.nofollow++,S.pages.add(e.url),H.set(h,S)}}F.set(e.url,x)}let ot=Promise.all([ae(X),ae(L),ae(v)]);j.end(),L.end(),v.end(),await ot,await Promise.all(Re);let W=new Map,we=oe(r.startUrl);W.set(we,0);let Se=[we];for(;Se.length>0;){let t=Se.shift(),e=W.get(t);for(let o of z.get(t)??[])W.has(o)||(W.set(o,e+1),Se.push(o))}let K=new Map;for(let t of f){let e=oe(t.url),o=D.get(e)??new Set,x=N.get(e),m=x?[...x.entries()].sort((l,u)=>u[1]-l[1]).slice(0,3).map(l=>l[0]):[],d=F.get(t.url)??{int:0,ext:0};K.set(t.url,{url:t.url,inlinks:o.size,uniqueInlinks:o.size,outlinksInternal:d.int,outlinksExternal:d.ext,crawlDepth:W.has(e)?W.get(e):null,orphan:o.size===0&&e!==we,topAnchors:m})}let je=He(f,K,ue),Q=We(r.startUrl,f,je,K),V=Ze(r),J={jobId:r.id,startUrl:r.startUrl,status:r.successfulUrls===0?"failed":r.failedUrls>0||r.remainingUrls>0||V.creditTruncated?"partial":"complete",...V,discovered:r.totalUrls,attempted:r.attemptedUrls,successful:r.successfulUrls,failed:r.failedUrls,remaining:r.remainingUrls,linkAnalysis:{truncated:he,analyzedEdges:q,streamedEdges:k,edgeCap:Ce},generatedAt:new Date().toISOString()};Q=["## Crawl delivery",`- Discovered: ${J.discovered}`,`- Attempted: ${J.attempted}`,`- Successful: ${J.successful}`,`- Failed: ${J.failed}`,`- Remaining: ${J.remaining}`,`- Requested page cap: ${V.requestedMaxPages}`,`- Funded page cap: ${V.effectiveMaxPages}`,`- Credit-truncated: ${V.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${he?` (capped at ${Ce})`:""}`,"",Q].join(`
+`)}if(e.imageLinks?.length)for(let[m,d]of e.imageLinks.entries()){let l=G("sha256").update(`${e.url}\0${d}`).digest("hex"),u={imageId:l,sourcePage:e.url,sourceUrl:d,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Te.push(u),!c)continue;if(m>=Me){u.status="skipped_page_limit",u.failureCode="page_image_limit",u.failureReason=`Only the first ${Me} image candidates are downloaded per page.`;continue}if(Z>=Ie){u.status="skipped_site_limit",u.failureCode="site_image_limit",u.failureReason=`Only the first ${Ie} image candidates are downloaded per site.`;continue}Z++;let h=`${l}-${vt(d,m)}`,S=a(n,"images");Re.push(it(()=>We(d,S,h,{expectedType:"image",maxBytes:nt,consumeBytes:at})).then(async R=>{let te=bt(R.savedPath),ne=await Ye({ownerId:String(r.userId),jobId:r.id,imageId:l,filename:h,contentType:R.mimeType??"application/octet-stream",content:te,sourceUrl:d,sourcePage:e.url});ve.push(ne),u.status="downloaded",u.path=`images/${h}`,u.artifactId=ne.key,u.mimeType=R.mimeType,u.bytes=R.sizeBytes,u.sha256=G("sha256").update(te).digest("hex")},R=>{ce++,u.status="failed",u.failureCode="image_download_failed",u.failureReason=(R instanceof Error?R.message:String(R)).replace(/\s+/g," ").slice(0,500)}))}let o=oe(e.url),x={int:0,ext:0};for(let m of e.outlinks??[]){let d=(m.rel??"").split(/\s+/).includes("nofollow"),l=oe(m.href),u=T.get(l)??null;if(await le(JSON.stringify({from:e.url,to:m.href,anchor:m.anchor,rel:m.rel,internal:m.internal,nofollow:d,targetStatus:u})+`
+`),k++,q>=Ce){he=!0;continue}if(q++,m.internal){if(x.int++,i++,D.has(l)||D.set(l,new Set),D.get(l).add(o),m.anchor){N.has(l)||N.set(l,new Map);let h=N.get(l);h.set(m.anchor,(h.get(m.anchor)??0)+1)}!d&&(T.get(l)===200||!T.has(l))&&(z.has(o)||z.set(o,new Set),z.get(o).add(l)),u!=null&&u>=400&&(y++,ue.add(e.url))}else{x.ext++;let h="";try{h=st(new URL(m.href).hostname)}catch{continue}if(!h||h===fe)continue;p++;let S=H.get(h)??{links:0,nofollow:0,pages:new Set};S.links++,d&&S.nofollow++,S.pages.add(e.url),H.set(h,S)}}F.set(e.url,x)}let ot=Promise.all([ae(X),ae(L),ae(v)]);j.end(),L.end(),v.end(),await ot,await Promise.all(Re);let W=new Map,we=oe(r.startUrl);W.set(we,0);let Se=[we];for(;Se.length>0;){let t=Se.shift(),e=W.get(t);for(let o of z.get(t)??[])W.has(o)||(W.set(o,e+1),Se.push(o))}let K=new Map;for(let t of f){let e=oe(t.url),o=D.get(e)??new Set,x=N.get(e),m=x?[...x.entries()].sort((l,u)=>u[1]-l[1]).slice(0,3).map(l=>l[0]):[],d=F.get(t.url)??{int:0,ext:0};K.set(t.url,{url:t.url,inlinks:o.size,uniqueInlinks:o.size,outlinksInternal:d.int,outlinksExternal:d.ext,crawlDepth:W.has(e)?W.get(e):null,orphan:o.size===0&&e!==we,topAnchors:m})}let je=Fe(f,K,ue),Q=He(r.startUrl,f,je,K),V=Ze(r),J={jobId:r.id,startUrl:r.startUrl,status:r.successfulUrls===0?"failed":r.failedUrls>0||r.remainingUrls>0||V.creditTruncated?"partial":"complete",...V,discovered:r.totalUrls,attempted:r.attemptedUrls,successful:r.successfulUrls,failed:r.failedUrls,remaining:r.remainingUrls,linkAnalysis:{truncated:he,analyzedEdges:q,streamedEdges:k,edgeCap:Ce},generatedAt:new Date().toISOString()};Q=["## Crawl delivery",`- Discovered: ${J.discovered}`,`- Attempted: ${J.attempted}`,`- Successful: ${J.successful}`,`- Failed: ${J.failed}`,`- Remaining: ${J.remaining}`,`- Requested page cap: ${V.requestedMaxPages}`,`- Funded page cap: ${V.effectiveMaxPages}`,`- Credit-truncated: ${V.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${he?` (capped at ${Ce})`:""}`,"",Q].join(`
 `),s.imageAudit&&(Q+=`
 
-${ze(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},$e=0,Ue=0;for(let t of me)$e+=t.inlinks,Ue+=t.outlinksInternal+t.outlinksExternal,t.inlinks===0?ee.zero++:t.inlinks<=2?ee.oneToTwo++:t.inlinks<=10?ee.threeToTen++:ee.elevenPlus++;let be=[...H.entries()].map(([t,e])=>({domain:t,links:e.links,nofollow:e.nofollow,pages:e.pages.size})).sort((t,e)=>e.links-t.links),xe={summary:{internal:{totalLinks:i,pages:f.length,orphans:me.filter(t=>t.orphan).length,brokenInternal:y,avgInlinks:f.length?Ee($e/f.length):0,avgOutlinks:f.length?Ee(Ue/f.length):0,distribution:ee,topByInlinks:[...me].sort((t,e)=>e.inlinks-t.inlinks).slice(0,20).map(t=>({url:t.url,inlinks:t.inlinks,outlinksInternal:t.outlinksInternal,outlinksExternal:t.outlinksExternal}))},external:{totalLinks:p,uniqueDomains:be.length,topDomains:be.slice(0,20)}},externalDomains:be},de=se(a(n,"pages.jsonl")),lt=new Map(C.map(t=>[t.pageId,t])),E=_?se(a(n,"capture-matrix.jsonl")):null,De=new Set,Be=(t,e)=>{let o=e;try{o=new URL(e).href}catch{}return`${t??""}\0${o}`};for await(let t of Ae(r.id))for(let e of t){let o=e.extractionStatus==="failed"?ke(e.failureCode,e.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:x,schema:m,outlinks:d,contentRef:l,...u}=e,h=K.get(e.url),S=e.pageId?lt.get(e.pageId):void 0,R={...u,...e.extractionStatus==="failed"?o:{},representations:S?{jsonPath:S.jsonPath,htmlPath:S.htmlPath,markdownPath:S.markdownPath,jsonSha256:S.jsonSha256,htmlSha256:S.htmlSha256,markdownSha256:S.markdownSha256,htmlBytes:S.htmlBytes,markdownBytes:S.markdownBytes}:null,inlinks:h?.inlinks??0,crawlDepth:h?.crawlDepth??null,orphan:h?.orphan??!1};if(de.write(JSON.stringify(R)+`
+${Je(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},$e=0,Ue=0;for(let t of me)$e+=t.inlinks,Ue+=t.outlinksInternal+t.outlinksExternal,t.inlinks===0?ee.zero++:t.inlinks<=2?ee.oneToTwo++:t.inlinks<=10?ee.threeToTen++:ee.elevenPlus++;let be=[...H.entries()].map(([t,e])=>({domain:t,links:e.links,nofollow:e.nofollow,pages:e.pages.size})).sort((t,e)=>e.links-t.links),xe={summary:{internal:{totalLinks:i,pages:f.length,orphans:me.filter(t=>t.orphan).length,brokenInternal:y,avgInlinks:f.length?Ee($e/f.length):0,avgOutlinks:f.length?Ee(Ue/f.length):0,distribution:ee,topByInlinks:[...me].sort((t,e)=>e.inlinks-t.inlinks).slice(0,20).map(t=>({url:t.url,inlinks:t.inlinks,outlinksInternal:t.outlinksInternal,outlinksExternal:t.outlinksExternal}))},external:{totalLinks:p,uniqueDomains:be.length,topDomains:be.slice(0,20)}},externalDomains:be},de=se(a(n,"pages.jsonl")),lt=new Map(C.map(t=>[t.pageId,t])),E=_?se(a(n,"capture-matrix.jsonl")):null,De=new Set,Be=(t,e)=>{let o=e;try{o=new URL(e).href}catch{}return`${t??""}\0${o}`};for await(let t of Ae(r.id))for(let e of t){let o=e.extractionStatus==="failed"?ke(e.failureCode,e.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:x,schema:m,outlinks:d,contentRef:l,...u}=e,h=K.get(e.url),S=e.pageId?lt.get(e.pageId):void 0,R={...u,...e.extractionStatus==="failed"?o:{},representations:S?{jsonPath:S.jsonPath,htmlPath:S.htmlPath,markdownPath:S.markdownPath,jsonSha256:S.jsonSha256,htmlSha256:S.htmlSha256,markdownSha256:S.markdownSha256,htmlBytes:S.htmlBytes,markdownBytes:S.markdownBytes}:null,inlinks:h?.inlinks??0,crawlDepth:h?.crawlDepth??null,orphan:h?.orphan??!1};if(de.write(JSON.stringify(R)+`
 `)||await Y(de,"drain"),E){let te=Be(e.archiveRequestedMonth,e.originalUrl??e.url);De.add(te);let ne={requestedMonth:e.archiveRequestedMonth??null,captureTimestamp:e.archiveTimestamp??null,originalUrl:e.originalUrl??e.url,archivedUrl:e.archivedUrl??null,status:e.extractionStatus==="successful"?"captured":"failed",httpStatus:e.status,contentHash:e.contentHash||null,wordCount:e.wordCount,failureCode:o.failureCode,failureReason:o.failureReason};E.write(JSON.stringify(ne)+`
 `)||await Y(E,"drain")}}if(E&&_?.timeline?.urls){let t=_.timeline,e=t.urls,o=(()=>{if(t.months?.length)return[...new Set(t.months)].sort();if(!t.from||!t.to)return[];let x=[],m=new Date(`${t.from}-01T00:00:00Z`),d=new Date(`${t.to}-01T00:00:00Z`);for(;m<=d&&x.length<60;)x.push(`${m.getUTCFullYear()}-${String(m.getUTCMonth()+1).padStart(2,"0")}`),m.setUTCMonth(m.getUTCMonth()+(t.intervalMonths??1));return x})();for(let x of o)for(let m of e)De.has(Be(x,m))||E.write(JSON.stringify({requestedMonth:x,captureTimestamp:null,originalUrl:m,archivedUrl:null,status:"missing",httpStatus:null,contentHash:null,wordCount:0,failureCode:"wayback_capture_missing",failureReason:"No successful HTML capture was available in the requested month."})+`
 `)||await Y(E,"drain")}let ut=ae(de),ct=E?ae(E):Promise.resolve();if(de.end(),E?.end(),await Promise.all([ut,ct]),w&&g(a(n,"rendered-dom","manifest.jsonl"),I.map(t=>JSON.stringify(t)).join(`
@@ -22,5 +22,5 @@ ${ze(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:
 `)),g(a(n,"similarity-table-schema.json"),JSON.stringify({tableNameSuggestion:`site_similarity_${new URL(r.startUrl).hostname.replace(/[^a-z0-9]+/gi,"_").replace(/^_+|_+$/g,"").toLowerCase()}`,defaultSort:[{column:"similarity",direction:"desc"}],columns:{source_url:"text",source_title:"text",target_url:"text",target_title:"text",similarity:"number",similarity_percent:"number",corpus_percentile:"number",exact_content_duplicate:"boolean",source_word_count:"integer",target_word_count:"integer",cluster_id:"text"}},null,2)),g(a(n,"similarity-summary.json"),JSON.stringify(l,null,2)),g(a(n,"content-clusters.json"),JSON.stringify(t.clusters,null,2)),Q+=["","## Rendered content similarity",`- Compared pages: ${t.comparedPages}`,`- Raw cosine threshold: ${t.threshold}`,`- Corpus score distribution: min ${t.scoreDistribution.min??"n/a"} / median ${t.scoreDistribution.median??"n/a"} / p90 ${t.scoreDistribution.p90??"n/a"} / max ${t.scoreDistribution.max??"n/a"}`,`- Qualifying pairs: ${t.qualifyingPairs}`,`- Returned table rows: ${t.returnedPairs}${t.pairsTruncated?" (capped)":""}`,`- Multi-page clusters: ${t.clusters.length}`,`- Embedding model: ${t.model} (${t.dimensions} dimensions)`,`- Corpus boilerplate removed: ${t.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${t.boilerplateRemoval.removedCharacters} characters`,`- Corpus SHA-256: ${t.corpusSha256}`].join(`
 `)}g(a(n,"report.md"),Q),g(a(n,"crawl-summary.json"),JSON.stringify(J,null,2)),g(a(n,"images-manifest.jsonl"),Te.map(t=>JSON.stringify(t)).join(`
 `)),g(a(n,"manifest.json"),JSON.stringify({version:"site-export.v1",jobId:r.id,startUrl:r.startUrl,generatedAt:J.generatedAt,outputProfile:{json:!0,html:!0,markdown:!0,imageManifest:!0,imageBytes:c},pages:[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId)),imagesManifestPath:"images-manifest.jsonl",download:"This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."},null,2)),_&&g(a(n,"wayback-manifest.json"),JSON.stringify({rootUrl:_.rootUrl??r.startUrl,requested:_.timeline??null,maxPagesPerSnapshot:_.maxPagesPerSnapshot??null,captureMatrix:"capture-matrix.jsonl"},null,2)),g(a(n,"issues.json"),JSON.stringify(je,null,2)),g(a(n,"link-metrics.jsonl"),me.map(t=>JSON.stringify(t)).join(`
-`)),g(a(n,"link-report.md"),Fe(xe)),g(a(n,"links-summary.json"),JSON.stringify(xe.summary,null,2)),g(a(n,"external-domains.json"),JSON.stringify(xe.externalDomains,null,2)),s.imageAudit&&(g(a(n,"images.jsonl"),s.imageAudit.rows.map(t=>JSON.stringify(t)).join(`
+`)),g(a(n,"link-report.md"),ze(xe)),g(a(n,"links-summary.json"),JSON.stringify(xe.summary,null,2)),g(a(n,"external-domains.json"),JSON.stringify(xe.externalDomains,null,2)),s.imageAudit&&(g(a(n,"images.jsonl"),s.imageAudit.rows.map(t=>JSON.stringify(t)).join(`
 `)),g(a(n,"images-summary.json"),JSON.stringify(s.imageAudit.summary,null,2))),s.seoAudit&&g(a(n,"seo-audit.json"),JSON.stringify(s.seoAudit,null,2)),s.branding&&g(a(n,"branding.json"),JSON.stringify(s.branding,null,2)),c&&g(a(n,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-ce,failed:ce,downloadedBytes:ye,perPageCap:Me,perSiteCap:Ie,perFileByteCap:nt,totalByteCap:rt},null,2));let P=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(w){P.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let t of I)P.push({name:t.path,type:"text/plain"})}$&&(P.push({name:"similarity-table.csv",type:"text/csv"}),P.push({name:"similarity.jsonl",type:"application/x-ndjson"}),P.push({name:"similarity-table-schema.json",type:"application/json"}),P.push({name:"similarity-summary.json",type:"application/json"}),P.push({name:"content-clusters.json",type:"application/json"})),_&&(P.push({name:"wayback-manifest.json",type:"application/json"}),P.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),s.imageAudit&&(P.push({name:"images.jsonl",type:"application/x-ndjson"}),P.push({name:"images-summary.json",type:"application/json"})),s.seoAudit&&P.push({name:"seo-audit.json",type:"application/json"}),s.branding&&P.push({name:"branding.json",type:"application/json"}),c&&P.push({name:"images-download-summary.json",type:"application/json"});let B=new Ct,mt=Ge({ownerId:String(r.userId),jobId:r.id,createdAt:new Date,content:B.outputStream});for(let t of P)B.addFile(a(n,t.name),t.name);let pe=[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId));for(let t of pe)t.markdownPath&&B.addFile(a(n,t.markdownPath),t.markdownPath);for(let t of pe)t.legacyMarkdownPath&&B.addFile(a(n,t.legacyMarkdownPath),t.legacyMarkdownPath);for(let t of pe)t.htmlPath&&B.addFile(a(n,t.htmlPath),t.htmlPath);for(let t of pe)B.addFile(a(n,t.jsonPath),t.jsonPath);if(c&&Z>ce){let t=a(n,"images");for(let e of xt(t,{recursive:!0})){let o=a(t,e);_t(o).isFile()&&B.addFile(o,`images/${e.split("\\").join("/")}`)}}return B.end(),[await mt,...ve]}finally{kt(n,{recursive:!0,force:!0})}}export{Ce as MAX_ANALYZED_LINK_EDGES,Rt as SITE_EXTRACT_PAGE_CHUNK,rn as assembleExtractArtifacts};

package/dist/{gmail-service-ZZDF52KK.js → gmail-service-ASX7YBC3.js}

@@ -1 +1 @@
-import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-H3QW5U67.js";import"./chunk-Y3ITV2OK.js";import"./chunk-YCZKKSTT.js";import"./chunk-HUV2WTRW.js";import"./chunk-MZN4U5BL.js";import"./chunk-WUVMD6ZX.js";import"./chunk-YQ5U5C2X.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};
+import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-Q4A2KIPZ.js";import"./chunk-5B42QVDC.js";import"./chunk-2GRE4BAO.js";import"./chunk-HUV2WTRW.js";import"./chunk-E4ME7J6G.js";import"./chunk-DKQZNSYM.js";import"./chunk-7BEWBRMT.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};