mcp-scraper 0.90.7 → 0.91.0

package/dist/db-B5XJTOGN.js

@@ -0,0 +1 @@
+import{$ as M,$a as Ma,$b as Mb,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Yb as Jb,Z as K,Za as Ka,Zb as Kb,_ as L,_a as La,_b as Lb,aa as N,ab as Na,ac as Nb,ba as O,bb as Oa,bc as Ob,ca as P,cb as Pa,cc as Pb,da as Q,db as Qa,dc as Qb,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-WJ4XFLS4.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,vb as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Jb as SiteAuditJobRowSchema,Kb as SiteAuditPhaseLogRowSchema,Aa as advanceWorkflowRunStep,ya as appendWorkflowArtifacts,mb as applySubscriptionInvoiceGrant,W as attachHarvestAttemptProviderSession,Ya as cancelJob,K as checkRateLimit,Ja as checkpointDurablePaaJob,Ia as checkpointJob,t as claimDueStripeEvents,Ha as claimDurablePaaJob,Bb as claimMonthlyFreeRefresh,Ga as claimPendingJob,_a as claimPendingKpoJob,La as completeDurablePaaJob,Oa as completeExpiredDurablePaaJob,Ua as completeJob,bb as completeKpoJob,Va as completeRunningJob,u as completeStripeEvent,va as completeWorkflowRunRecord,n as consumeCode,M as consumePasswordResetToken,Fa as countActiveJobsForUser,Ea as countActiveUsers,R as createJob,S as createJobWithId,Za as createKpoJob,L as createPasswordResetToken,T as createRunningJob,J as createUser,qa as createWorkflowRun,ja as createWorkflowSchedule,yb as creditMc,zb as creditMcIdempotent,Q as deactivateUser,w as deadLetterStripeEvent,Cb as debitMc,Db as debitMcIdempotent,na as deleteWorkflowSchedule,s as enqueueStripeEvent,wb as expireOldLots,Ma as failDurablePaaJob,Pa as failExpiredDurablePaaJob,Wa as failJob,cb as failKpoJob,Xa as failRunningJob,wa as failWorkflowRunRecord,X as finalizeExpiredHarvestAttempts,Y as finishHarvestAttempt,z as generateApiKey,Gb as getBillingLedgerSummary,l as getClient,nb as getConnectedAccountBillingState,e as getDb,Mb as getInboxMessage,aa as getJob,$a as getKpoJob,eb as getKpoPhaseLog,Fb as getLedger,rb as getMemoryPlan,sb as getMemoryProvisioned,U as getOrCreateRunningJob,fa as getPageSnapshot,p as getRefresh,x as getStripeEventQueueHealth,D as getUserByApiKey,C as getUserByEmail,E as getUserById,gb as getUserByStripeCustomerId,F as getUserMemoryCreds,I as getUserStats,Ca as getWorkflowArtifact,ra as getWorkflowRun,sa as getWorkflowRunByIdempotencyKey,za as getWorkflowRunStepState,ka as getWorkflowSchedule,A as hashPassword,Pb as insertInboxMessage,d as inspectCurrentCoreSchema,Ab as ledgerExistsForOperation,Hb as ledgerExistsForStripePI,Qb as listActiveUserIds,oa as listDueWorkflowSchedules,$ as listHarvestAttempts,_ as listHarvestAttemptsPendingProviderTelemetry,Lb as listInboxMessages,ea as listJobs,fb as listKpoJobs,ca as listPendingExtractUrlDispatches,Qa as listRecentTerminalDurablePaaJobs,Sa as listRecoverableDurablePaaJobs,ia as listRequestEvents,Ra as listStaleRunningDurablePaaJobs,P as listUsers,pb as listUsersDueForConnectedAccountBillingReconciliation,Ba as listWorkflowArtifacts,ta as listWorkflowRuns,la as listWorkflowSchedules,db as logKpoPhaseComplete,ha as logRequestEvent,ba as markDispatchedJobRunning,Nb as markInboxMessageOpened,Ta as markJobBillingSettled,ua as markWorkflowRunRunning,pa as markWorkflowScheduleRan,i as migrate,j as migrateCoreSchemaForRelease,xb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,ma as patchWorkflowSchedule,m as putCode,o as putRefresh,Ib as reconcileBalanceMc,da as recordExtractUrlDispatchAttempt,Da as recordWorkflowWebhookDelivery,k as registerClient,Ka as renewDurablePaaLease,xa as replaceWorkflowArtifacts,Na as requeueExpiredDurablePaaJob,ub as resolveUserByMemorySubscription,v as retryStripeEvent,O as revokeApiKey,q as revokeRefresh,N as rotateApiKey,r as rotateRefresh,jb as setConcurrencyPackQuantity,kb as setConcurrencySubId,ob as setConnectedAccountBillingState,f as setDbForTesting,ib as setExtraConcurrencySlots,qb as setMemoryPlan,tb as setMemoryProvisioned,H as setPassword,hb as setStripeCustomerId,lb as setSubscriptionTier,G as setUserMemoryCreds,Eb as settleDebitMcIdempotent,Ob as softDeleteInboxMessage,V as startHarvestAttempt,y as stripeEventAlreadyProcessed,Z as updateHarvestAttemptProviderTelemetry,ab as updateKpoJobState,ga as upsertPageSnapshot,B as verifyPassword,h as verifySecuredMainOAuthSchema};

package/dist/{extract-bundle-INT6JMJP.js → extract-bundle-TMS6LLIT.js}

@@ -1,26 +1,26 @@
-import{c as We,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-TBTA7VVZ.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-LPEIHMYH.js";import"./chunk-XG6GCEUE.js";import"./chunk-M5QHXNFZ.js";import{c as Je,e as ze,f as Fe,g as He}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-CTX3NMZQ.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-TOJBPATJ.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
+import{c as Ge,h as ke,i as Qe,j as Ve,l as et}from"./chunk-IK5BG7MO.js";import"./chunk-XG6GCEUE.js";import{b as Ye,c as Xe,h as Ze,j as Ke}from"./chunk-POTKJRLE.js";import"./chunk-M5QHXNFZ.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import{c as Je}from"./chunk-RK2VCTZI.js";import"./chunk-ORB4RHCK.js";import"./chunk-TMB56NCA.js";import"./chunk-72YO6PKN.js";import"./chunk-6DTXIZY2.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-WJ4XFLS4.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as xt,readdirSync as kt,rmSync as _t,statSync as Pt,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Mt,join as a}from"path";import{tmpdir as It}from"os";import{createGzip as Ct}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as At}from"yazl";import Rt from"p-limit";import{createHash as _e}from"crypto";var gt=500,ht=.9,ft=1e4,yt=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function wt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function St(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function tt(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function bt(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(tt).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=tt(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
 
 `).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
 
-${C}`}),minimumPageCount:n,removedBlockSignatures:b.size,removedCharacters:w}}var Pe=class{parent;constructor(s){this.parent=Array.from({length:s},(n,c)=>c)}find(s){let n=this.parent[s];return n!==s&&(this.parent[s]=this.find(n)),this.parent[s]}union(s,n){let c=this.find(s),b=this.find(n);c!==b&&(this.parent[b]=c)}};async function tt(r,s={}){let n=r.filter(i=>i.bodyMarkdown.trim().length>0).slice(0,pt),c=Number.isFinite(s.threshold)?s.threshold:gt,b=Number.isFinite(s.maxPairs)?s.maxPairs:ht,w=Math.max(0,Math.min(1,c)),$=Math.max(1,Math.min(ft,Math.floor(b))),f=s.model??Ke(),M=s.dimensions??Qe(),I=s.embedTexts??Ve,A=St(n),C=[],_=new Map,T=[];for(let[i,p]of n.entries()){let y=p.contentHash||_e("sha256").update(p.bodyMarkdown).digest("hex"),k=_.get(y);k==null&&(k=C.length,_.set(y,k),C.push(A.texts[i])),T.push(k)}let j=[];for(let i=0;i<C.length;i+=64)j.push(...await I(C.slice(i,i+64)));if(j.length!==C.length)throw new Error(`Embedding provider returned ${j.length} vectors for ${C.length} unique page bodies.`);let X=T.map(i=>j[i]),le=new Pe(n.length),L=[],O=[];for(let i=0;i<n.length;i++)for(let p=i+1;p<n.length;p++){let y=yt(X[i],X[p]);L.push(y),!(y<w)&&(O.push({source:i,target:p,score:y}),le.union(i,p))}O.sort((i,p)=>p.score-i.score||n[i.source].url.localeCompare(n[p.source].url)||n[i.target].url.localeCompare(n[p.target].url));let v=[...L].sort((i,p)=>i-p),U=i=>v.length?ge(v[Math.floor((v.length-1)*i)],6):null,D=new Map;for(let i=0;i<n.length;i++){let p=le.find(i),y=D.get(p)??[];y.push(i),D.set(p,y)}let N=new Map,z=[...D.values()].filter(i=>i.length>1).sort((i,p)=>p.length-i.length||n[i[0]].url.localeCompare(n[p[0]].url)).map((i,p)=>{let y=`cluster-${String(p+1).padStart(3,"0")}`;for(let k of i)N.set(k,y);return{clusterId:y,pageCount:i.length,urls:i.map(k=>n[k].url).sort()}}),F=O.slice(0,$).map((i,p)=>{let y=n[i.source],k=n[i.target],q=ge(i.score,6);return{sourceUrl:y.url,sourceTitle:y.title,targetUrl:k.url,targetTitle:k.title,similarity:q,similarityPercent:ge(q*100,2),corpusPercentile:v.length<=1?100:ge((1-p/(v.length-1))*100,2),exactContentDuplicate:!!(y.contentHash&&y.contentHash===k.contentHash),sourceWordCount:y.wordCount,targetWordCount:k.wordCount,clusterId:N.get(i.source)??null}}),H=wt(n),ue=_e("sha256").update(JSON.stringify({corpusSha256:H,model:f,dimensions:M,threshold:w,maxPairs:$,boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures}})).digest("hex");return{generatedAt:new Date().toISOString(),provider:"jina",model:f,dimensions:M,threshold:w,requestedMaxPairs:$,comparedPages:n.length,possiblePairs:n.length*Math.max(0,n.length-1)/2,qualifyingPairs:O.length,returnedPairs:F.length,pairsTruncated:O.length>F.length,corpusSha256:H,analysisSha256:ue,scoreDistribution:{min:U(0),p25:U(.25),median:U(.5),p75:U(.75),p90:U(.9),max:U(1)},boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures,removedCharacters:A.removedCharacters},rows:F,clusters:z}}var Rt=25,Me=20,Ie=500,Tt=8,nt=10*1024*1024,rt=100*1024*1024,Ce=25e4;function oe(r){return r.split("#")[0].replace(/\/$/,"")}function st(r){let s=r.replace(/^www\./,"").toLowerCase(),n=s.split(".");return n.length<=2?s:n.slice(-2).join(".")}function vt(r,s){try{let n=new URL(r);return Pt(n.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`image-${s}`}catch{return`image-${s}`}}function jt(){let r=new Map;return s=>{let n=s.replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)||"page",c=r.get(n)??0;return r.set(n,c+1),c?`${n}-${c}`:n}}async function*Ae(r){let s=qe(),n=0;for(;;){let c=await s.execute({sql:`SELECT rowid AS cursor_id, page
+${C}`}),minimumPageCount:n,removedBlockSignatures:b.size,removedCharacters:w}}var Pe=class{parent;constructor(s){this.parent=Array.from({length:s},(n,c)=>c)}find(s){let n=this.parent[s];return n!==s&&(this.parent[s]=this.find(n)),this.parent[s]}union(s,n){let c=this.find(s),b=this.find(n);c!==b&&(this.parent[b]=c)}};async function nt(r,s={}){let n=r.filter(i=>i.bodyMarkdown.trim().length>0).slice(0,gt),c=Number.isFinite(s.threshold)?s.threshold:ht,b=Number.isFinite(s.maxPairs)?s.maxPairs:ft,w=Math.max(0,Math.min(1,c)),$=Math.max(1,Math.min(yt,Math.floor(b))),f=s.model??Qe(),M=s.dimensions??Ve(),I=s.embedTexts??et,A=bt(n),C=[],_=new Map,T=[];for(let[i,p]of n.entries()){let y=p.contentHash||_e("sha256").update(p.bodyMarkdown).digest("hex"),k=_.get(y);k==null&&(k=C.length,_.set(y,k),C.push(A.texts[i])),T.push(k)}let j=[];for(let i=0;i<C.length;i+=64)j.push(...await I(C.slice(i,i+64)));if(j.length!==C.length)throw new Error(`Embedding provider returned ${j.length} vectors for ${C.length} unique page bodies.`);let X=T.map(i=>j[i]),le=new Pe(n.length),L=[],O=[];for(let i=0;i<n.length;i++)for(let p=i+1;p<n.length;p++){let y=wt(X[i],X[p]);L.push(y),!(y<w)&&(O.push({source:i,target:p,score:y}),le.union(i,p))}O.sort((i,p)=>p.score-i.score||n[i.source].url.localeCompare(n[p.source].url)||n[i.target].url.localeCompare(n[p.target].url));let v=[...L].sort((i,p)=>i-p),U=i=>v.length?ge(v[Math.floor((v.length-1)*i)],6):null,D=new Map;for(let i=0;i<n.length;i++){let p=le.find(i),y=D.get(p)??[];y.push(i),D.set(p,y)}let N=new Map,z=[...D.values()].filter(i=>i.length>1).sort((i,p)=>p.length-i.length||n[i[0]].url.localeCompare(n[p[0]].url)).map((i,p)=>{let y=`cluster-${String(p+1).padStart(3,"0")}`;for(let k of i)N.set(k,y);return{clusterId:y,pageCount:i.length,urls:i.map(k=>n[k].url).sort()}}),F=O.slice(0,$).map((i,p)=>{let y=n[i.source],k=n[i.target],q=ge(i.score,6);return{sourceUrl:y.url,sourceTitle:y.title,targetUrl:k.url,targetTitle:k.title,similarity:q,similarityPercent:ge(q*100,2),corpusPercentile:v.length<=1?100:ge((1-p/(v.length-1))*100,2),exactContentDuplicate:!!(y.contentHash&&y.contentHash===k.contentHash),sourceWordCount:y.wordCount,targetWordCount:k.wordCount,clusterId:N.get(i.source)??null}}),H=St(n),ue=_e("sha256").update(JSON.stringify({corpusSha256:H,model:f,dimensions:M,threshold:w,maxPairs:$,boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures}})).digest("hex");return{generatedAt:new Date().toISOString(),provider:"jina",model:f,dimensions:M,threshold:w,requestedMaxPairs:$,comparedPages:n.length,possiblePairs:n.length*Math.max(0,n.length-1)/2,qualifyingPairs:O.length,returnedPairs:F.length,pairsTruncated:O.length>F.length,corpusSha256:H,analysisSha256:ue,scoreDistribution:{min:U(0),p25:U(.25),median:U(.5),p75:U(.75),p90:U(.9),max:U(1)},boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures,removedCharacters:A.removedCharacters},rows:F,clusters:z}}var Tt=25,Me=20,Ie=500,vt=8,rt=10*1024*1024,st=100*1024*1024,Ce=25e4;function oe(r){return r.split("#")[0].replace(/\/$/,"")}function it(r){let s=r.replace(/^www\./,"").toLowerCase(),n=s.split(".");return n.length<=2?s:n.slice(-2).join(".")}function jt(r,s){try{let n=new URL(r);return Mt(n.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`image-${s}`}catch{return`image-${s}`}}function Et(){let r=new Map;return s=>{let n=s.replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)||"page",c=r.get(n)??0;return r.set(n,c+1),c?`${n}-${c}`:n}}async function*Ae(r){let s=qe(),n=0;for(;;){let c=await s.execute({sql:`SELECT rowid AS cursor_id, page
             FROM site_extract_pages
             WHERE job_id = ? AND rowid > ?
             ORDER BY rowid
-            LIMIT ${Rt}`,args:[r,n]});if(!c.rows.length)return;if(n=Number(c.rows.at(-1)?.cursor_id??n),!Number.isSafeInteger(n)||n<=0)throw new Error("site extract page cursor is invalid");yield c.rows.map(b=>JSON.parse(String(b.page)))}}function Et(r){let s=r==null?"":String(r);return/[",\r\n]/.test(s)?`"${s.replace(/"/g,'""')}"`:s}function $t(r){return{source_url:r.sourceUrl,source_title:r.sourceTitle,target_url:r.targetUrl,target_title:r.targetTitle,similarity:r.similarity,similarity_percent:r.similarityPercent,corpus_percentile:r.corpusPercentile,exact_content_duplicate:r.exactContentDuplicate,source_word_count:r.sourceWordCount,target_word_count:r.targetWordCount,cluster_id:r.clusterId}}async function rn(r,s={}){let n=a(Mt(),`extract-bundle-${r.id}-${Date.now()}`),c=r.options.downloadImages===!0;if(r.userId==null)throw new Error("site extract artifact owner is missing");let b=Xe(String(r.userId));ie(n,{recursive:!0}),c&&ie(a(n,"images"),{recursive:!0});let w=r.options.captureRenderedDom===!0,$=r.options.semanticSimilarity===!0;w&&ie(a(n,"rendered-dom"),{recursive:!0});try{let f=[],M=[],I=[],A=jt(),C=[],_=r.options.waybackTimeline,T=new Map;for await(let t of Ae(r.id))for(let e of t){T.set(oe(e.url),e.status),f.push({url:e.url,status:e.status,title:e.title,titleLength:e.titleLength,titlePixels:e.titlePixels,metaDescription:e.metaDescription,metaDescLength:e.metaDescLength,h1:e.h1,h1_2:e.h1_2,h2Count:e.h2Count,extractionStatus:e.extractionStatus,indexable:e.indexable,indexabilityReason:e.indexabilityReason,canonicalUrl:e.canonicalUrl?e.url:null,wordCount:e.wordCount,contentHash:e.contentHash,imagesMissingAlt:e.imagesMissingAlt,schemaTypes:e.schemaTypes.length>0?["present"]:[],outlinks:[]});let o=e.pageId??G("sha256").update(e.archivedUrl??e.url).digest("hex"),x=a(n,"pages",o);ie(x,{recursive:!0});let m=e.contentRef?await b(e.contentRef):null,d=m?.html??null,l=m?.markdown??e.bodyMarkdown??null;if($&&e.extractionStatus==="successful"&&l?.trim()&&M.push({url:e.url,title:e.title,bodyMarkdown:l,contentHash:e.contentHash,wordCount:e.wordCount}),w&&d!=null){let Ne=`rendered-dom/${A(e.url)}.html.txt`;g(a(n,Ne),d),I.push({url:e.url,path:Ne,bytes:Buffer.byteLength(d),truncated:e.renderedDomTruncated===!0,sanitized:e.renderedDomSanitized===!0})}let u=d==null?null:`pages/${o}/page.html`,h=l==null?null:`pages/${o}/page.md`,S=l==null||!e.archiveRequestedMonth?null:`pages/${e.archiveRequestedMonth}/${o}.md`;d!=null&&u&&g(a(n,u),d),l!=null&&h&&g(a(n,h),l),l!=null&&S&&(ie(a(n,"pages",e.archiveRequestedMonth),{recursive:!0}),g(a(n,S),l));let{bodyMarkdown:R,contentRef:te,discoveryLinks:ne,...dt}=e,re={version:"site-export-page.v1",...dt,pageId:o,content:{htmlPath:u,markdownPath:h,htmlBytes:d==null?0:Buffer.byteLength(d),markdownBytes:l==null?0:Buffer.byteLength(l),htmlSha256:d==null?null:G("sha256").update(d).digest("hex"),markdownSha256:l==null?null:G("sha256").update(l).digest("hex")}},Le=JSON.stringify(re,null,2),Oe=`pages/${o}/page.json`;g(a(n,Oe),Le),C.push({pageId:o,url:e.archivedUrl??e.url,jsonPath:Oe,htmlPath:u,markdownPath:h,legacyMarkdownPath:S,jsonSha256:G("sha256").update(Le).digest("hex"),htmlSha256:re.content.htmlSha256,markdownSha256:re.content.markdownSha256,htmlBytes:re.content.htmlBytes,markdownBytes:re.content.markdownBytes})}let j=It(),X=se(a(n,"links.jsonl.gz"));j.pipe(X);let le=async t=>{j.write(t)||await Y(j,"drain")},L=se(a(n,"schema.jsonl")),O=async t=>{L.write(t)||await Y(L,"drain")},v=se(a(n,"failures.jsonl")),U=async t=>{v.write(t)||await Y(v,"drain")},D=new Map,N=new Map,z=new Map,F=new Map,H=new Map,ue=new Set,i=0,p=0,y=0,k=0,q=0,he=!1,fe="";try{fe=st(new URL(r.startUrl).hostname)}catch{fe=""}let it=At(Tt),Re=[],Te=[],ve=[],Z=0,ce=0,ye=0,at=t=>ye+t>rt?!1:(ye+=t,!0);for await(let t of Ae(r.id))for(let e of t){if(e.schema?.length&&await O(JSON.stringify({url:e.url,schema:e.schema})+`
+            LIMIT ${Tt}`,args:[r,n]});if(!c.rows.length)return;if(n=Number(c.rows.at(-1)?.cursor_id??n),!Number.isSafeInteger(n)||n<=0)throw new Error("site extract page cursor is invalid");yield c.rows.map(b=>JSON.parse(String(b.page)))}}function $t(r){let s=r==null?"":String(r);return/[",\r\n]/.test(s)?`"${s.replace(/"/g,'""')}"`:s}function Ut(r){return{source_url:r.sourceUrl,source_title:r.sourceTitle,target_url:r.targetUrl,target_title:r.targetTitle,similarity:r.similarity,similarity_percent:r.similarityPercent,corpus_percentile:r.corpusPercentile,exact_content_duplicate:r.exactContentDuplicate,source_word_count:r.sourceWordCount,target_word_count:r.targetWordCount,cluster_id:r.clusterId}}async function an(r,s={}){let n=a(It(),`extract-bundle-${r.id}-${Date.now()}`),c=r.options.downloadImages===!0;if(r.userId==null)throw new Error("site extract artifact owner is missing");let b=Ze(String(r.userId));ie(n,{recursive:!0}),c&&ie(a(n,"images"),{recursive:!0});let w=r.options.captureRenderedDom===!0,$=r.options.semanticSimilarity===!0;w&&ie(a(n,"rendered-dom"),{recursive:!0});try{let f=[],M=[],I=[],A=Et(),C=[],_=r.options.waybackTimeline,T=new Map;for await(let t of Ae(r.id))for(let e of t){T.set(oe(e.url),e.status),f.push({url:e.url,status:e.status,title:e.title,titleLength:e.titleLength,titlePixels:e.titlePixels,metaDescription:e.metaDescription,metaDescLength:e.metaDescLength,h1:e.h1,h1_2:e.h1_2,h2Count:e.h2Count,extractionStatus:e.extractionStatus,indexable:e.indexable,indexabilityReason:e.indexabilityReason,canonicalUrl:e.canonicalUrl?e.url:null,wordCount:e.wordCount,contentHash:e.contentHash,imagesMissingAlt:e.imagesMissingAlt,schemaTypes:e.schemaTypes.length>0?["present"]:[],outlinks:[]});let o=e.pageId??G("sha256").update(e.archivedUrl??e.url).digest("hex"),x=a(n,"pages",o);ie(x,{recursive:!0});let m=e.contentRef?await b(e.contentRef):null,d=m?.html??null,l=m?.markdown??e.bodyMarkdown??null;if($&&e.extractionStatus==="successful"&&l?.trim()&&M.push({url:e.url,title:e.title,bodyMarkdown:l,contentHash:e.contentHash,wordCount:e.wordCount}),w&&d!=null){let Ne=`rendered-dom/${A(e.url)}.html.txt`;g(a(n,Ne),d),I.push({url:e.url,path:Ne,bytes:Buffer.byteLength(d),truncated:e.renderedDomTruncated===!0,sanitized:e.renderedDomSanitized===!0})}let u=d==null?null:`pages/${o}/page.html`,h=l==null?null:`pages/${o}/page.md`,S=l==null||!e.archiveRequestedMonth?null:`pages/${e.archiveRequestedMonth}/${o}.md`;d!=null&&u&&g(a(n,u),d),l!=null&&h&&g(a(n,h),l),l!=null&&S&&(ie(a(n,"pages",e.archiveRequestedMonth),{recursive:!0}),g(a(n,S),l));let{bodyMarkdown:R,contentRef:te,discoveryLinks:ne,...pt}=e,re={version:"site-export-page.v1",...pt,pageId:o,content:{htmlPath:u,markdownPath:h,htmlBytes:d==null?0:Buffer.byteLength(d),markdownBytes:l==null?0:Buffer.byteLength(l),htmlSha256:d==null?null:G("sha256").update(d).digest("hex"),markdownSha256:l==null?null:G("sha256").update(l).digest("hex")}},Le=JSON.stringify(re,null,2),Oe=`pages/${o}/page.json`;g(a(n,Oe),Le),C.push({pageId:o,url:e.archivedUrl??e.url,jsonPath:Oe,htmlPath:u,markdownPath:h,legacyMarkdownPath:S,jsonSha256:G("sha256").update(Le).digest("hex"),htmlSha256:re.content.htmlSha256,markdownSha256:re.content.markdownSha256,htmlBytes:re.content.htmlBytes,markdownBytes:re.content.markdownBytes})}let j=Ct(),X=se(a(n,"links.jsonl.gz"));j.pipe(X);let le=async t=>{j.write(t)||await Y(j,"drain")},L=se(a(n,"schema.jsonl")),O=async t=>{L.write(t)||await Y(L,"drain")},v=se(a(n,"failures.jsonl")),U=async t=>{v.write(t)||await Y(v,"drain")},D=new Map,N=new Map,z=new Map,F=new Map,H=new Map,ue=new Set,i=0,p=0,y=0,k=0,q=0,he=!1,fe="";try{fe=it(new URL(r.startUrl).hostname)}catch{fe=""}let at=Rt(vt),Re=[],Te=[],ve=[],Z=0,ce=0,ye=0,ot=t=>ye+t>st?!1:(ye+=t,!0);for await(let t of Ae(r.id))for(let e of t){if(e.schema?.length&&await O(JSON.stringify({url:e.url,schema:e.schema})+`
 `),e.extractionStatus==="failed"){let{failureCode:m,failureReason:d}=ke(e.failureCode,e.failureReason);await U(JSON.stringify({url:e.url,status:e.status,via:e.via,failureCode:m,failureReason:d,fetchAttempts:e.fetchAttempts??[]})+`
-`)}if(e.imageLinks?.length)for(let[m,d]of e.imageLinks.entries()){let l=G("sha256").update(`${e.url}\0${d}`).digest("hex"),u={imageId:l,sourcePage:e.url,sourceUrl:d,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Te.push(u),!c)continue;if(m>=Me){u.status="skipped_page_limit",u.failureCode="page_image_limit",u.failureReason=`Only the first ${Me} image candidates are downloaded per page.`;continue}if(Z>=Ie){u.status="skipped_site_limit",u.failureCode="site_image_limit",u.failureReason=`Only the first ${Ie} image candidates are downloaded per site.`;continue}Z++;let h=`${l}-${vt(d,m)}`,S=a(n,"images");Re.push(it(()=>We(d,S,h,{expectedType:"image",maxBytes:nt,consumeBytes:at})).then(async R=>{let te=bt(R.savedPath),ne=await Ye({ownerId:String(r.userId),jobId:r.id,imageId:l,filename:h,contentType:R.mimeType??"application/octet-stream",content:te,sourceUrl:d,sourcePage:e.url});ve.push(ne),u.status="downloaded",u.path=`images/${h}`,u.artifactId=ne.key,u.mimeType=R.mimeType,u.bytes=R.sizeBytes,u.sha256=G("sha256").update(te).digest("hex")},R=>{ce++,u.status="failed",u.failureCode="image_download_failed",u.failureReason=(R instanceof Error?R.message:String(R)).replace(/\s+/g," ").slice(0,500)}))}let o=oe(e.url),x={int:0,ext:0};for(let m of e.outlinks??[]){let d=(m.rel??"").split(/\s+/).includes("nofollow"),l=oe(m.href),u=T.get(l)??null;if(await le(JSON.stringify({from:e.url,to:m.href,anchor:m.anchor,rel:m.rel,internal:m.internal,nofollow:d,targetStatus:u})+`
-`),k++,q>=Ce){he=!0;continue}if(q++,m.internal){if(x.int++,i++,D.has(l)||D.set(l,new Set),D.get(l).add(o),m.anchor){N.has(l)||N.set(l,new Map);let h=N.get(l);h.set(m.anchor,(h.get(m.anchor)??0)+1)}!d&&(T.get(l)===200||!T.has(l))&&(z.has(o)||z.set(o,new Set),z.get(o).add(l)),u!=null&&u>=400&&(y++,ue.add(e.url))}else{x.ext++;let h="";try{h=st(new URL(m.href).hostname)}catch{continue}if(!h||h===fe)continue;p++;let S=H.get(h)??{links:0,nofollow:0,pages:new Set};S.links++,d&&S.nofollow++,S.pages.add(e.url),H.set(h,S)}}F.set(e.url,x)}let ot=Promise.all([ae(X),ae(L),ae(v)]);j.end(),L.end(),v.end(),await ot,await Promise.all(Re);let W=new Map,we=oe(r.startUrl);W.set(we,0);let Se=[we];for(;Se.length>0;){let t=Se.shift(),e=W.get(t);for(let o of z.get(t)??[])W.has(o)||(W.set(o,e+1),Se.push(o))}let K=new Map;for(let t of f){let e=oe(t.url),o=D.get(e)??new Set,x=N.get(e),m=x?[...x.entries()].sort((l,u)=>u[1]-l[1]).slice(0,3).map(l=>l[0]):[],d=F.get(t.url)??{int:0,ext:0};K.set(t.url,{url:t.url,inlinks:o.size,uniqueInlinks:o.size,outlinksInternal:d.int,outlinksExternal:d.ext,crawlDepth:W.has(e)?W.get(e):null,orphan:o.size===0&&e!==we,topAnchors:m})}let je=Fe(f,K,ue),Q=He(r.startUrl,f,je,K),V=Ze(r),J={jobId:r.id,startUrl:r.startUrl,status:r.successfulUrls===0?"failed":r.failedUrls>0||r.remainingUrls>0||V.creditTruncated?"partial":"complete",...V,discovered:r.totalUrls,attempted:r.attemptedUrls,successful:r.successfulUrls,failed:r.failedUrls,remaining:r.remainingUrls,linkAnalysis:{truncated:he,analyzedEdges:q,streamedEdges:k,edgeCap:Ce},generatedAt:new Date().toISOString()};Q=["## Crawl delivery",`- Discovered: ${J.discovered}`,`- Attempted: ${J.attempted}`,`- Successful: ${J.successful}`,`- Failed: ${J.failed}`,`- Remaining: ${J.remaining}`,`- Requested page cap: ${V.requestedMaxPages}`,`- Funded page cap: ${V.effectiveMaxPages}`,`- Credit-truncated: ${V.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${he?` (capped at ${Ce})`:""}`,"",Q].join(`
+`)}if(e.imageLinks?.length)for(let[m,d]of e.imageLinks.entries()){let l=G("sha256").update(`${e.url}\0${d}`).digest("hex"),u={imageId:l,sourcePage:e.url,sourceUrl:d,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Te.push(u),!c)continue;if(m>=Me){u.status="skipped_page_limit",u.failureCode="page_image_limit",u.failureReason=`Only the first ${Me} image candidates are downloaded per page.`;continue}if(Z>=Ie){u.status="skipped_site_limit",u.failureCode="site_image_limit",u.failureReason=`Only the first ${Ie} image candidates are downloaded per site.`;continue}Z++;let h=`${l}-${jt(d,m)}`,S=a(n,"images");Re.push(at(()=>Ge(d,S,h,{expectedType:"image",maxBytes:rt,consumeBytes:ot})).then(async R=>{let te=xt(R.savedPath),ne=await Xe({ownerId:String(r.userId),jobId:r.id,imageId:l,filename:h,contentType:R.mimeType??"application/octet-stream",content:te,sourceUrl:d,sourcePage:e.url});ve.push(ne),u.status="downloaded",u.path=`images/${h}`,u.artifactId=ne.key,u.mimeType=R.mimeType,u.bytes=R.sizeBytes,u.sha256=G("sha256").update(te).digest("hex")},R=>{ce++,u.status="failed",u.failureCode="image_download_failed",u.failureReason=(R instanceof Error?R.message:String(R)).replace(/\s+/g," ").slice(0,500)}))}let o=oe(e.url),x={int:0,ext:0};for(let m of e.outlinks??[]){let d=(m.rel??"").split(/\s+/).includes("nofollow"),l=oe(m.href),u=T.get(l)??null;if(await le(JSON.stringify({from:e.url,to:m.href,anchor:m.anchor,rel:m.rel,internal:m.internal,nofollow:d,targetStatus:u})+`
+`),k++,q>=Ce){he=!0;continue}if(q++,m.internal){if(x.int++,i++,D.has(l)||D.set(l,new Set),D.get(l).add(o),m.anchor){N.has(l)||N.set(l,new Map);let h=N.get(l);h.set(m.anchor,(h.get(m.anchor)??0)+1)}!d&&(T.get(l)===200||!T.has(l))&&(z.has(o)||z.set(o,new Set),z.get(o).add(l)),u!=null&&u>=400&&(y++,ue.add(e.url))}else{x.ext++;let h="";try{h=it(new URL(m.href).hostname)}catch{continue}if(!h||h===fe)continue;p++;let S=H.get(h)??{links:0,nofollow:0,pages:new Set};S.links++,d&&S.nofollow++,S.pages.add(e.url),H.set(h,S)}}F.set(e.url,x)}let lt=Promise.all([ae(X),ae(L),ae(v)]);j.end(),L.end(),v.end(),await lt,await Promise.all(Re);let W=new Map,we=oe(r.startUrl);W.set(we,0);let Se=[we];for(;Se.length>0;){let t=Se.shift(),e=W.get(t);for(let o of z.get(t)??[])W.has(o)||(W.set(o,e+1),Se.push(o))}let K=new Map;for(let t of f){let e=oe(t.url),o=D.get(e)??new Set,x=N.get(e),m=x?[...x.entries()].sort((l,u)=>u[1]-l[1]).slice(0,3).map(l=>l[0]):[],d=F.get(t.url)??{int:0,ext:0};K.set(t.url,{url:t.url,inlinks:o.size,uniqueInlinks:o.size,outlinksInternal:d.int,outlinksExternal:d.ext,crawlDepth:W.has(e)?W.get(e):null,orphan:o.size===0&&e!==we,topAnchors:m})}let je=He(f,K,ue),Q=We(r.startUrl,f,je,K),V=Ke(r),J={jobId:r.id,startUrl:r.startUrl,status:r.successfulUrls===0?"failed":r.failedUrls>0||r.remainingUrls>0||V.creditTruncated?"partial":"complete",...V,discovered:r.totalUrls,attempted:r.attemptedUrls,successful:r.successfulUrls,failed:r.failedUrls,remaining:r.remainingUrls,linkAnalysis:{truncated:he,analyzedEdges:q,streamedEdges:k,edgeCap:Ce},generatedAt:new Date().toISOString()};Q=["## Crawl delivery",`- Discovered: ${J.discovered}`,`- Attempted: ${J.attempted}`,`- Successful: ${J.successful}`,`- Failed: ${J.failed}`,`- Remaining: ${J.remaining}`,`- Requested page cap: ${V.requestedMaxPages}`,`- Funded page cap: ${V.effectiveMaxPages}`,`- Credit-truncated: ${V.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${he?` (capped at ${Ce})`:""}`,"",Q].join(`
 `),s.imageAudit&&(Q+=`
 
-${Je(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},$e=0,Ue=0;for(let t of me)$e+=t.inlinks,Ue+=t.outlinksInternal+t.outlinksExternal,t.inlinks===0?ee.zero++:t.inlinks<=2?ee.oneToTwo++:t.inlinks<=10?ee.threeToTen++:ee.elevenPlus++;let be=[...H.entries()].map(([t,e])=>({domain:t,links:e.links,nofollow:e.nofollow,pages:e.pages.size})).sort((t,e)=>e.links-t.links),xe={summary:{internal:{totalLinks:i,pages:f.length,orphans:me.filter(t=>t.orphan).length,brokenInternal:y,avgInlinks:f.length?Ee($e/f.length):0,avgOutlinks:f.length?Ee(Ue/f.length):0,distribution:ee,topByInlinks:[...me].sort((t,e)=>e.inlinks-t.inlinks).slice(0,20).map(t=>({url:t.url,inlinks:t.inlinks,outlinksInternal:t.outlinksInternal,outlinksExternal:t.outlinksExternal}))},external:{totalLinks:p,uniqueDomains:be.length,topDomains:be.slice(0,20)}},externalDomains:be},de=se(a(n,"pages.jsonl")),lt=new Map(C.map(t=>[t.pageId,t])),E=_?se(a(n,"capture-matrix.jsonl")):null,De=new Set,Be=(t,e)=>{let o=e;try{o=new URL(e).href}catch{}return`${t??""}\0${o}`};for await(let t of Ae(r.id))for(let e of t){let o=e.extractionStatus==="failed"?ke(e.failureCode,e.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:x,schema:m,outlinks:d,contentRef:l,...u}=e,h=K.get(e.url),S=e.pageId?lt.get(e.pageId):void 0,R={...u,...e.extractionStatus==="failed"?o:{},representations:S?{jsonPath:S.jsonPath,htmlPath:S.htmlPath,markdownPath:S.markdownPath,jsonSha256:S.jsonSha256,htmlSha256:S.htmlSha256,markdownSha256:S.markdownSha256,htmlBytes:S.htmlBytes,markdownBytes:S.markdownBytes}:null,inlinks:h?.inlinks??0,crawlDepth:h?.crawlDepth??null,orphan:h?.orphan??!1};if(de.write(JSON.stringify(R)+`
+${ze(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},$e=0,Ue=0;for(let t of me)$e+=t.inlinks,Ue+=t.outlinksInternal+t.outlinksExternal,t.inlinks===0?ee.zero++:t.inlinks<=2?ee.oneToTwo++:t.inlinks<=10?ee.threeToTen++:ee.elevenPlus++;let be=[...H.entries()].map(([t,e])=>({domain:t,links:e.links,nofollow:e.nofollow,pages:e.pages.size})).sort((t,e)=>e.links-t.links),xe={summary:{internal:{totalLinks:i,pages:f.length,orphans:me.filter(t=>t.orphan).length,brokenInternal:y,avgInlinks:f.length?Ee($e/f.length):0,avgOutlinks:f.length?Ee(Ue/f.length):0,distribution:ee,topByInlinks:[...me].sort((t,e)=>e.inlinks-t.inlinks).slice(0,20).map(t=>({url:t.url,inlinks:t.inlinks,outlinksInternal:t.outlinksInternal,outlinksExternal:t.outlinksExternal}))},external:{totalLinks:p,uniqueDomains:be.length,topDomains:be.slice(0,20)}},externalDomains:be},de=se(a(n,"pages.jsonl")),ut=new Map(C.map(t=>[t.pageId,t])),E=_?se(a(n,"capture-matrix.jsonl")):null,De=new Set,Be=(t,e)=>{let o=e;try{o=new URL(e).href}catch{}return`${t??""}\0${o}`};for await(let t of Ae(r.id))for(let e of t){let o=e.extractionStatus==="failed"?ke(e.failureCode,e.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:x,schema:m,outlinks:d,contentRef:l,...u}=e,h=K.get(e.url),S=e.pageId?ut.get(e.pageId):void 0,R={...u,...e.extractionStatus==="failed"?o:{},representations:S?{jsonPath:S.jsonPath,htmlPath:S.htmlPath,markdownPath:S.markdownPath,jsonSha256:S.jsonSha256,htmlSha256:S.htmlSha256,markdownSha256:S.markdownSha256,htmlBytes:S.htmlBytes,markdownBytes:S.markdownBytes}:null,inlinks:h?.inlinks??0,crawlDepth:h?.crawlDepth??null,orphan:h?.orphan??!1};if(de.write(JSON.stringify(R)+`
 `)||await Y(de,"drain"),E){let te=Be(e.archiveRequestedMonth,e.originalUrl??e.url);De.add(te);let ne={requestedMonth:e.archiveRequestedMonth??null,captureTimestamp:e.archiveTimestamp??null,originalUrl:e.originalUrl??e.url,archivedUrl:e.archivedUrl??null,status:e.extractionStatus==="successful"?"captured":"failed",httpStatus:e.status,contentHash:e.contentHash||null,wordCount:e.wordCount,failureCode:o.failureCode,failureReason:o.failureReason};E.write(JSON.stringify(ne)+`
 `)||await Y(E,"drain")}}if(E&&_?.timeline?.urls){let t=_.timeline,e=t.urls,o=(()=>{if(t.months?.length)return[...new Set(t.months)].sort();if(!t.from||!t.to)return[];let x=[],m=new Date(`${t.from}-01T00:00:00Z`),d=new Date(`${t.to}-01T00:00:00Z`);for(;m<=d&&x.length<60;)x.push(`${m.getUTCFullYear()}-${String(m.getUTCMonth()+1).padStart(2,"0")}`),m.setUTCMonth(m.getUTCMonth()+(t.intervalMonths??1));return x})();for(let x of o)for(let m of e)De.has(Be(x,m))||E.write(JSON.stringify({requestedMonth:x,captureTimestamp:null,originalUrl:m,archivedUrl:null,status:"missing",httpStatus:null,contentHash:null,wordCount:0,failureCode:"wayback_capture_missing",failureReason:"No successful HTML capture was available in the requested month."})+`
-`)||await Y(E,"drain")}let ut=ae(de),ct=E?ae(E):Promise.resolve();if(de.end(),E?.end(),await Promise.all([ut,ct]),w&&g(a(n,"rendered-dom","manifest.jsonl"),I.map(t=>JSON.stringify(t)).join(`
-`)),$){let t=await tt(M,{threshold:Number(r.options.similarityThreshold??void 0),maxPairs:Number(r.options.similarityMaxPairs??void 0),embedTexts:s.embedSimilarityTexts}),e=t.rows.map($t),o=e.length>0?Object.keys(e[0]):["source_url","source_title","target_url","target_title","similarity","similarity_percent","corpus_percentile","exact_content_duplicate","source_word_count","target_word_count","cluster_id"],x=[o.join(","),...e.map(u=>o.map(h=>Et(u[h]??null)).join(","))].join(`
+`)||await Y(E,"drain")}let ct=ae(de),mt=E?ae(E):Promise.resolve();if(de.end(),E?.end(),await Promise.all([ct,mt]),w&&g(a(n,"rendered-dom","manifest.jsonl"),I.map(t=>JSON.stringify(t)).join(`
+`)),$){let t=await nt(M,{threshold:Number(r.options.similarityThreshold??void 0),maxPairs:Number(r.options.similarityMaxPairs??void 0),embedTexts:s.embedSimilarityTexts}),e=t.rows.map(Ut),o=e.length>0?Object.keys(e[0]):["source_url","source_title","target_url","target_title","similarity","similarity_percent","corpus_percentile","exact_content_duplicate","source_word_count","target_word_count","cluster_id"],x=[o.join(","),...e.map(u=>o.map(h=>$t(u[h]??null)).join(","))].join(`
 `)+`
 `,{rows:m,clusters:d,...l}=t;g(a(n,"similarity-table.csv"),x),g(a(n,"similarity.jsonl"),e.map(u=>JSON.stringify(u)).join(`
 `)),g(a(n,"similarity-table-schema.json"),JSON.stringify({tableNameSuggestion:`site_similarity_${new URL(r.startUrl).hostname.replace(/[^a-z0-9]+/gi,"_").replace(/^_+|_+$/g,"").toLowerCase()}`,defaultSort:[{column:"similarity",direction:"desc"}],columns:{source_url:"text",source_title:"text",target_url:"text",target_title:"text",similarity:"number",similarity_percent:"number",corpus_percentile:"number",exact_content_duplicate:"boolean",source_word_count:"integer",target_word_count:"integer",cluster_id:"text"}},null,2)),g(a(n,"similarity-summary.json"),JSON.stringify(l,null,2)),g(a(n,"content-clusters.json"),JSON.stringify(t.clusters,null,2)),Q+=["","## Rendered content similarity",`- Compared pages: ${t.comparedPages}`,`- Raw cosine threshold: ${t.threshold}`,`- Corpus score distribution: min ${t.scoreDistribution.min??"n/a"} / median ${t.scoreDistribution.median??"n/a"} / p90 ${t.scoreDistribution.p90??"n/a"} / max ${t.scoreDistribution.max??"n/a"}`,`- Qualifying pairs: ${t.qualifyingPairs}`,`- Returned table rows: ${t.returnedPairs}${t.pairsTruncated?" (capped)":""}`,`- Multi-page clusters: ${t.clusters.length}`,`- Embedding model: ${t.model} (${t.dimensions} dimensions)`,`- Corpus boilerplate removed: ${t.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${t.boilerplateRemoval.removedCharacters} characters`,`- Corpus SHA-256: ${t.corpusSha256}`].join(`
 `)}g(a(n,"report.md"),Q),g(a(n,"crawl-summary.json"),JSON.stringify(J,null,2)),g(a(n,"images-manifest.jsonl"),Te.map(t=>JSON.stringify(t)).join(`
 `)),g(a(n,"manifest.json"),JSON.stringify({version:"site-export.v1",jobId:r.id,startUrl:r.startUrl,generatedAt:J.generatedAt,outputProfile:{json:!0,html:!0,markdown:!0,imageManifest:!0,imageBytes:c},pages:[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId)),imagesManifestPath:"images-manifest.jsonl",download:"This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."},null,2)),_&&g(a(n,"wayback-manifest.json"),JSON.stringify({rootUrl:_.rootUrl??r.startUrl,requested:_.timeline??null,maxPagesPerSnapshot:_.maxPagesPerSnapshot??null,captureMatrix:"capture-matrix.jsonl"},null,2)),g(a(n,"issues.json"),JSON.stringify(je,null,2)),g(a(n,"link-metrics.jsonl"),me.map(t=>JSON.stringify(t)).join(`
-`)),g(a(n,"link-report.md"),ze(xe)),g(a(n,"links-summary.json"),JSON.stringify(xe.summary,null,2)),g(a(n,"external-domains.json"),JSON.stringify(xe.externalDomains,null,2)),s.imageAudit&&(g(a(n,"images.jsonl"),s.imageAudit.rows.map(t=>JSON.stringify(t)).join(`
-`)),g(a(n,"images-summary.json"),JSON.stringify(s.imageAudit.summary,null,2))),s.seoAudit&&g(a(n,"seo-audit.json"),JSON.stringify(s.seoAudit,null,2)),s.branding&&g(a(n,"branding.json"),JSON.stringify(s.branding,null,2)),c&&g(a(n,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-ce,failed:ce,downloadedBytes:ye,perPageCap:Me,perSiteCap:Ie,perFileByteCap:nt,totalByteCap:rt},null,2));let P=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(w){P.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let t of I)P.push({name:t.path,type:"text/plain"})}$&&(P.push({name:"similarity-table.csv",type:"text/csv"}),P.push({name:"similarity.jsonl",type:"application/x-ndjson"}),P.push({name:"similarity-table-schema.json",type:"application/json"}),P.push({name:"similarity-summary.json",type:"application/json"}),P.push({name:"content-clusters.json",type:"application/json"})),_&&(P.push({name:"wayback-manifest.json",type:"application/json"}),P.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),s.imageAudit&&(P.push({name:"images.jsonl",type:"application/x-ndjson"}),P.push({name:"images-summary.json",type:"application/json"})),s.seoAudit&&P.push({name:"seo-audit.json",type:"application/json"}),s.branding&&P.push({name:"branding.json",type:"application/json"}),c&&P.push({name:"images-download-summary.json",type:"application/json"});let B=new Ct,mt=Ge({ownerId:String(r.userId),jobId:r.id,createdAt:new Date,content:B.outputStream});for(let t of P)B.addFile(a(n,t.name),t.name);let pe=[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId));for(let t of pe)t.markdownPath&&B.addFile(a(n,t.markdownPath),t.markdownPath);for(let t of pe)t.legacyMarkdownPath&&B.addFile(a(n,t.legacyMarkdownPath),t.legacyMarkdownPath);for(let t of pe)t.htmlPath&&B.addFile(a(n,t.htmlPath),t.htmlPath);for(let t of pe)B.addFile(a(n,t.jsonPath),t.jsonPath);if(c&&Z>ce){let t=a(n,"images");for(let e of xt(t,{recursive:!0})){let o=a(t,e);_t(o).isFile()&&B.addFile(o,`images/${e.split("\\").join("/")}`)}}return B.end(),[await mt,...ve]}finally{kt(n,{recursive:!0,force:!0})}}export{Ce as MAX_ANALYZED_LINK_EDGES,Rt as SITE_EXTRACT_PAGE_CHUNK,rn as assembleExtractArtifacts};
+`)),g(a(n,"link-report.md"),Fe(xe)),g(a(n,"links-summary.json"),JSON.stringify(xe.summary,null,2)),g(a(n,"external-domains.json"),JSON.stringify(xe.externalDomains,null,2)),s.imageAudit&&(g(a(n,"images.jsonl"),s.imageAudit.rows.map(t=>JSON.stringify(t)).join(`
+`)),g(a(n,"images-summary.json"),JSON.stringify(s.imageAudit.summary,null,2))),s.seoAudit&&g(a(n,"seo-audit.json"),JSON.stringify(s.seoAudit,null,2)),s.branding&&g(a(n,"branding.json"),JSON.stringify(s.branding,null,2)),c&&g(a(n,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-ce,failed:ce,downloadedBytes:ye,perPageCap:Me,perSiteCap:Ie,perFileByteCap:rt,totalByteCap:st},null,2));let P=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(w){P.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let t of I)P.push({name:t.path,type:"text/plain"})}$&&(P.push({name:"similarity-table.csv",type:"text/csv"}),P.push({name:"similarity.jsonl",type:"application/x-ndjson"}),P.push({name:"similarity-table-schema.json",type:"application/json"}),P.push({name:"similarity-summary.json",type:"application/json"}),P.push({name:"content-clusters.json",type:"application/json"})),_&&(P.push({name:"wayback-manifest.json",type:"application/json"}),P.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),s.imageAudit&&(P.push({name:"images.jsonl",type:"application/x-ndjson"}),P.push({name:"images-summary.json",type:"application/json"})),s.seoAudit&&P.push({name:"seo-audit.json",type:"application/json"}),s.branding&&P.push({name:"branding.json",type:"application/json"}),c&&P.push({name:"images-download-summary.json",type:"application/json"});let B=new At,dt=Ye({ownerId:String(r.userId),jobId:r.id,createdAt:new Date,content:B.outputStream});for(let t of P)B.addFile(a(n,t.name),t.name);let pe=[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId));for(let t of pe)t.markdownPath&&B.addFile(a(n,t.markdownPath),t.markdownPath);for(let t of pe)t.legacyMarkdownPath&&B.addFile(a(n,t.legacyMarkdownPath),t.legacyMarkdownPath);for(let t of pe)t.htmlPath&&B.addFile(a(n,t.htmlPath),t.htmlPath);for(let t of pe)B.addFile(a(n,t.jsonPath),t.jsonPath);if(c&&Z>ce){let t=a(n,"images");for(let e of kt(t,{recursive:!0})){let o=a(t,e);Pt(o).isFile()&&B.addFile(o,`images/${e.split("\\").join("/")}`)}}return B.end(),[await Je("export_packaging",()=>dt),...ve]}finally{_t(n,{recursive:!0,force:!0})}}export{Ce as MAX_ANALYZED_LINK_EDGES,Tt as SITE_EXTRACT_PAGE_CHUNK,an as assembleExtractArtifacts};

package/dist/gmail-service-LXBBOEIQ.js

@@ -0,0 +1 @@
+import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-VEP42LGR.js";import"./chunk-C3QWOZUH.js";import"./chunk-QTDLZTQ7.js";import"./chunk-HUV2WTRW.js";import"./chunk-72YO6PKN.js";import"./chunk-2SP57VCG.js";import"./chunk-6DTXIZY2.js";import"./chunk-WJ4XFLS4.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};