mcp-scraper 0.90.4 → 0.90.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +2 -2
  3. package/dist/{analytics-repository-UX6VU22H.js → analytics-repository-WNJESD4A.js} +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/mcp-scraper-cli.js +1 -1
  6. package/dist/bin/mcp-scraper-core.js +1 -1
  7. package/dist/bin/mcp-scraper-install.js +1 -1
  8. package/dist/bin/mcp-stdio-server.js +1 -1
  9. package/dist/bin/paa-harvest.js +1 -1
  10. package/dist/{chunk-HR2A2WTL.js → chunk-4OTWTGKC.js} +1 -1
  11. package/dist/{chunk-DKQZNSYM.js → chunk-56UT5M5Y.js} +4 -4
  12. package/dist/{chunk-7QHYBK6U.js → chunk-5MMN27Q6.js} +2 -2
  13. package/dist/{chunk-7AC66V3T.js → chunk-AAJG5OHF.js} +1 -1
  14. package/dist/{chunk-HR2ALIEH.js → chunk-AQSMJCTF.js} +1 -1
  15. package/dist/{chunk-GW6ITK76.js → chunk-AQXYXVOP.js} +8 -8
  16. package/dist/{chunk-JSNFZPPB.js → chunk-BVSK6YHX.js} +1 -1
  17. package/dist/{chunk-LZTDANNZ.js → chunk-COIA4RW4.js} +1 -1
  18. package/dist/{chunk-YEFXVTMJ.js → chunk-CTX3NMZQ.js} +1 -1
  19. package/dist/{chunk-7BEWBRMT.js → chunk-KKHTMYCD.js} +198 -87
  20. package/dist/{chunk-HJ7OGU2J.js → chunk-MADTJXNI.js} +1 -1
  21. package/dist/chunk-NOUHD54C.js +48 -0
  22. package/dist/{chunk-ZMLCK45S.js → chunk-NWRRVZKW.js} +1 -1
  23. package/dist/{chunk-2GRE4BAO.js → chunk-Q5SUDDSQ.js} +1 -1
  24. package/dist/chunk-UM4WSLKQ.js +1 -0
  25. package/dist/{chunk-WTOKMWDS.js → chunk-WDB5CTRY.js} +1 -1
  26. package/dist/chunk-XARSC34L.js +1279 -0
  27. package/dist/{chunk-JVU2THSY.js → chunk-XXK5PLRB.js} +1 -1
  28. package/dist/{chunk-3R5QLQT3.js → chunk-YDOBZMLS.js} +1 -1
  29. package/dist/{db-UKQ74OVT.js → db-XN7YPDGC.js} +1 -1
  30. package/dist/{extract-bundle-7BI2K26D.js → extract-bundle-CP6WBOJI.js} +1 -1
  31. package/dist/{gmail-service-Y2UZX3YW.js → gmail-service-5NH2GVQ5.js} +1 -1
  32. package/dist/index.cjs +46 -46
  33. package/dist/index.js +1 -1
  34. package/dist/{lead-list-enrichment-repository-GPDHJVM3.js → lead-list-enrichment-repository-AIMOP7YM.js} +1 -1
  35. package/dist/{location-data-repository-MXUNVDHW.js → location-data-repository-25D3GHXE.js} +1 -1
  36. package/dist/server-TQJ6Z3ZF.js +4931 -0
  37. package/dist/{site-extract-repository-T23S2T7L.js → site-extract-repository-KG6FQU55.js} +1 -1
  38. package/dist/{stripe-event-worker-HVCB6SF3.js → stripe-event-worker-ZNNQEVK4.js} +1 -1
  39. package/dist/worker-4LFWAEX2.js +1 -0
  40. package/package.json +1 -1
  41. package/dist/chunk-KJA62B7P.js +0 -1
  42. package/dist/chunk-LTT2B7NY.js +0 -1280
  43. package/dist/chunk-PHBU6X5B.js +0 -1
  44. package/dist/server-ZZVUT4N6.js +0 -6517
  45. package/dist/worker-KFILERH3.js +0 -1
@@ -1,4 +1,4 @@
1
- import{a as N}from"./chunk-XG6GCEUE.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,t as x}from"./chunk-7BEWBRMT.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
1
+ import{a as N}from"./chunk-XG6GCEUE.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,t as x}from"./chunk-KKHTMYCD.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
2
2
  `).matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?([^"')&]+)(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?\s*\)/gi))n(l[1],`static-css-url${i}`,"image");for(let l of d.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi))n(l[1],`static-bare-url${i}`)},c=typeof e=="string"?P(e):e;return a(c),c.$("script").each((d,i)=>{let o=c.$(i).text(),h=o.replace(/\\u003[cC]/g,"<").replace(/\\u003[eE]/g,">").replace(/\\u0026/gi,"&").replace(/\\["']/g,l=>l.slice(1)).replace(/\\\//g,"/");h!==o&&/<(?:img|source|video|audio)\b/i.test(h)&&a(P(h),"-json-unescaped")}),[...r.values()]}function fe(e,t,r=100){let n=new Map;for(let a of e)A(n,a);for(let a of t?.assets??[])A(n,{...a,type:a.type});let g=new Map;for(let a of n.values()){let c=pe(a.url),d=g.get(c)??[];d.push(a),g.set(c,d)}return[...g.values()].map(a=>{let c=[...a].sort((i,o)=>S(o)-S(i)||i.url.localeCompare(o.url)),d={...c[0]};return d.discoveryMethods=[...new Set(a.flatMap(i=>i.discoveryMethods))],d.altTexts=[...new Set(a.flatMap(i=>i.altTexts))],d.contexts=[...new Set(a.flatMap(i=>i.contexts))],{...d,variants:c.slice(1).map(i=>i.url)}}).sort((a,c)=>S(c)-S(a)||a.url.localeCompare(c.url)).slice(0,Math.max(1,r))}async function ye(e,t,r,n={}){let g=Math.max(1,Math.min(n.maxBytes??52428800,104857600)),a=e,c=null;for(let p=0;p<=5;p++){let y=await L(a,{field:"media URL"});if(y.error||!y.parsed)throw new Error(y.error??"Media URL was rejected");if(c=await fetch(y.parsed.href,{signal:AbortSignal.timeout(15e3),redirect:"manual"}),c.status>=300&&c.status<400){let w=c.headers.get("location");if(!w)throw new Error(`HTTP ${c.status} redirect did not include Location`);a=new URL(w,y.parsed.href).href,c=null;continue}break}if(!c)throw new Error("Media download exceeded five redirects");if(!c.ok)throw new Error(`HTTP ${c.status}`);if(!c.body)throw new Error("Empty response body");let d=c.headers.get("content-type")?.split(";")[0].trim()??null;if(n.expectedType&&(!d||q(d)!==n.expectedType))throw new Error(`Expected ${n.expectedType} content but received ${d??"no content-type"}`);let i=Number(c.headers.get("content-length")??0);if(Number.isFinite(i)&&i>g)throw new Error(`Media asset exceeds ${g} byte limit`);let o=I(t,r);if(d&&!k(r)){let y={"image/jpeg":".jpg","image/png":".png","image/webp":".webp","image/gif":".gif","image/svg+xml":".svg","image/avif":".avif","video/mp4":".mp4","video/webm":".webm","audio/mpeg":".mp3","audio/ogg":".ogg","audio/wav":".wav"}[d];y&&(o=o+y)}let h=Z(o);await new Promise((p,y)=>{h.once("open",()=>p()),h.once("error",y)});let l=0,u=new ie({transform(p,y,w){let _=p.length;if(l+_>g){w(new Error(`Media asset exceeds ${g} byte limit`));return}if(n.consumeBytes&&!n.consumeBytes(_)){w(new Error("Media export total byte limit exceeded"));return}l+=_,w(null,p)}});try{await ne(oe.fromWeb(c.body),u,h)}catch(p){throw h.destroy(),te(o,{force:!0}),p}let{statSync:s}=await import("fs"),m=s(o).size;return{savedPath:o,sizeBytes:m,mimeType:d}}async function Le(e,t,r={}){let n=r.types??["image","video","audio"],g=new Set(n),a=he(e,t),c=Math.max(1,Math.min(r.maxAssets??100,250)),d=fe(a,r.rendered,2e3),i=d.slice(0,c),o=new Set([...a.map(f=>f.url),...(r.rendered?.assets??[]).map(f=>f.url)]).size,h=i.flatMap(f=>{let b=f.type??F(f.url);return b&&g.has(b)?[{...f,type:b}]:[]}),l=o-h.length,u=d.length>c,s=r.rendered?.exhausted===!0&&!u,m=u?"asset_limit":r.rendered?.stopReason??"render_unavailable",p=u?[`Media discovery found ${d.length} responsive families; only the requested ${c} records were retained.`]:r.rendered?r.rendered.exhausted?[]:[`Rendered media discovery stopped with ${r.rendered.stopReason}; the inventory may be incomplete.`]:["Rendered media discovery was unavailable; this manifest contains static-source evidence only."],y={pageUrl:t,staticFound:a.length,renderedFound:r.rendered?.assets.length??0,filteredCount:l,totalFound:o,completeness:s?"complete":"partial",exhausted:s,stopReason:m,scrollRounds:r.rendered?.scrollRounds??0,warnings:p,artifact:null};if(r.outputDir===null)return{...y,outputDir:null,assets:h.map((f,b)=>({...f,mimeType:null,filename:U(f.url,b),savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"not_attempted",downloadError:null,inlinePreview:null}))};let w=(()=>{try{return new URL(t).hostname.replace(/^www\./,"")}catch{return"unknown"}})(),_=new Date().toISOString().replace(/[:.]/g,"-").slice(0,19),C=r.outputDir??I(re(),"Downloads","mcp-scraper","media",`${_}-${w}`);ee(C,{recursive:!0});let T=[];return await Promise.allSettled(h.map(async(f,b)=>{let R=U(f.url,b);try{let{savedPath:v,sizeBytes:Q,mimeType:D}=await ye(f.url,C,R),G=D?q(D)??f.type:f.type;T.push({...f,type:G,mimeType:D,filename:B(v),savedPath:v,sizeBytes:Q,sha256:null,downloadStatus:"downloaded",downloadError:null,inlinePreview:null})}catch(v){T.push({...f,mimeType:null,filename:R,savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"failed",downloadError:v instanceof Error?v.message:"media_download_failed",inlinePreview:null})}})),T.sort((f,b)=>f.savedPath&&!b.savedPath?-1:!f.savedPath&&b.savedPath?1:f.url.localeCompare(b.url)),{...y,outputDir:C,assets:T}}var be=new Set(["browser_navigation_blocked","browser_navigation_failed","browser_timeout","browser_session_failed","browser_result_missing"]);function we(e){return e instanceof Error?e.code:void 0}function _e(e){return e instanceof Error?e.message:String(e)}function H(e){let t=e?.match(/^browser_http_(\d+)$/);return t?Number(t[1]):null}function ve(e){return e===404?"page_not_found":e===403?"page_forbidden":e===429?"page_rate_limited":e>=500&&e<=599?"page_server_error":"page_http_error"}function xe(e){let t=_e(e),r=we(e);if(N(e,t))return{errorCode:"vendor_unavailable",httpStatus:null,retryable:!0};let n=H(r)??H(t.match(/^(browser_http_\d+)/)?.[1]);return n!=null?{errorCode:ve(n),httpStatus:n,retryable:n===429||n>=500}:r==="browser_challenge_unresolved"||/^browser_challenge_unresolved/.test(t)?{errorCode:"bot_check_unresolved",httpStatus:null,retryable:!0}:r==="browser_response_too_large"||/exceeds \d+ byte limit/i.test(t)?{errorCode:"page_too_large",httpStatus:null,retryable:!1}:/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(t)?{errorCode:"browser_session_interrupted",httpStatus:null,retryable:!0}:be.has(r??"")||/timeout|navigation_failed|ERR_/i.test(t)?{errorCode:"page_unreachable",httpStatus:null,retryable:!0}:{errorCode:"extraction_failed",httpStatus:null,retryable:!1}}function j(e,t){switch(e){case"page_not_found":return`The page could not be found (HTTP ${t??404}). It may have been moved or deleted.`;case"page_forbidden":return`The site refused the request (HTTP ${t??403}).`;case"page_rate_limited":return"The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";case"page_server_error":return`The page's own server returned an error (HTTP ${t}). This is on the target site's side, not something a retry here can fix.`;case"page_http_error":return`The page returned an unexpected HTTP ${t} response.`;case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. A retry uses a fresh browser session.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"extraction_failed":return"The page could not be extracted. Please retry, or contact support if this persists for the same URL."}}function Ie(e,t={}){let r=j(e.errorCode,e.httpStatus);return{...$({errorCode:e.errorCode,retryable:e.retryable,chargeStatus:t.chargeStatus,details:{...t.details??{},...e.httpStatus!=null?{http_status:e.httpStatus}:{}}}),message:r,error:r}}function ke(e){return e.httpStatus===404?404:e.httpStatus===403?403:e.httpStatus===429?429:e.httpStatus!=null&&e.httpStatus>=500&&e.httpStatus<=599?500:e.httpStatus!=null?502:e.errorCode==="vendor_unavailable"?503:502}var Ee="wayback_capture_missing";function Be(e,t){if(e==null)return{failureCode:null,failureReason:null};if(e===Ee)return{failureCode:e,failureReason:t??null};let r=new Error(t??e);r.code=e;let n=xe(r);return{failureCode:n.errorCode,failureReason:j(n.errorCode,n.httpStatus)}}import{neon as Te}from"@neondatabase/serverless";var Se="jina",M=null,z=!1;function W(){return(process.env.JINA_EMBED_MODEL??"jina-embeddings-v5-omni-small").trim()}function Y(){return Number((process.env.JINA_EMBED_DIM??"1024").trim())}function J(){return!!(process.env.JINA_API_KEY?.trim()&&process.env.MEMORY_DATABASE_URL?.trim())}function E(){if(M)return M;let e=process.env.MEMORY_DATABASE_URL?.trim();if(!e)throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");return M=Te(e),M}async function K(){if(z)return;let e=Y();await E().query("CREATE EXTENSION IF NOT EXISTS vector"),await E().query(`
3
3
  CREATE TABLE IF NOT EXISTS commons_index_vectors (
4
4
  document_id TEXT PRIMARY KEY,
@@ -1,4 +1,4 @@
1
- import{k as K}from"./chunk-WTOKMWDS.js";import{a as X}from"./chunk-2GRE4BAO.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-7BEWBRMT.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
1
+ import{k as K}from"./chunk-WDB5CTRY.js";import{a as X}from"./chunk-Q5SUDDSQ.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-KKHTMYCD.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
2
2
  CREATE TABLE IF NOT EXISTS gmail_selections (
3
3
  selection_id TEXT PRIMARY KEY,
4
4
  owner_identity TEXT NOT NULL,
@@ -1 +1 @@
1
- import{$ as M,$a as Ma,$b as Mb,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Yb as Jb,Z as K,Za as Ka,Zb as Kb,_ as L,_a as La,_b as Lb,aa as N,ab as Na,ac as Nb,ba as O,bb as Oa,bc as Ob,ca as P,cb as Pa,cc as Pb,da as Q,db as Qa,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-7BEWBRMT.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,ub as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Ib as SiteAuditJobRowSchema,Jb as SiteAuditPhaseLogRowSchema,za as advanceWorkflowRunStep,xa as appendWorkflowArtifacts,lb as applySubscriptionInvoiceGrant,V as attachHarvestAttemptProviderSession,Xa as cancelJob,J as checkRateLimit,Ia as checkpointDurablePaaJob,Ha as checkpointJob,s as claimDueStripeEvents,Ga as claimDurablePaaJob,Ab as claimMonthlyFreeRefresh,Fa as claimPendingJob,Za as claimPendingKpoJob,Ka as completeDurablePaaJob,Na as completeExpiredDurablePaaJob,Ta as completeJob,ab as completeKpoJob,Ua as completeRunningJob,t as completeStripeEvent,ua as completeWorkflowRunRecord,m as consumeCode,L as consumePasswordResetToken,Ea as countActiveJobsForUser,Da as countActiveUsers,Q as createJob,R as createJobWithId,Ya as createKpoJob,K as createPasswordResetToken,S as createRunningJob,I as createUser,pa as createWorkflowRun,ia as createWorkflowSchedule,xb as creditMc,yb as creditMcIdempotent,P as deactivateUser,v as deadLetterStripeEvent,Bb as debitMc,Cb as debitMcIdempotent,ma as deleteWorkflowSchedule,r as enqueueStripeEvent,vb as expireOldLots,La as failDurablePaaJob,Oa as failExpiredDurablePaaJob,Va as failJob,bb as failKpoJob,Wa as failRunningJob,va as failWorkflowRunRecord,W as finalizeExpiredHarvestAttempts,X as finishHarvestAttempt,y as generateApiKey,Fb as getBillingLedgerSummary,k as getClient,mb as getConnectedAccountBillingState,e as getDb,Lb as getInboxMessage,$ as getJob,_a as getKpoJob,db as getKpoPhaseLog,Eb as getLedger,qb as getMemoryPlan,rb as getMemoryProvisioned,T as getOrCreateRunningJob,ea as getPageSnapshot,o as getRefresh,w as getStripeEventQueueHealth,C as getUserByApiKey,B as getUserByEmail,D as getUserById,fb as getUserByStripeCustomerId,E as getUserMemoryCreds,H as getUserStats,Ba as getWorkflowArtifact,qa as getWorkflowRun,ra as getWorkflowRunByIdempotencyKey,ya as getWorkflowRunStepState,ja as getWorkflowSchedule,z as hashPassword,Ob as insertInboxMessage,d as inspectCurrentCoreSchema,zb as ledgerExistsForOperation,Gb as ledgerExistsForStripePI,Pb as listActiveUserIds,na as listDueWorkflowSchedules,_ as listHarvestAttempts,Z as listHarvestAttemptsPendingProviderTelemetry,Kb as listInboxMessages,da as listJobs,eb as listKpoJobs,ba as listPendingExtractUrlDispatches,Pa as listRecentTerminalDurablePaaJobs,Ra as listRecoverableDurablePaaJobs,ha as listRequestEvents,Qa as listStaleRunningDurablePaaJobs,O as listUsers,ob as listUsersDueForConnectedAccountBillingReconciliation,Aa as listWorkflowArtifacts,sa as listWorkflowRuns,ka as listWorkflowSchedules,cb as logKpoPhaseComplete,ga as logRequestEvent,aa as markDispatchedJobRunning,Mb as markInboxMessageOpened,Sa as markJobBillingSettled,ta as markWorkflowRunRunning,oa as markWorkflowScheduleRan,i as migrate,wb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,la as patchWorkflowSchedule,l as putCode,n as putRefresh,Hb as reconcileBalanceMc,ca as recordExtractUrlDispatchAttempt,Ca as recordWorkflowWebhookDelivery,j as registerClient,Ja as renewDurablePaaLease,wa as replaceWorkflowArtifacts,Ma as requeueExpiredDurablePaaJob,tb as resolveUserByMemorySubscription,u as retryStripeEvent,N as revokeApiKey,p as revokeRefresh,M as rotateApiKey,q as rotateRefresh,ib as setConcurrencyPackQuantity,jb as setConcurrencySubId,nb as setConnectedAccountBillingState,f as setDbForTesting,hb as setExtraConcurrencySlots,pb as setMemoryPlan,sb as setMemoryProvisioned,G as setPassword,gb as setStripeCustomerId,kb as setSubscriptionTier,F as setUserMemoryCreds,Db as settleDebitMcIdempotent,Nb as softDeleteInboxMessage,U as startHarvestAttempt,x as stripeEventAlreadyProcessed,Y as updateHarvestAttemptProviderTelemetry,$a as updateKpoJobState,fa as upsertPageSnapshot,A as verifyPassword,h as verifySecuredMainOAuthSchema};
1
+ import{$ as M,$a as Ma,$b as Mb,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Yb as Jb,Z as K,Za as Ka,Zb as Kb,_ as L,_a as La,_b as Lb,aa as N,ab as Na,ac as Nb,ba as O,bb as Oa,bc as Ob,ca as P,cb as Pa,cc as Pb,da as Q,db as Qa,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-KKHTMYCD.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,ub as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Ib as SiteAuditJobRowSchema,Jb as SiteAuditPhaseLogRowSchema,za as advanceWorkflowRunStep,xa as appendWorkflowArtifacts,lb as applySubscriptionInvoiceGrant,V as attachHarvestAttemptProviderSession,Xa as cancelJob,J as checkRateLimit,Ia as checkpointDurablePaaJob,Ha as checkpointJob,s as claimDueStripeEvents,Ga as claimDurablePaaJob,Ab as claimMonthlyFreeRefresh,Fa as claimPendingJob,Za as claimPendingKpoJob,Ka as completeDurablePaaJob,Na as completeExpiredDurablePaaJob,Ta as completeJob,ab as completeKpoJob,Ua as completeRunningJob,t as completeStripeEvent,ua as completeWorkflowRunRecord,m as consumeCode,L as consumePasswordResetToken,Ea as countActiveJobsForUser,Da as countActiveUsers,Q as createJob,R as createJobWithId,Ya as createKpoJob,K as createPasswordResetToken,S as createRunningJob,I as createUser,pa as createWorkflowRun,ia as createWorkflowSchedule,xb as creditMc,yb as creditMcIdempotent,P as deactivateUser,v as deadLetterStripeEvent,Bb as debitMc,Cb as debitMcIdempotent,ma as deleteWorkflowSchedule,r as enqueueStripeEvent,vb as expireOldLots,La as failDurablePaaJob,Oa as failExpiredDurablePaaJob,Va as failJob,bb as failKpoJob,Wa as failRunningJob,va as failWorkflowRunRecord,W as finalizeExpiredHarvestAttempts,X as finishHarvestAttempt,y as generateApiKey,Fb as getBillingLedgerSummary,k as getClient,mb as getConnectedAccountBillingState,e as getDb,Lb as getInboxMessage,$ as getJob,_a as getKpoJob,db as getKpoPhaseLog,Eb as getLedger,qb as getMemoryPlan,rb as getMemoryProvisioned,T as getOrCreateRunningJob,ea as getPageSnapshot,o as getRefresh,w as getStripeEventQueueHealth,C as getUserByApiKey,B as getUserByEmail,D as getUserById,fb as getUserByStripeCustomerId,E as getUserMemoryCreds,H as getUserStats,Ba as getWorkflowArtifact,qa as getWorkflowRun,ra as getWorkflowRunByIdempotencyKey,ya as getWorkflowRunStepState,ja as getWorkflowSchedule,z as hashPassword,Ob as insertInboxMessage,d as inspectCurrentCoreSchema,zb as ledgerExistsForOperation,Gb as ledgerExistsForStripePI,Pb as listActiveUserIds,na as listDueWorkflowSchedules,_ as listHarvestAttempts,Z as listHarvestAttemptsPendingProviderTelemetry,Kb as listInboxMessages,da as listJobs,eb as listKpoJobs,ba as listPendingExtractUrlDispatches,Pa as listRecentTerminalDurablePaaJobs,Ra as listRecoverableDurablePaaJobs,ha as listRequestEvents,Qa as listStaleRunningDurablePaaJobs,O as listUsers,ob as listUsersDueForConnectedAccountBillingReconciliation,Aa as listWorkflowArtifacts,sa as listWorkflowRuns,ka as listWorkflowSchedules,cb as logKpoPhaseComplete,ga as logRequestEvent,aa as markDispatchedJobRunning,Mb as markInboxMessageOpened,Sa as markJobBillingSettled,ta as markWorkflowRunRunning,oa as markWorkflowScheduleRan,i as migrate,wb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,la as patchWorkflowSchedule,l as putCode,n as putRefresh,Hb as reconcileBalanceMc,ca as recordExtractUrlDispatchAttempt,Ca as recordWorkflowWebhookDelivery,j as registerClient,Ja as renewDurablePaaLease,wa as replaceWorkflowArtifacts,Ma as requeueExpiredDurablePaaJob,tb as resolveUserByMemorySubscription,u as retryStripeEvent,N as revokeApiKey,p as revokeRefresh,M as rotateApiKey,q as rotateRefresh,ib as setConcurrencyPackQuantity,jb as setConcurrencySubId,nb as setConnectedAccountBillingState,f as setDbForTesting,hb as setExtraConcurrencySlots,pb as setMemoryPlan,sb as setMemoryProvisioned,G as setPassword,gb as setStripeCustomerId,kb as setSubscriptionTier,F as setUserMemoryCreds,Db as settleDebitMcIdempotent,Nb as softDeleteInboxMessage,U as startHarvestAttempt,x as stripeEventAlreadyProcessed,Y as updateHarvestAttemptProviderTelemetry,$a as updateKpoJobState,fa as upsertPageSnapshot,A as verifyPassword,h as verifySecuredMainOAuthSchema};
@@ -1,4 +1,4 @@
1
- import{c as We,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-JVU2THSY.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-ZMLCK45S.js";import"./chunk-XG6GCEUE.js";import"./chunk-M5QHXNFZ.js";import{c as Je,e as ze,f as Fe,g as He}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-YEFXVTMJ.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-7BEWBRMT.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
1
+ import{c as We,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-XXK5PLRB.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-NWRRVZKW.js";import"./chunk-XG6GCEUE.js";import"./chunk-M5QHXNFZ.js";import{c as Je,e as ze,f as Fe,g as He}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-CTX3NMZQ.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-KKHTMYCD.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
2
2
 
3
3
  `).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
4
4
 
@@ -1 +1 @@
1
- import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-3R5QLQT3.js";import"./chunk-WTOKMWDS.js";import"./chunk-2GRE4BAO.js";import"./chunk-HUV2WTRW.js";import"./chunk-YEFXVTMJ.js";import"./chunk-DKQZNSYM.js";import"./chunk-7BEWBRMT.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};
1
+ import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-YDOBZMLS.js";import"./chunk-WDB5CTRY.js";import"./chunk-Q5SUDDSQ.js";import"./chunk-HUV2WTRW.js";import"./chunk-CTX3NMZQ.js";import"./chunk-56UT5M5Y.js";import"./chunk-KKHTMYCD.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};