mcp-scraper 0.89.4 → 0.89.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +13 -1
  2. package/README.md +1 -1
  3. package/dist/{analytics-repository-5CAFUXYD.js → analytics-repository-RG7CFK6W.js} +1 -1
  4. package/dist/bin/api-server.js +1 -1
  5. package/dist/bin/mcp-scraper-cli.js +1 -1
  6. package/dist/bin/mcp-scraper-core.js +1 -1
  7. package/dist/bin/mcp-scraper-install.js +1 -1
  8. package/dist/bin/mcp-stdio-server.js +1 -1
  9. package/dist/bin/paa-harvest.js +1 -1
  10. package/dist/{chunk-2O3UYISB.js → chunk-2ICI4TIX.js} +1 -1
  11. package/dist/{chunk-HYQDXTS7.js → chunk-3UPA3XHF.js} +1 -1
  12. package/dist/{chunk-QN3QHTVK.js → chunk-4A5F7IUA.js} +1 -1
  13. package/dist/{chunk-CYOEMHTB.js → chunk-4SQ4RKDX.js} +1 -1
  14. package/dist/{chunk-D4NOJAUN.js → chunk-53HIX3X7.js} +1 -1
  15. package/dist/{chunk-Z66Z2HV4.js → chunk-7JAMFXNL.js} +55 -55
  16. package/dist/{chunk-OXFWXWQG.js → chunk-A2CZ7WRV.js} +1 -1
  17. package/dist/{chunk-OBDMJCCF.js → chunk-DBDMD7V7.js} +1 -1
  18. package/dist/{chunk-VEJQ4XPI.js → chunk-JJDYKM5I.js} +1 -1
  19. package/dist/{chunk-Z27SYWYD.js → chunk-NGQYSQQA.js} +1 -1
  20. package/dist/{chunk-S7RA23GR.js → chunk-O2QKRYLS.js} +1 -1
  21. package/dist/{chunk-EN66HBAW.js → chunk-RZSJC7FX.js} +1 -1
  22. package/dist/chunk-SCHTLEMZ.js +1 -0
  23. package/dist/{chunk-J7EVLE6H.js → chunk-SEHBXTMX.js} +1 -1
  24. package/dist/{chunk-DMCYIWB3.js → chunk-UB5N2ICD.js} +1 -1
  25. package/dist/{chunk-K7KMLT33.js → chunk-WYCXAFID.js} +1 -1
  26. package/dist/db-DV4HHCYP.js +1 -0
  27. package/dist/{extract-bundle-TXH3VNXH.js → extract-bundle-RVOBMGU2.js} +1 -1
  28. package/dist/{gmail-service-J2DWSQJX.js → gmail-service-FJTBCZPH.js} +1 -1
  29. package/dist/index.js +1 -1
  30. package/dist/{lead-list-enrichment-repository-G7Z5DQZR.js → lead-list-enrichment-repository-GKVCX7BM.js} +1 -1
  31. package/dist/{location-data-repository-TEXZ3I2N.js → location-data-repository-WK27TSHZ.js} +1 -1
  32. package/dist/{server-RORLM2FG.js → server-7TSXB24M.js} +241 -241
  33. package/dist/{site-extract-repository-MBSCAP4D.js → site-extract-repository-FAHOTQGX.js} +1 -1
  34. package/dist/{worker-SUCJKWU2.js → worker-ROG7SO3R.js} +1 -1
  35. package/package.json +1 -1
  36. package/dist/chunk-RNDORDA2.js +0 -1
  37. package/dist/db-NLGNTQC3.js +0 -1
@@ -1,4 +1,4 @@
1
- import{a as Q,d as Se}from"./chunk-Z27SYWYD.js";import{createHash as pe,createHmac as Yt,randomBytes as P,randomUUID as y}from"crypto";import{Pool as Wt}from"pg";var ct=[[/perplexity/i,"llm","Perplexity"],[/chatgpt|chat\.openai|openai/i,"llm","ChatGPT"],[/claude|anthropic/i,"llm","Claude"],[/\bgrok\b|xai/i,"llm","Grok"],[/ai[ _-]?overview|google[ _-]?ai|\bsge\b|gemini/i,"llm","Google AI Overview"],[/instagram/i,"social","Instagram"],[/facebook|\bfb\b|meta/i,"social","Facebook"],[/linkedin/i,"social","LinkedIn"],[/tiktok/i,"social","TikTok"],[/youtube|youtu\.be/i,"social","YouTube"],[/twitter|x\.com/i,"social","X"],[/\bg2\b|g2\.com/i,"review","G2"],[/trustpilot/i,"review","Trustpilot"],[/\bbbb\b|better business bureau/i,"review","BBB"],[/capterra/i,"review","Capterra"],[/yelp/i,"review","Yelp"],[/google|bing|duckduckgo|yahoo/i,"search","Search"]];function q(e,t=240){return typeof e=="string"&&e.trim()?e.trim().slice(0,t):null}function Ce(e){try{return e?new URL(e).hostname.replace(/^www\./,""):""}catch{return""}}function Re(e){let t=e.properties??{},i=q(e.source,180)||Ce(e.referrer)||"(direct)",n=q(e.medium,180)||(i==="(direct)"?"(none)":"referral"),s=e.clickIds??{},a=`${i} ${n} ${Ce(e.referrer)}`,r=i==="(direct)"?"direct":/email|newsletter/i.test(n)?"email":"referral",o=i==="(direct)"?"Direct":i;for(let[d,_,u]of ct)if(d.test(a)){r=_,o=u;break}return s.fbclid?(r="social",o=/instagram/i.test(a)?"Instagram":"Facebook"):s.ttclid?(r="social",o="TikTok"):s.rdt_cid?(r="social",o="Reddit"):s.gclid||s.gbraid||s.wbraid?(r="search",o="Google"):s.msclkid&&(r="search",o="Microsoft Ads"),{channelFamily:r,platform:o,source:i,medium:n,campaign:q(e.campaign),campaignId:q(t.campaign_id??t.utm_id),adSetId:q(t.adset_id??t.ad_set_id??t.adgroup_id??t.ad_group_id),adId:q(t.ad_id),creativeId:q(t.creative_id),placement:q(t.placement),term:q(t.utm_term),content:q(t.utm_content)}}import{createHash as yt,randomUUID as ee}from"crypto";import{createCipheriv as _t,createDecipheriv as ut,createHmac as fe,randomBytes as Et,scryptSync as lt,timingSafeEqual as Nt}from"crypto";var Z="xray1";function mt(){return process.env.ANALYTICS_RESTRICTED_DATA_SECRET?.trim()||Q()}function _e(e){return lt(mt(),`xray-restricted:${e}:v1`,32)}function De(e,t){let i=t.trim().slice(0,80);if(!i)throw new Error("analytics encryption purpose required");let n=Et(12),s=_t("aes-256-gcm",_e(i),n);s.setAAD(Buffer.from(`${Z}:${i}`,"utf8"));let a=Buffer.concat([s.update(e,"utf8"),s.final()]),r=s.getAuthTag();return[Z,Buffer.from(i,"utf8").toString("base64url"),n.toString("base64url"),r.toString("base64url"),a.toString("base64url")].join(".")}function Ue(e,t){try{let[i,n,s,a,r]=e.split(".");if(i!==Z||!n||!s||!a||!r)return null;let o=Buffer.from(n,"base64url").toString("utf8");if(o!==t)return null;let d=ut("aes-256-gcm",_e(o),Buffer.from(s,"base64url"));return d.setAAD(Buffer.from(`${Z}:${o}`,"utf8")),d.setAuthTag(Buffer.from(a,"base64url")),Buffer.concat([d.update(Buffer.from(r,"base64url")),d.final()]).toString("utf8")}catch{return null}}function bi(e,t){return fe("sha256",_e(`identity:${e}`)).update(t.trim().toLowerCase()).digest("hex")}function hi(e,t,i,n){return fe("sha256",n).update(`${e}.${t}.${i}`).digest("hex")}function xi(e,t){if(!e||e.length!==t.length)return!1;try{return Nt(Buffer.from(e),Buffer.from(t))}catch{return!1}}var we="enhanced_matching",pt=90,Tt=10,H=class extends Error{constructor(i,n,s){super(n);this.code=i;this.status=s;this.name="AnalyticsIdentityProfileError"}code;status};function te(e,t){return e?.trim().replace(/\s+/g," ").slice(0,t)||null}function Lt(e){if(!e)return null;let[t,i]=e.split("@");return!t||!i?null:`${t.slice(0,1)}${"*".repeat(Math.min(6,Math.max(2,t.length-1)))}@${i}`}function At(e){if(!e)return null;let t=e.replace(/\D/g,"");return t.length<4?null:`***-***-${t.slice(-4)}`}function ue(e){return yt("sha256").update(e).digest("hex")}async function ie(e,t){if(!t.signals.length)return;let i=t.signals.map(a=>a.kind),n=t.signals.map(a=>a.valueHmac);if((await e.query(`SELECT 1 FROM analytics_identity_tombstones
1
+ import{a as Q,d as Se}from"./chunk-NGQYSQQA.js";import{createHash as pe,createHmac as Yt,randomBytes as P,randomUUID as y}from"crypto";import{Pool as Wt}from"pg";var ct=[[/perplexity/i,"llm","Perplexity"],[/chatgpt|chat\.openai|openai/i,"llm","ChatGPT"],[/claude|anthropic/i,"llm","Claude"],[/\bgrok\b|xai/i,"llm","Grok"],[/ai[ _-]?overview|google[ _-]?ai|\bsge\b|gemini/i,"llm","Google AI Overview"],[/instagram/i,"social","Instagram"],[/facebook|\bfb\b|meta/i,"social","Facebook"],[/linkedin/i,"social","LinkedIn"],[/tiktok/i,"social","TikTok"],[/youtube|youtu\.be/i,"social","YouTube"],[/twitter|x\.com/i,"social","X"],[/\bg2\b|g2\.com/i,"review","G2"],[/trustpilot/i,"review","Trustpilot"],[/\bbbb\b|better business bureau/i,"review","BBB"],[/capterra/i,"review","Capterra"],[/yelp/i,"review","Yelp"],[/google|bing|duckduckgo|yahoo/i,"search","Search"]];function q(e,t=240){return typeof e=="string"&&e.trim()?e.trim().slice(0,t):null}function Ce(e){try{return e?new URL(e).hostname.replace(/^www\./,""):""}catch{return""}}function Re(e){let t=e.properties??{},i=q(e.source,180)||Ce(e.referrer)||"(direct)",n=q(e.medium,180)||(i==="(direct)"?"(none)":"referral"),s=e.clickIds??{},a=`${i} ${n} ${Ce(e.referrer)}`,r=i==="(direct)"?"direct":/email|newsletter/i.test(n)?"email":"referral",o=i==="(direct)"?"Direct":i;for(let[d,_,u]of ct)if(d.test(a)){r=_,o=u;break}return s.fbclid?(r="social",o=/instagram/i.test(a)?"Instagram":"Facebook"):s.ttclid?(r="social",o="TikTok"):s.rdt_cid?(r="social",o="Reddit"):s.gclid||s.gbraid||s.wbraid?(r="search",o="Google"):s.msclkid&&(r="search",o="Microsoft Ads"),{channelFamily:r,platform:o,source:i,medium:n,campaign:q(e.campaign),campaignId:q(t.campaign_id??t.utm_id),adSetId:q(t.adset_id??t.ad_set_id??t.adgroup_id??t.ad_group_id),adId:q(t.ad_id),creativeId:q(t.creative_id),placement:q(t.placement),term:q(t.utm_term),content:q(t.utm_content)}}import{createHash as yt,randomUUID as ee}from"crypto";import{createCipheriv as _t,createDecipheriv as ut,createHmac as fe,randomBytes as Et,scryptSync as lt,timingSafeEqual as Nt}from"crypto";var Z="xray1";function mt(){return process.env.ANALYTICS_RESTRICTED_DATA_SECRET?.trim()||Q()}function _e(e){return lt(mt(),`xray-restricted:${e}:v1`,32)}function De(e,t){let i=t.trim().slice(0,80);if(!i)throw new Error("analytics encryption purpose required");let n=Et(12),s=_t("aes-256-gcm",_e(i),n);s.setAAD(Buffer.from(`${Z}:${i}`,"utf8"));let a=Buffer.concat([s.update(e,"utf8"),s.final()]),r=s.getAuthTag();return[Z,Buffer.from(i,"utf8").toString("base64url"),n.toString("base64url"),r.toString("base64url"),a.toString("base64url")].join(".")}function Ue(e,t){try{let[i,n,s,a,r]=e.split(".");if(i!==Z||!n||!s||!a||!r)return null;let o=Buffer.from(n,"base64url").toString("utf8");if(o!==t)return null;let d=ut("aes-256-gcm",_e(o),Buffer.from(s,"base64url"));return d.setAAD(Buffer.from(`${Z}:${o}`,"utf8")),d.setAuthTag(Buffer.from(a,"base64url")),Buffer.concat([d.update(Buffer.from(r,"base64url")),d.final()]).toString("utf8")}catch{return null}}function bi(e,t){return fe("sha256",_e(`identity:${e}`)).update(t.trim().toLowerCase()).digest("hex")}function hi(e,t,i,n){return fe("sha256",n).update(`${e}.${t}.${i}`).digest("hex")}function xi(e,t){if(!e||e.length!==t.length)return!1;try{return Nt(Buffer.from(e),Buffer.from(t))}catch{return!1}}var we="enhanced_matching",pt=90,Tt=10,H=class extends Error{constructor(i,n,s){super(n);this.code=i;this.status=s;this.name="AnalyticsIdentityProfileError"}code;status};function te(e,t){return e?.trim().replace(/\s+/g," ").slice(0,t)||null}function Lt(e){if(!e)return null;let[t,i]=e.split("@");return!t||!i?null:`${t.slice(0,1)}${"*".repeat(Math.min(6,Math.max(2,t.length-1)))}@${i}`}function At(e){if(!e)return null;let t=e.replace(/\D/g,"");return t.length<4?null:`***-***-${t.slice(-4)}`}function ue(e){return yt("sha256").update(e).digest("hex")}async function ie(e,t){if(!t.signals.length)return;let i=t.signals.map(a=>a.kind),n=t.signals.map(a=>a.valueHmac);if((await e.query(`SELECT 1 FROM analytics_identity_tombstones
2
2
  WHERE site_id=$1 AND (kind,value_hmac) IN (
3
3
  SELECT * FROM unnest($2::text[],$3::text[])
4
4
  ) LIMIT 1`,[t.siteId,i,n])).rowCount)throw new H("analytics_identity_deleted","This identity cannot be automatically recreated.",410)}async function ne(e,t){let i=te(t.displayName,160),n=te(t.email,320)?.toLowerCase()??null,s=te(t.phone,40);if(!i&&!n&&!s)return!1;let r=(await e.query(`SELECT p.public_ref FROM analytics_people p
@@ -1,4 +1,4 @@
1
- var s={message:"Account limits now report accurately, repeated URL extraction requests behave consistently across REST and MCP clients, and retry messages point to the correct recovery path."};var _={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${_[e]}${t}${_.reset}`:t}function o(t,e,n){let a=t.padEnd(9," ");return` ${r(a,"muted",n)} ${e.join(r(" . ","muted",n))}`}function p(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",a=String.raw`
1
+ var s={message:"PAA retries now continue without an unnecessary storage wait, and production releases verify the required database schema before the new server goes live."};var _={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${_[e]}${t}${_.reset}`:t}function o(t,e,n){let a=t.padEnd(9," ");return` ${r(a,"muted",n)} ${e.join(r(" . ","muted",n))}`}function p(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",a=String.raw`
2
2
  __ __ ____ ____
3
3
  | \/ |/ ___| _ \
4
4
  | |\/| | | | |_) |
@@ -1,4 +1,4 @@
1
- import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{W as w}from"./chunk-MZN4U5BL.js";import{Bb as N,Kb as D,d as m,e as I,i as u,s}from"./chunk-Z66Z2HV4.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
1
+ import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{W as w}from"./chunk-MZN4U5BL.js";import{Cb as N,Lb as D,d as m,e as I,i as u,t as s}from"./chunk-7JAMFXNL.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
2
2
  VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
3
3
  (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
4
4
  VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs
@@ -1,4 +1,4 @@
1
- import{M as p,s as o}from"./chunk-Z66Z2HV4.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
1
+ import{N as p,t as o}from"./chunk-7JAMFXNL.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
2
2
  CREATE TABLE IF NOT EXISTS service_connections (
3
3
  id TEXT PRIMARY KEY,
4
4
  user_id INTEGER NOT NULL REFERENCES users(id),
@@ -1 +1 @@
1
- import{g as _,i as c}from"./chunk-Z66Z2HV4.js";var g={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},a=/error|message/i,w=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),E=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,k=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),S=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),f=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function b(e){return e.replace(f,"[browser-service]")}var R=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),m=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),h=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),I=new Set(["provider_session_api","client_runtime","extractor","unknown"]),y=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),T=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),P=512,A=320;function t(e,r){return typeof e=="string"&&r.has(e)?e:null}function O(e){return typeof e!="string"?null:y.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function D(e){return typeof e!="string"||e.length<1||e.length>P||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function p(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function l(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function u(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let o=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return b(o).slice(0,A)}function v(e){return typeof e=="boolean"?e:null}function N(e){return{browser_provider:t(e.browser_provider,R),provider_session_id:D(e.provider_session_id),provider_session_observed_at:l(e.provider_session_observed_at),provider_lookup_status:t(e.provider_lookup_status,m),provider_status:t(e.provider_status,h),provider_error_code:O(e.provider_error_code),provider_error_message:u(e.provider_error_message),provider_reason_source:t(e.provider_reason_source,I),provider_duration_ms:p(e.provider_duration_ms),provider_bandwidth_bytes:p(e.provider_bandwidth_bytes),provider_captcha:v(e.provider_captcha),provider_checked_at:l(e.provider_checked_at),provider_disconnect_observed:v(e.provider_disconnect_observed),provider_disconnect_message:u(e.provider_disconnect_message)}}function i(e,r=""){if(typeof e=="string"){let o=b(e);return a.test(r)&&(w.test(e)||E.test(e))?o=_("service_unavailable"):a.test(r)&&(o=c(o)),o}if(Array.isArray(e))return e.map(o=>i(o,r));if(e!==null&&typeof e=="object"){let o={};for(let[n,d]of Object.entries(e)){if(k.has(n))continue;let s=g[n]??n;if(S.has(n)){o[s]=null;continue}o[s]=i(d,n)}return o}return e}function L(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return i(r);let o=r,n=i(o);return Object.keys(o).some(s=>T.has(s))?{...n,...N(o)}:n})}function C(e){let r=e?.diagnostics;return r?.debug?{...e,diagnostics:{...r,debug:i(r.debug)}}:e}export{i as a,L as b,C as c};
1
+ import{g as _,i as c}from"./chunk-7JAMFXNL.js";var g={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},a=/error|message/i,w=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),E=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,k=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),S=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),f=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function b(e){return e.replace(f,"[browser-service]")}var R=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),m=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),h=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),I=new Set(["provider_session_api","client_runtime","extractor","unknown"]),y=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),T=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),P=512,A=320;function t(e,r){return typeof e=="string"&&r.has(e)?e:null}function O(e){return typeof e!="string"?null:y.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function D(e){return typeof e!="string"||e.length<1||e.length>P||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function p(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function l(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function u(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let o=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return b(o).slice(0,A)}function v(e){return typeof e=="boolean"?e:null}function N(e){return{browser_provider:t(e.browser_provider,R),provider_session_id:D(e.provider_session_id),provider_session_observed_at:l(e.provider_session_observed_at),provider_lookup_status:t(e.provider_lookup_status,m),provider_status:t(e.provider_status,h),provider_error_code:O(e.provider_error_code),provider_error_message:u(e.provider_error_message),provider_reason_source:t(e.provider_reason_source,I),provider_duration_ms:p(e.provider_duration_ms),provider_bandwidth_bytes:p(e.provider_bandwidth_bytes),provider_captcha:v(e.provider_captcha),provider_checked_at:l(e.provider_checked_at),provider_disconnect_observed:v(e.provider_disconnect_observed),provider_disconnect_message:u(e.provider_disconnect_message)}}function i(e,r=""){if(typeof e=="string"){let o=b(e);return a.test(r)&&(w.test(e)||E.test(e))?o=_("service_unavailable"):a.test(r)&&(o=c(o)),o}if(Array.isArray(e))return e.map(o=>i(o,r));if(e!==null&&typeof e=="object"){let o={};for(let[n,d]of Object.entries(e)){if(k.has(n))continue;let s=g[n]??n;if(S.has(n)){o[s]=null;continue}o[s]=i(d,n)}return o}return e}function L(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return i(r);let o=r,n=i(o);return Object.keys(o).some(s=>T.has(s))?{...n,...N(o)}:n})}function C(e){let r=e?.diagnostics;return r?.debug?{...e,diagnostics:{...r,debug:i(r.debug)}}:e}export{i as a,L as b,C as c};
@@ -1,4 +1,4 @@
1
- import{s as r}from"./chunk-Z66Z2HV4.js";import{createHash as b,randomUUID as O}from"crypto";var p="mcp-scraper/lead-list-enrichment.requested",A=new Set(["complete","partial","empty","failed","cancelled"]),U=new Set(["complete","failed","cancelled"]),T=10,g=360,S=60,h=null,m=null;function o(e,t,s){let n=e.trim();if(!n||n.length>s)throw new Error(`${t} is required and must be ${s} characters or fewer`);return n}function d(e,t){if(!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer`);return e}function u(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function R(e){let t=new WeakSet,s=n=>{if(n===null||typeof n=="string"||typeof n=="boolean")return n;if(typeof n=="number"){if(!Number.isFinite(n))throw new Error("canonical fingerprint input must contain finite numbers");return Object.is(n,-0)?0:n}if(Array.isArray(n))return n.map(s);if(typeof n=="object"){if(t.has(n))throw new Error("canonical fingerprint input must not be cyclic");t.add(n);let i={};for(let _ of Object.keys(n).sort()){let c=n[_];c!==void 0&&(i[_]=s(c))}return t.delete(n),i}throw new Error(`canonical fingerprint input contains unsupported ${typeof n}`)};return JSON.stringify(s(e))}function y(e){return b("sha256").update(R(e)).digest("hex")}function I(e){return b("sha256").update(R(e)).digest("hex")}function j(){return{mapsAttempts:0,pageAttempts:0,pageSuccesses:0,serpSearches:0}}function E(e){let t=u(e.input_json);if(!t)throw new Error(`lead list enrichment job ${String(e.id??"")} has invalid durable input`);return{id:String(e.id),userId:Number(e.user_id),idempotencyKey:String(e.idempotency_key),requestFingerprint:String(e.request_fingerprint),input:t,status:String(e.status),totalRows:Number(e.total_rows??0),doneRows:Number(e.done_rows??0),failedRows:Number(e.failed_rows??0),cancelledRows:Number(e.cancelled_rows??0),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},concurrencyLockId:e.concurrency_lock_id==null?null:String(e.concurrency_lock_id),debitKey:String(e.debit_key),holdMc:Number(e.hold_mc??0),billedMc:e.billed_mc==null?null:Number(e.billed_mc),result:u(e.result_json),csvArtifact:u(e.csv_artifact_json),xlsxArtifact:u(e.xlsx_artifact_json),warnings:u(e.warnings_json)??[],cancelRequestedAt:e.cancel_requested_at==null?null:String(e.cancel_requested_at),errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:e.retryable==null?null:Number(e.retryable)===1,createdAt:String(e.created_at??""),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function w(e){return{jobId:String(e.job_id),ordinal:Number(e.ordinal),inputDigest:String(e.input_digest),status:String(e.status),result:u(e.result_json),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:Number(e.retryable??0)===1,durationMs:Number(e.duration_ms??0),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function M(e){return{id:String(e.id),jobId:String(e.job_id),eventName:p,status:String(e.status),attempts:Number(e.attempts??0),claimToken:e.locked_by==null?null:String(e.locked_by),lockedUntil:e.locked_until==null?null:String(e.locked_until),nextAttemptAt:e.next_attempt_at==null?null:String(e.next_attempt_at),lastError:e.last_error==null?null:String(e.last_error),createdAt:String(e.created_at??""),dispatchedAt:e.dispatched_at==null?null:String(e.dispatched_at),updatedAt:String(e.updated_at??"")}}async function a(){let e=r();return h!==e&&(h=e,m=null),m||(m=(async()=>{await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_jobs (
1
+ import{t as r}from"./chunk-7JAMFXNL.js";import{createHash as b,randomUUID as O}from"crypto";var p="mcp-scraper/lead-list-enrichment.requested",A=new Set(["complete","partial","empty","failed","cancelled"]),U=new Set(["complete","failed","cancelled"]),T=10,g=360,S=60,h=null,m=null;function o(e,t,s){let n=e.trim();if(!n||n.length>s)throw new Error(`${t} is required and must be ${s} characters or fewer`);return n}function d(e,t){if(!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer`);return e}function u(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function R(e){let t=new WeakSet,s=n=>{if(n===null||typeof n=="string"||typeof n=="boolean")return n;if(typeof n=="number"){if(!Number.isFinite(n))throw new Error("canonical fingerprint input must contain finite numbers");return Object.is(n,-0)?0:n}if(Array.isArray(n))return n.map(s);if(typeof n=="object"){if(t.has(n))throw new Error("canonical fingerprint input must not be cyclic");t.add(n);let i={};for(let _ of Object.keys(n).sort()){let c=n[_];c!==void 0&&(i[_]=s(c))}return t.delete(n),i}throw new Error(`canonical fingerprint input contains unsupported ${typeof n}`)};return JSON.stringify(s(e))}function y(e){return b("sha256").update(R(e)).digest("hex")}function I(e){return b("sha256").update(R(e)).digest("hex")}function j(){return{mapsAttempts:0,pageAttempts:0,pageSuccesses:0,serpSearches:0}}function E(e){let t=u(e.input_json);if(!t)throw new Error(`lead list enrichment job ${String(e.id??"")} has invalid durable input`);return{id:String(e.id),userId:Number(e.user_id),idempotencyKey:String(e.idempotency_key),requestFingerprint:String(e.request_fingerprint),input:t,status:String(e.status),totalRows:Number(e.total_rows??0),doneRows:Number(e.done_rows??0),failedRows:Number(e.failed_rows??0),cancelledRows:Number(e.cancelled_rows??0),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},concurrencyLockId:e.concurrency_lock_id==null?null:String(e.concurrency_lock_id),debitKey:String(e.debit_key),holdMc:Number(e.hold_mc??0),billedMc:e.billed_mc==null?null:Number(e.billed_mc),result:u(e.result_json),csvArtifact:u(e.csv_artifact_json),xlsxArtifact:u(e.xlsx_artifact_json),warnings:u(e.warnings_json)??[],cancelRequestedAt:e.cancel_requested_at==null?null:String(e.cancel_requested_at),errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:e.retryable==null?null:Number(e.retryable)===1,createdAt:String(e.created_at??""),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function w(e){return{jobId:String(e.job_id),ordinal:Number(e.ordinal),inputDigest:String(e.input_digest),status:String(e.status),result:u(e.result_json),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:Number(e.retryable??0)===1,durationMs:Number(e.duration_ms??0),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function M(e){return{id:String(e.id),jobId:String(e.job_id),eventName:p,status:String(e.status),attempts:Number(e.attempts??0),claimToken:e.locked_by==null?null:String(e.locked_by),lockedUntil:e.locked_until==null?null:String(e.locked_until),nextAttemptAt:e.next_attempt_at==null?null:String(e.next_attempt_at),lastError:e.last_error==null?null:String(e.last_error),createdAt:String(e.created_at??""),dispatchedAt:e.dispatched_at==null?null:String(e.dispatched_at),updatedAt:String(e.updated_at??"")}}async function a(){let e=r();return h!==e&&(h=e,m=null),m||(m=(async()=>{await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_jobs (
2
2
  id TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id), idempotency_key TEXT NOT NULL,
3
3
  request_fingerprint TEXT NOT NULL, input_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'initializing',
4
4
  total_rows INTEGER NOT NULL DEFAULT 0, done_rows INTEGER NOT NULL DEFAULT 0,
@@ -0,0 +1 @@
1
+ var o="0.89.5";export{o as a};
@@ -1,4 +1,4 @@
1
- import{f as q}from"./chunk-QPWPR5XG.js";import{a as ye}from"./chunk-4FROKQJN.js";import{a as ke,b as pe,c as Nt,d as me,g as ve}from"./chunk-D4NOJAUN.js";import{a as ce,c as we,d as Ut,g as ge,l as Ne}from"./chunk-3GP5CYZX.js";import{a as Ce,i as Me,j as E,k as vt,m as _,n as ht,o as Kt}from"./chunk-Z66Z2HV4.js";import{chromium as _t}from"playwright-extra";import fn from"puppeteer-extra-plugin-stealth";import{chromium as Pn}from"playwright";var H={container:".eJH8qe.adDDi",dataInitq:"[data-initq]",item:".related-question-pair",itemDataQ:"data-q",itemDataInitQ:"data-initq",itemQuestionEl:".JlqpRe",answerContainer:".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",aiAnswerBody:".pWvJNd",aiCitation:"a.PMDqCb[href]",aiSourceCard:"a.vIWmYe[href], a.Zbfntb[href]",sourceTitle:"h3",sourceSite:".VuuXrf",sourceCite:"cite",clickTarget:".JlqpRe",expandedClass:"aoRk1c",captchaMarker:'#captcha-form, #recaptcha, form[action*="/sorry/"], .g-recaptcha, [data-sitekey]'},Te={container:'div[jscontroller="HWk0Gf"]',sectionHeading:'.mgAbYb[role="heading"]',item:"a.rIRoqf"},Tt={udm:"39",item:"a.rIRoqf",durationPattern:/^\d+:\d+$/,platforms:["YouTube","TikTok","Instagram","Facebook","X"]},fe={section:".ULSxyf",item:"a.KYg7td.INpicf",title:".hyYc0c",source:".K4ETW"},Pe={sectionTag:"g-section-with-header",sectionHeadingText:"What people are saying",card:'.dRzkFf[role="listitem"]',cardLink:'a.WlydOe[jsname="YKoRaf"]',titleH1:"h1.WQWxe",titleDiv:".eAaXgc",popularCommentLabel:".qgdis",source:".sTl1Td",platformBadge:".appd0, .KrMNbf",ytChannel:".sjVJQd",ytDate:".PLq9Je",authorNote:".nDgy9d"},be={root:"[data-lhcontainer][data-streaming-container][eid]",legacyRoot:'[data-hveid="CBMQAA"]',wrapper:".Fgyi2e",controller:'[jscontroller="AkrxPe"]',contentSubtree:'[data-subtree="mfc"]',header:".heWuVc",heading:".Fzsovc.cwYVJe.RJPOee",showMoreButton:'[aria-label="Show more AI Overview"]',sourcesPanel:".OZ9ddf.WAUd4",disclaimer:".DuQANe.MSJHRb"},We={root:'[data-hveid="CAUQAA"]',wrapper:".Fgyi2e",citations:'.Fgyi2e [data-hveid] a[jsname="pxBnId"]'},Ie={result:".wHYlTd.tF2Cxc",title:"h3.LC20lb",siteName:".VuuXrf",cite:"cite.tjvcx",snippet:".VwiC3b",redditCite:"cite.qLRx3b",ratingWrap:".Y0A0hc",ratingValue:".yi40Hd",reviewCount:".RDApEe"},Ae={headingText:"Businesses",card:".w7Dbne",name:".OSrXXb",ratingValue:".yi40Hd",reviewCount:".RDApEe"};var _a={ratingAndCount:"div.F7nice",hoursTable:"table.eK4R0e",reviewScrollPane:'div.m6QErb[tabindex="-1"]',reviewScrollPaneFallback:'[role="main"] div[tabindex="-1"]',reviewCardAuthor:"div.d4r55, span.d4r55, span.RPZfBb",reviewCardDate:"span.rsqaWe",reviewCardText:"span.wiI7pd",reviewCardOwnerBlock:"div.CDe7pd",hoursTableAlt:'table[aria-label*="Hour"]',reviewCard:"[data-review-id]",reviewStars:'[role="img"][aria-label*="star"]',reviewTab:'button[role="tab"][aria-label*="Review"]',aboutTab:'button[role="tab"][aria-label*="About"]',expandReview:'[data-review-id] button[aria-label*="See more"], [data-review-id] button.w8nwRe'};async function ft(e,t,n,a){let s,i=!1,d,l=()=>{},U=()=>a?.reason instanceof DOMException&&a.reason.name==="TimeoutError"?a.reason:new _,u=()=>l(U()),S=new Promise((r,h)=>{l=h}),o=async r=>{let h;try{await Promise.race([r.close().catch(()=>{}),new Promise(c=>{h=setTimeout(c,2e3)})])}finally{h&&clearTimeout(h)}};if(a?.aborted)throw U();a?.addEventListener("abort",u,{once:!0}),d=setTimeout(()=>l(new DOMException("Optional pagination timed out","TimeoutError")),3e4);try{return await Promise.race([(async()=>{let r=await e.newPage();if(i)throw await o(r),new _;if(s=r,await t(r),i)throw new _;return n(r)})(),S])}finally{i=!0,d&&clearTimeout(d),a?.removeEventListener("abort",u),s&&await o(s)}}var Tn=/\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;async function Pt(e){try{let t=await e.evaluate(()=>navigator.userAgent);return typeof t=="string"&&Tn.test(t)?t:null}catch{return null}}async function bt(e){let t=null;try{let n=e.contexts().flatMap(a=>a.pages())[0];return n||(t=await e.newContext(),n=t.pages()[0]??await t.newPage()),await Pt(n)}catch{return null}finally{t&&await t.close().catch(()=>{})}}function ja(e,t){return t?{...e,userAgent:t}:e}_t.use(fn());var bn=180,Wn=3e3,In=5e3;function An(e,t){let n=process.env[e];if(!n)return t;let a=Number(n);return Number.isInteger(a)&&a>0?a:t}function Jt(e){return e?e.slice(-6):null}function Le(e,t){let n=(t??process.env.SERP_REDIRECT_BASE)?.trim().replace(/\/+$/,"");return n?`${n}/g/${Buffer.from(e,"utf8").toString("base64url")}`:e}function Ln(e){return e?/united states|u\.?s\.?a?\.?|^us$/i.test(e.trim()):!1}function ut(e){return e instanceof Error?e.message:String(e)}function On(e){return/\b409\b|conflict|already exists/i.test(ut(e))}async function xn(e,t){try{await e.profiles.create({name:t})}catch(n){if(!On(n))throw n}}function Wt(e,t){return{viewport:e.viewport,locale:e.locale,...t?{userAgent:t}:{},...e.deviceScaleFactor?{deviceScaleFactor:e.deviceScaleFactor}:{},...e.isMobile!==void 0?{isMobile:e.isMobile}:{},...e.hasTouch!==void 0?{hasTouch:e.hasTouch}:{}}}async function ot(e,t,n){let a;try{return await Promise.race([e,new Promise((s,i)=>{a=setTimeout(()=>i(new Error(`${n} timed out after ${t}ms`)),t)})])}finally{a&&clearTimeout(a)}}function Bn(e){let t=e.trim();if(!t)throw new Error("channelHandle is required");if(/^https?:\/\//i.test(t)||/^(www\.|m\.)?youtube\.com\//i.test(t)){let d=new URL(/^https?:\/\//i.test(t)?t:`https://${t}`);if(d.hostname.replace(/^www\./,"").replace(/^m\./,"").toLowerCase()!=="youtube.com")throw new Error("channel URL must be on youtube.com");let U=d.pathname.split("/").filter(Boolean),u=U[0]??"",S=U[1]??"";if(u.startsWith("@"))return`https://www.youtube.com/${u}/videos`;if(u==="channel"&&S)return`https://www.youtube.com/channel/${S}/videos`;if((u==="c"||u==="user")&&S)return`https://www.youtube.com/${u}/${S}/videos`;throw new Error("channel URL must be a YouTube handle, /channel/UC..., /c/..., or /user/... URL")}let s=t.replace(/^\/+/,"").replace(/\/+$/,"").replace(/\/videos$/i,"");if(/^UC[\w-]{20,}$/.test(s))return`https://www.youtube.com/channel/${s}/videos`;let i=s.startsWith("@")?s:`@${s}`;if(!/^@[\w.-]+$/.test(i))throw new Error("channelHandle must be an @handle, UC channel ID, or YouTube channel URL");return`https://www.youtube.com/${i}/videos`}var It=class{browser=null;context=null;page=null;\u006b\u0065\u0072\u006e\u0065\u006cClient=null;\u006b\u0065\u0072\u006e\u0065\u006cSessionId=null;\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=null;\u006b\u0065\u0072\u006e\u0065\u006cStealth=null;\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed=null;\u006b\u0065\u0072\u006e\u0065\u006cProxySource=null;\u006b\u0065\u0072\u006e\u0065\u006cProxyType=null;\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=null;debugEnabled=!1;debugSnapshot={\u006b\u0065\u0072\u006e\u0065\u006c:null,context:null,networkLocation:null,serpNavigation:null};async launch(t){this.debugEnabled=t.debug===!0;let n=t.proxyMode??(t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId?"configured":"none"),a=t.isMobile?"mobile":"desktop";if(this.debugSnapshot={\u006b\u0065\u0072\u006e\u0065\u006c:null,context:{viewport:t.viewport,locale:t.locale,device:a,userAgent:null,deviceScaleFactor:t.deviceScaleFactor??null,isMobile:t.isMobile===!0,hasTouch:t.hasTouch===!0},networkLocation:null,serpNavigation:null},t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey){this.\u006b\u0065\u0072\u006e\u0065\u006cClient=new q({apiKey:t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey});let i=Number.isInteger(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds)&&Number(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds)>0?Number(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds):An("\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_TIMEOUT_SECONDS",bn);t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName&&t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges===!0&&await xn(this.\u006b\u0065\u0072\u006e\u0065\u006cClient,t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName);let d=Nt(),l;t.headlessMode==="headful"||d?.forceHeadful===!0?l=!1:t.headlessMode==="headless"?l=!0:l=d?.forceHeadless===!0||process.env.\u004b\u0045\u0052\u004e\u0045\u004c_HEADLESS_DISABLED!=="true"&&!!d?.op&&me.has(d.op),this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=l?!0:null,d?.headlessSentOut&&(d.headlessSentOut.value=this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent);let U=await this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.create({stealth:t.\u006b\u0065\u0072\u006e\u0065\u006cStealth===!0,timeout_seconds:i,...l?{headless:!0}:{},...t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId?{proxy_id:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId}:{},...t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName?{profile:{name:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,...typeof t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges=="boolean"?{save_changes:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges}:{}}}:{}});this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId=U.session_id,this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=Date.now(),this.\u006b\u0065\u0072\u006e\u0065\u006cStealth=typeof U.stealth=="boolean"?U.stealth:null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed=typeof U.proxy_id=="string"?!!U.proxy_id:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource=t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.source??null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType=t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.proxyType??null;let u=null,S=null;if(!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId&&!t.keepDefaultProxy)try{await ot(this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.update(this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,{disable_default_proxy:!0}),5e3,`\u004b\u0065\u0072\u006e\u0065\u006c session ${this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId} disable default proxy`),u=!0}catch(r){u=!1,S=ut(r)}let o={sessionId:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,proxyMode:n,requestedProxyIdPresent:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,requestedProxyIdSuffix:Jt(t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId),createdProxyIdPresent:typeof U.proxy_id=="string"?!!U.proxy_id:null,createdProxyIdSuffix:Jt(U.proxy_id),retrievedProxyIdPresent:null,retrievedProxyIdSuffix:null,retrievedProxyIdMatchesRequested:null,defaultProxyDisabled:u,defaultProxyDisableError:S,proxyResolution:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution??null,timeoutSeconds:i,stealth:typeof U.stealth=="boolean"?U.stealth:null,profilePresent:null,poolPresent:null,retrieveError:null};if(this.debugSnapshot.\u006b\u0065\u0072\u006e\u0065\u006c=o,console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_created",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,timeout_seconds:i,proxy_mode:n,proxy_id_present:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,profile_name_present:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,proxy_resolution_source:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.source})),this.debugEnabled&&await this.populate\u004b\u0065\u0072\u006e\u0065\u006cRetrieveDebug(o,t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId),this.browser=await Pn.connectOverCDP(U.cdp_ws_url),t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName||t.preserve\u004b\u0065\u0072\u006e\u0065\u006cContext)this.context=this.browser.contexts()[0]??await this.browser.newContext(Wt(t)),this.page=this.context.pages()[0]??await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{let r=await bt(this.browser);this.debugSnapshot.context.userAgent=r,this.context=await this.browser.newContext(Wt(t,r)),this.page=await this.context.newPage()}if(await this.installEsbuildHelperShims(this.context),t.preserve\u004b\u0065\u0072\u006e\u0065\u006cContext||await this.page.setViewportSize(t.viewport),(this.debugEnabled||t.requireUsEgress)&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation()),t.requireUsEgress){let r=this.debugSnapshot.networkLocation?.country??null;if(!Ln(r))throw new E(`Non-US egress (${r??"unknown"}) \u2014 retrying with a fresh session`)}return}let s={headless:t.headless,proxy:t.proxy?{server:t.proxy}:void 0,...t.browserExecutablePath?{executablePath:t.browserExecutablePath}:{},...t.ignoreDefaultArgs?{ignoreDefaultArgs:t.ignoreDefaultArgs}:{}};if(t.profileDir)this.context=await _t.launchPersistentContext(t.profileDir,{...s,...Wt(t)}),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{this.browser=await _t.launch(s);let i=await bt(this.browser);this.debugSnapshot.context.userAgent=i,this.context=await this.browser.newContext(Wt(t,i)),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage()}this.debugEnabled&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation())}async populate\u004b\u0065\u0072\u006e\u0065\u006cRetrieveDebug(t,n){if(!(!this.\u006b\u0065\u0072\u006e\u0065\u006cClient||!this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId))try{let a=await ot(this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.retrieve(this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId),5e3,`\u004b\u0065\u0072\u006e\u0065\u006c session ${this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId} retrieve`);t.retrievedProxyIdPresent=typeof a.proxy_id=="string"?!!a.proxy_id:!1,t.retrievedProxyIdSuffix=Jt(a.proxy_id),t.retrievedProxyIdMatchesRequested=n?a.proxy_id===n:!a.proxy_id,t.timeoutSeconds=typeof a.timeout_seconds=="number"?a.timeout_seconds:t.timeoutSeconds,t.stealth=typeof a.stealth=="boolean"?a.stealth:t.stealth,t.profilePresent=!!a.profile,t.poolPresent=!!a.pool}catch(a){t.retrieveError=ut(a)}}async captureBrowserNetworkLocation(){let t=(a,s="ipapi.co")=>({source:s,ip:null,city:null,region:null,country:null,org:null,timezone:null,error:a});if(!this.context)return t("browser context is not available");let n=null;try{n=await this.context.newPage();let a=await this.loadJsonInDebugPage(n,"https://ipwho.is/");if(a){let l=typeof a.connection=="object"&&a.connection!==null?a.connection:{};return{source:"ipwho.is",ip:typeof a.ip=="string"?a.ip:null,city:typeof a.city=="string"?a.city:null,region:typeof a.region=="string"?a.region:null,country:typeof a.country=="string"?a.country:null,org:typeof l.org=="string"?l.org:null,timezone:typeof a.timezone=="object"&&a.timezone!==null&&typeof a.timezone.id=="string"?a.timezone.id:null,error:null}}let s=await this.loadJsonInDebugPage(n,"https://api64.ipify.org?format=json");if(s)return{source:"api64.ipify.org",ip:typeof s.ip=="string"?s.ip:null,city:null,region:null,country:null,org:null,timezone:null,error:null};await ot(n.goto("https://ipapi.co/json/",{waitUntil:"domcontentloaded",timeout:7e3}),8e3,"browser network location navigation");let i=await n.locator("body").innerText({timeout:2e3}),d=JSON.parse(i);return{source:"ipapi.co",ip:typeof d.ip=="string"?d.ip:null,city:typeof d.city=="string"?d.city:null,region:typeof d.region=="string"?d.region:null,country:typeof d.country_name=="string"?d.country_name:typeof d.country=="string"?d.country:null,org:typeof d.org=="string"?d.org:null,timezone:typeof d.timezone=="string"?d.timezone:null,error:null}}catch(a){return t(ut(a))}finally{await n?.close().catch(()=>{})}}async loadJsonInDebugPage(t,n){try{await ot(t.goto(n,{waitUntil:"domcontentloaded",timeout:7e3}),8e3,`browser network location navigation ${n}`);let a=await t.locator("body").innerText({timeout:2e3});return JSON.parse(a)}catch{return null}}async installEsbuildHelperShims(t){await t.addInitScript(()=>{let n=globalThis;typeof n.__name!="function"&&(n.__name=a=>a),typeof n.__publicField!="function"&&(n.__publicField=(a,s,i)=>(a[s]=i,i))})}async withTemporaryPage(t,n){if(!this.context)throw new Error("Browser context is not available");return ft(this.context,async()=>{},t,n)}async navigateToSERP(t,n,a,s,i){let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n);let l="https://www.google.com/search?"+d.toString(),U=Le(l),u=i?.debug?{requestedUrl:l,finalUrl:null,title:null,bodySnippet:null,hasPaa:null,captchaDetected:null,googleSorryUrl:null,redirected:null}:null;u&&(this.debugSnapshot.serpNavigation=u);try{await this.page.goto(U,{waitUntil:"domcontentloaded",timeout:45e3})}catch(o){await this.updateSerpNavigationDebug(u,l,{hasPaa:null,captchaDetected:null});let r=await this.captureDiagnostics(l);throw new vt(`page.goto failed: ${o.message} | ${r}`)}if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());if(await this.page.waitForSelector(H.item,{timeout:4e3}).catch(()=>null))return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());for(let o=1;o<=6;o++)if(await this.page.evaluate(h=>{window.scrollTo(0,document.body.scrollHeight*h)},o/6),await this.page.waitForTimeout(600),await this.page.locator(H.item).count()>0)return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};return await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!1}),{hasPaa:!1}}async isSerpChallengeVisible(){if(!this.page)return!1;if(await this.page.locator(H.captchaMarker).count().catch(()=>0)>0)return!0;let n=this.page.url();if(/google\.[^/]+\/sorry\//i.test(n))return!0;let a=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,1e3)).catch(()=>"");return/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(a)}async updateSerpNavigationDebug(t,n,a){if(!(!t||!this.page))try{let s=this.page.url(),i=await this.page.title().catch(()=>""),d=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,500)).catch(()=>""),l=/recaptcha|unusual traffic|are you a robot/i.test(d);t.finalUrl=s,t.title=i,t.bodySnippet=d,t.hasPaa=a.hasPaa,t.captchaDetected=a.captchaDetected??l,t.googleSorryUrl=/google\.[^/]+\/sorry\//i.test(s),t.redirected=s!==n}catch(s){t.bodySnippet=`debug capture failed: ${ut(s)}`}}async captureDiagnostics(t){try{let n=this.page.url(),a=await this.page.title().catch(()=>""),s=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,400)).catch(()=>""),i=/consent\.google\./.test(n)||/before you continue/i.test(s),d=/recaptcha|unusual traffic|are you a robot/i.test(s),l=[i?"CONSENT_WALL":"",d?"BOT_CHALLENGE":"",n!==t?"REDIRECTED":""].filter(Boolean).join(",");return`intended=${t} | final=${n} | title="${a}" | flags=[${l}] | body="${s}"`}catch(n){return`diagnostics-failed: ${n.message}`}}captchaMessage(){return this.\u006b\u0065\u0072\u006e\u0065\u006cClient?"Google returned a CAPTCHA on this session \u2014 retrying with a fresh session.":Ce}async navigateTo(t){let n=process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim(),a=n?Le(t,n):t;try{if(await this.page.goto(a,{waitUntil:"domcontentloaded",timeout:45e3}),/google\.[^/]+\/search\?/i.test(t)&&await this.isSerpChallengeVisible())throw new E(this.captchaMessage())}catch(s){if(s instanceof E)throw s;let i=await this.captureDiagnostics(t);throw new vt(`page.goto failed: ${s.message} | ${i}`)}}async navigateToChannel(t){let n=Bn(t);try{await this.page.goto(n,{waitUntil:"networkidle",timeout:3e4})}catch(a){let s=await this.captureDiagnostics(n);throw new vt(`navigateToChannel failed: ${a.message} | ${s}`)}}async evaluate(t,n){return this.page.evaluate(t,n)}getPage(){return this.page}get\u004b\u0065\u0072\u006e\u0065\u006cSessionId(){return this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId}getProviderSessionMetadata(){return{provider:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId?"\u006b\u0065\u0072\u006e\u0065\u006c":"local",providerSessionId:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,disconnectObservation:null}}getDebugSnapshot(){return this.debugSnapshot}async close(){if(this.browser){let t=this.browser,n=this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,a=this.\u006b\u0065\u0072\u006e\u0065\u006cClient,s=this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs,i=this.\u006b\u0065\u0072\u006e\u0065\u006cStealth,d=this.\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed,l=this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource,U=this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType,u=this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent,S=this.getProviderSessionMetadata();if(this.browser=null,this.context=null,this.page=null,this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId=null,this.\u006b\u0065\u0072\u006e\u0065\u006cClient=null,this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=null,this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource=null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType=null,a&&n&&s!=null&&ve({\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n,source:"browser_driver",stealth:i,headlessSent:u,proxyUsed:d,proxySource:l,proxyType:U,openedAtMs:s,closedAtMs:Date.now()}),a&&n){console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_started",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n}));let o=ot(a.browsers.deleteByID(n),In,`\u004b\u0065\u0072\u006e\u0065\u006c session ${n} delete`),r=ot(t.close(),Wn,`\u004b\u0065\u0072\u006e\u0065\u006c browser ${n} close`),[h,c]=await Promise.allSettled([o,r]),C={\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!0,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:h.status==="fulfilled",\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:h.status==="rejected"?h.reason instanceof Error?h.reason.message:String(h.reason):null,browserCloseSucceeded:c.status==="fulfilled",browserCloseError:c.status==="rejected"?c.reason instanceof Error?c.reason.message:String(c.reason):null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null};return h.status==="rejected"?(console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_failed",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n,message:C.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError})),console.warn(`\u004b\u0065\u0072\u006e\u0065\u006c session cleanup failed for ${n}:`,h.reason)):console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_succeeded",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n})),c.status==="rejected"&&(console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_close_failed",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n,message:C.browserCloseError})),console.warn(`\u004b\u0065\u0072\u006e\u0065\u006c browser close failed for ${n}:`,c.reason)),C}return await t.close(),{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null}}else if(this.context){let t=this.context;return this.context=null,this.page=null,await t.close(),{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}return{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:null,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}};import{chromium as Rn}from"playwright";function Hn(e,t){if(typeof e!="string")return null;let n=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();return n?n.slice(0,t):null}function Oe(e){let t=Hn(e,320);return t?t.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization):\s*\S+/gi,"$1: [redacted]").replace(/\bBearer\s+\S+/gi,"Bearer [redacted]").slice(0,320):null}function xe(e){return Oe(e)}function Be(e){if(typeof e!="string"||e.length===0||e.length>512||e.trim()!==e||/[\s\u0000-\u001f\u007f]/.test(e)||/:\/\//.test(e))throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 did not return a valid browser session ID");return e}function zt(e,t,n=new Date().toISOString()){let a=Oe(t);return{source:e,message:a??e.replaceAll("_"," "),observedAt:n}}function Fn(e){return{provider:e.provider,providerDisconnectObserved:e.disconnectObservation!==null,providerDisconnectMessage:e.disconnectObservation?.message??null}}function He(e){return{...Fn(e),providerSessionId:e.providerSessionId}}var Gn=new Set(["image","media","font","stylesheet","ping"]),Dn=["doubleclick.net","googleadservices.com","googlesyndication.com","google-analytics.com"];function Fe(){return process.env.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_WSS?.trim()||null}function Re(){return process.env.SERP_BROWSER_PROVIDER?.trim().toLowerCase()==="\u006b\u0065\u0072\u006e\u0065\u006c"?!1:!!Fe()}async function qt(e,t=!1){let n=e.request();try{let a=await n.allHeaders(),s=await fetch(n.url(),{method:n.method(),headers:{accept:a.accept??"*/*","accept-language":a["accept-language"]??"en-US,en;q=0.9","user-agent":a["user-agent"]??"Mozilla/5.0",...t?{cookie:a.cookie??"",referer:a.referer??"https://www.google.com/",...a["x-client-data"]?{"x-client-data":a["x-client-data"]}:{}}:{}},body:n.method()==="GET"||n.method()==="HEAD"?void 0:n.postData()});if(!s.ok)return!1;let i=Buffer.from(await s.arrayBuffer());return await e.fulfill({status:s.status,contentType:s.headers.get("content-type")??"application/octet-stream",headers:{"cache-control":"public, max-age=3600"},body:i}),!0}catch{return!1}}var At=class{browser=null;context=null;page=null;cdpSession=null;providerSessionId=null;disconnectObservation=null;closing=!1;observeDisconnect(t,n){this.closing||this.disconnectObservation||(this.disconnectObservation=zt(t,n))}async launch(t){if(this.page)return;this.closing=!1,this.providerSessionId=null,this.disconnectObservation=null;let n=Fe();if(!n)throw new Error("SERP browser service is not configured");this.browser=await Rn.connectOverCDP(n),this.context=this.browser.contexts()[0]??await this.browser.newContext(),await this.context.addInitScript(()=>{let i=globalThis;typeof i.__name!="function"&&(i.__name=d=>d),typeof i.__publicField!="function"&&(i.__publicField=(d,l,U)=>(d[l]=U,U))}),this.page=this.context.pages()[0]??await this.context.newPage(),this.browser.on("disconnected",()=>this.observeDisconnect("browser_disconnected")),this.context.on("close",()=>this.observeDisconnect("context_closed")),this.page.on("close",()=>this.observeDisconnect("page_closed")),this.page.on("crash",()=>this.observeDisconnect("page_crashed")),this.cdpSession=await this.context.newCDPSession(this.page);let s=await this.cdpSession.send("Browser.getSessionId");this.providerSessionId=Be(s?.sessionId),await this.setupPage(this.page)}async setupPage(t){await t.route("**/*",async n=>{let a=n.request(),s=a.resourceType(),i;try{i=new URL(a.url())}catch{await n.continue();return}let d=i.hostname.toLowerCase();if(!(i.pathname==="/sgasync"&&await qt(n,!0))&&!(i.pathname.includes("/shared_dict/")&&await qt(n))&&!(s==="script"&&(d==="www.google.com"||d==="www.gstatic.com")&&await qt(n))){if(Gn.has(s)){await n.abort("blockedbyclient");return}if(Dn.some(l=>d===l||d.endsWith(`.${l}`))){await n.abort("blockedbyclient");return}await n.continue()}})}async withTemporaryPage(t,n){if(!this.context||this.closing)throw new Error("Browser context is not available");return ft(this.context,a=>this.setupPage(a),t,n)}async navigateToSERP(t,n,a,s,i){if(!this.page)throw new Error("Browser page is not available");let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});return i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n),await this.page.goto(`https://www.google.com/search?${d}`,{waitUntil:"load",timeout:45e3}),{hasPaa:!!await this.page.waitForSelector(H.item,{timeout:8e3}).catch(()=>null)}}async navigateTo(t){if(!this.page)throw new Error("Browser page is not available");await this.page.goto(t,{waitUntil:"load",timeout:45e3})}async navigateToChannel(){throw new Error("YouTube navigation is not supported by the SERP browser driver")}async evaluate(t,n){if(!this.page)throw new Error("Browser page is not available");return this.page.evaluate(t,n)}getPage(){if(!this.page)throw new Error("Browser page is not available");return this.page}get\u004b\u0065\u0072\u006e\u0065\u006cSessionId(){return null}getProviderSessionMetadata(){return{provider:"\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api",providerSessionId:this.providerSessionId,disconnectObservation:this.disconnectObservation}}getDebugSnapshot(){return{\u006b\u0065\u0072\u006e\u0065\u006c:null,context:null,networkLocation:null,serpNavigation:null}}async close(){this.closing=!0;let t=null;try{await this.browser?.close()}catch(a){t=xe(a instanceof Error?a.message:String(a))??"\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 browser close failed",this.disconnectObservation??=zt("browser_close_error",t)}let n=He(this.getProviderSessionMetadata());return this.browser=null,this.context=null,this.page=null,this.cdpSession=null,{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:t===null,browserCloseError:t,...n}}};function Ge(e,t){let n=!!(e?.serpIdentity||e?.\u006b\u0065\u0072\u006e\u0065\u006cProfileName);return t?.forceManaged===!0||Re()&&!n?new At:new It}var tt={austin:"Austin,Texas,United States","new york":"New York,New York,United States","new york city":"New York,New York,United States",nyc:"New York,New York,United States","los angeles":"Los Angeles,California,United States",la:"Los Angeles,California,United States",chicago:"Chicago,Illinois,United States",houston:"Houston,Texas,United States",phoenix:"Phoenix,Arizona,United States",philadelphia:"Philadelphia,Pennsylvania,United States",philly:"Philadelphia,Pennsylvania,United States","san antonio":"San Antonio,Texas,United States",dallas:"Dallas,Texas,United States",miami:"Miami,Florida,United States",seattle:"Seattle,Washington,United States",denver:"Denver,Colorado,United States",loveland:"Loveland,Colorado,United States","loveland co":"Loveland,Colorado,United States","fort collins":"Fort Collins,Colorado,United States",boulder:"Boulder,Colorado,United States","colorado springs":"Colorado Springs,Colorado,United States",boston:"Boston,Massachusetts,United States",atlanta:"Atlanta,Georgia,United States","san francisco":"San Francisco,California,United States",sf:"San Francisco,California,United States",portland:"Portland,Oregon,United States","las vegas":"Las Vegas,Nevada,United States",minneapolis:"Minneapolis,Minnesota,United States",detroit:"Detroit,Michigan,United States",nashville:"Nashville,Tennessee,United States",charlotte:"Charlotte,North Carolina,United States",orlando:"Orlando,Florida,United States","san diego":"San Diego,California,United States",baltimore:"Baltimore,Maryland,United States",sacramento:"Sacramento,California,United States",columbus:"Columbus,Ohio,United States",indianapolis:"Indianapolis,Indiana,United States","san jose":"San Jose,California,United States","fort worth":"Fort Worth,Texas,United States",jacksonville:"Jacksonville,Florida,United States",memphis:"Memphis,Tennessee,United States",louisville:"Louisville,Kentucky,United States",raleigh:"Raleigh,North Carolina,United States",richmond:"Richmond,Virginia,United States","salt lake city":"Salt Lake City,Utah,United States",toronto:"Toronto,Ontario,Canada",vancouver:"Vancouver,British Columbia,Canada",montreal:"Montreal,Quebec,Canada",calgary:"Calgary,Alberta,Canada",ottawa:"Ottawa,Ontario,Canada",london:"London,England,United Kingdom",manchester:"Manchester,England,United Kingdom",birmingham:"Birmingham,England,United Kingdom",edinburgh:"Edinburgh,Scotland,United Kingdom",glasgow:"Glasgow,Scotland,United Kingdom",leeds:"Leeds,England,United Kingdom",sydney:"Sydney,New South Wales,Australia",melbourne:"Melbourne,Victoria,Australia",brisbane:"Brisbane,Queensland,Australia",perth:"Perth,Western Australia,Australia",adelaide:"Adelaide,South Australia,Australia",dublin:"Dublin,Leinster,Ireland"};var De=`3, Nanjemoy,Maryland,United States
1
+ import{f as q}from"./chunk-QPWPR5XG.js";import{a as ye}from"./chunk-4FROKQJN.js";import{a as ke,b as pe,c as Nt,d as me,g as ve}from"./chunk-53HIX3X7.js";import{a as ce,c as we,d as Ut,g as ge,l as Ne}from"./chunk-3GP5CYZX.js";import{a as Ce,i as Me,j as E,k as vt,m as _,n as ht,o as Kt}from"./chunk-7JAMFXNL.js";import{chromium as _t}from"playwright-extra";import fn from"puppeteer-extra-plugin-stealth";import{chromium as Pn}from"playwright";var H={container:".eJH8qe.adDDi",dataInitq:"[data-initq]",item:".related-question-pair",itemDataQ:"data-q",itemDataInitQ:"data-initq",itemQuestionEl:".JlqpRe",answerContainer:".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",aiAnswerBody:".pWvJNd",aiCitation:"a.PMDqCb[href]",aiSourceCard:"a.vIWmYe[href], a.Zbfntb[href]",sourceTitle:"h3",sourceSite:".VuuXrf",sourceCite:"cite",clickTarget:".JlqpRe",expandedClass:"aoRk1c",captchaMarker:'#captcha-form, #recaptcha, form[action*="/sorry/"], .g-recaptcha, [data-sitekey]'},Te={container:'div[jscontroller="HWk0Gf"]',sectionHeading:'.mgAbYb[role="heading"]',item:"a.rIRoqf"},Tt={udm:"39",item:"a.rIRoqf",durationPattern:/^\d+:\d+$/,platforms:["YouTube","TikTok","Instagram","Facebook","X"]},fe={section:".ULSxyf",item:"a.KYg7td.INpicf",title:".hyYc0c",source:".K4ETW"},Pe={sectionTag:"g-section-with-header",sectionHeadingText:"What people are saying",card:'.dRzkFf[role="listitem"]',cardLink:'a.WlydOe[jsname="YKoRaf"]',titleH1:"h1.WQWxe",titleDiv:".eAaXgc",popularCommentLabel:".qgdis",source:".sTl1Td",platformBadge:".appd0, .KrMNbf",ytChannel:".sjVJQd",ytDate:".PLq9Je",authorNote:".nDgy9d"},be={root:"[data-lhcontainer][data-streaming-container][eid]",legacyRoot:'[data-hveid="CBMQAA"]',wrapper:".Fgyi2e",controller:'[jscontroller="AkrxPe"]',contentSubtree:'[data-subtree="mfc"]',header:".heWuVc",heading:".Fzsovc.cwYVJe.RJPOee",showMoreButton:'[aria-label="Show more AI Overview"]',sourcesPanel:".OZ9ddf.WAUd4",disclaimer:".DuQANe.MSJHRb"},We={root:'[data-hveid="CAUQAA"]',wrapper:".Fgyi2e",citations:'.Fgyi2e [data-hveid] a[jsname="pxBnId"]'},Ie={result:".wHYlTd.tF2Cxc",title:"h3.LC20lb",siteName:".VuuXrf",cite:"cite.tjvcx",snippet:".VwiC3b",redditCite:"cite.qLRx3b",ratingWrap:".Y0A0hc",ratingValue:".yi40Hd",reviewCount:".RDApEe"},Ae={headingText:"Businesses",card:".w7Dbne",name:".OSrXXb",ratingValue:".yi40Hd",reviewCount:".RDApEe"};var _a={ratingAndCount:"div.F7nice",hoursTable:"table.eK4R0e",reviewScrollPane:'div.m6QErb[tabindex="-1"]',reviewScrollPaneFallback:'[role="main"] div[tabindex="-1"]',reviewCardAuthor:"div.d4r55, span.d4r55, span.RPZfBb",reviewCardDate:"span.rsqaWe",reviewCardText:"span.wiI7pd",reviewCardOwnerBlock:"div.CDe7pd",hoursTableAlt:'table[aria-label*="Hour"]',reviewCard:"[data-review-id]",reviewStars:'[role="img"][aria-label*="star"]',reviewTab:'button[role="tab"][aria-label*="Review"]',aboutTab:'button[role="tab"][aria-label*="About"]',expandReview:'[data-review-id] button[aria-label*="See more"], [data-review-id] button.w8nwRe'};async function ft(e,t,n,a){let s,i=!1,d,l=()=>{},U=()=>a?.reason instanceof DOMException&&a.reason.name==="TimeoutError"?a.reason:new _,u=()=>l(U()),S=new Promise((r,h)=>{l=h}),o=async r=>{let h;try{await Promise.race([r.close().catch(()=>{}),new Promise(c=>{h=setTimeout(c,2e3)})])}finally{h&&clearTimeout(h)}};if(a?.aborted)throw U();a?.addEventListener("abort",u,{once:!0}),d=setTimeout(()=>l(new DOMException("Optional pagination timed out","TimeoutError")),3e4);try{return await Promise.race([(async()=>{let r=await e.newPage();if(i)throw await o(r),new _;if(s=r,await t(r),i)throw new _;return n(r)})(),S])}finally{i=!0,d&&clearTimeout(d),a?.removeEventListener("abort",u),s&&await o(s)}}var Tn=/\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;async function Pt(e){try{let t=await e.evaluate(()=>navigator.userAgent);return typeof t=="string"&&Tn.test(t)?t:null}catch{return null}}async function bt(e){let t=null;try{let n=e.contexts().flatMap(a=>a.pages())[0];return n||(t=await e.newContext(),n=t.pages()[0]??await t.newPage()),await Pt(n)}catch{return null}finally{t&&await t.close().catch(()=>{})}}function ja(e,t){return t?{...e,userAgent:t}:e}_t.use(fn());var bn=180,Wn=3e3,In=5e3;function An(e,t){let n=process.env[e];if(!n)return t;let a=Number(n);return Number.isInteger(a)&&a>0?a:t}function Jt(e){return e?e.slice(-6):null}function Le(e,t){let n=(t??process.env.SERP_REDIRECT_BASE)?.trim().replace(/\/+$/,"");return n?`${n}/g/${Buffer.from(e,"utf8").toString("base64url")}`:e}function Ln(e){return e?/united states|u\.?s\.?a?\.?|^us$/i.test(e.trim()):!1}function ut(e){return e instanceof Error?e.message:String(e)}function On(e){return/\b409\b|conflict|already exists/i.test(ut(e))}async function xn(e,t){try{await e.profiles.create({name:t})}catch(n){if(!On(n))throw n}}function Wt(e,t){return{viewport:e.viewport,locale:e.locale,...t?{userAgent:t}:{},...e.deviceScaleFactor?{deviceScaleFactor:e.deviceScaleFactor}:{},...e.isMobile!==void 0?{isMobile:e.isMobile}:{},...e.hasTouch!==void 0?{hasTouch:e.hasTouch}:{}}}async function ot(e,t,n){let a;try{return await Promise.race([e,new Promise((s,i)=>{a=setTimeout(()=>i(new Error(`${n} timed out after ${t}ms`)),t)})])}finally{a&&clearTimeout(a)}}function Bn(e){let t=e.trim();if(!t)throw new Error("channelHandle is required");if(/^https?:\/\//i.test(t)||/^(www\.|m\.)?youtube\.com\//i.test(t)){let d=new URL(/^https?:\/\//i.test(t)?t:`https://${t}`);if(d.hostname.replace(/^www\./,"").replace(/^m\./,"").toLowerCase()!=="youtube.com")throw new Error("channel URL must be on youtube.com");let U=d.pathname.split("/").filter(Boolean),u=U[0]??"",S=U[1]??"";if(u.startsWith("@"))return`https://www.youtube.com/${u}/videos`;if(u==="channel"&&S)return`https://www.youtube.com/channel/${S}/videos`;if((u==="c"||u==="user")&&S)return`https://www.youtube.com/${u}/${S}/videos`;throw new Error("channel URL must be a YouTube handle, /channel/UC..., /c/..., or /user/... URL")}let s=t.replace(/^\/+/,"").replace(/\/+$/,"").replace(/\/videos$/i,"");if(/^UC[\w-]{20,}$/.test(s))return`https://www.youtube.com/channel/${s}/videos`;let i=s.startsWith("@")?s:`@${s}`;if(!/^@[\w.-]+$/.test(i))throw new Error("channelHandle must be an @handle, UC channel ID, or YouTube channel URL");return`https://www.youtube.com/${i}/videos`}var It=class{browser=null;context=null;page=null;\u006b\u0065\u0072\u006e\u0065\u006cClient=null;\u006b\u0065\u0072\u006e\u0065\u006cSessionId=null;\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=null;\u006b\u0065\u0072\u006e\u0065\u006cStealth=null;\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed=null;\u006b\u0065\u0072\u006e\u0065\u006cProxySource=null;\u006b\u0065\u0072\u006e\u0065\u006cProxyType=null;\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=null;debugEnabled=!1;debugSnapshot={\u006b\u0065\u0072\u006e\u0065\u006c:null,context:null,networkLocation:null,serpNavigation:null};async launch(t){this.debugEnabled=t.debug===!0;let n=t.proxyMode??(t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId?"configured":"none"),a=t.isMobile?"mobile":"desktop";if(this.debugSnapshot={\u006b\u0065\u0072\u006e\u0065\u006c:null,context:{viewport:t.viewport,locale:t.locale,device:a,userAgent:null,deviceScaleFactor:t.deviceScaleFactor??null,isMobile:t.isMobile===!0,hasTouch:t.hasTouch===!0},networkLocation:null,serpNavigation:null},t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey){this.\u006b\u0065\u0072\u006e\u0065\u006cClient=new q({apiKey:t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey});let i=Number.isInteger(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds)&&Number(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds)>0?Number(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds):An("\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_TIMEOUT_SECONDS",bn);t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName&&t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges===!0&&await xn(this.\u006b\u0065\u0072\u006e\u0065\u006cClient,t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName);let d=Nt(),l;t.headlessMode==="headful"||d?.forceHeadful===!0?l=!1:t.headlessMode==="headless"?l=!0:l=d?.forceHeadless===!0||process.env.\u004b\u0045\u0052\u004e\u0045\u004c_HEADLESS_DISABLED!=="true"&&!!d?.op&&me.has(d.op),this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=l?!0:null,d?.headlessSentOut&&(d.headlessSentOut.value=this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent);let U=await this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.create({stealth:t.\u006b\u0065\u0072\u006e\u0065\u006cStealth===!0,timeout_seconds:i,...l?{headless:!0}:{},...t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId?{proxy_id:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId}:{},...t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName?{profile:{name:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,...typeof t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges=="boolean"?{save_changes:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges}:{}}}:{}});this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId=U.session_id,this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=Date.now(),this.\u006b\u0065\u0072\u006e\u0065\u006cStealth=typeof U.stealth=="boolean"?U.stealth:null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed=typeof U.proxy_id=="string"?!!U.proxy_id:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource=t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.source??null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType=t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.proxyType??null;let u=null,S=null;if(!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId&&!t.keepDefaultProxy)try{await ot(this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.update(this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,{disable_default_proxy:!0}),5e3,`\u004b\u0065\u0072\u006e\u0065\u006c session ${this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId} disable default proxy`),u=!0}catch(r){u=!1,S=ut(r)}let o={sessionId:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,proxyMode:n,requestedProxyIdPresent:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,requestedProxyIdSuffix:Jt(t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId),createdProxyIdPresent:typeof U.proxy_id=="string"?!!U.proxy_id:null,createdProxyIdSuffix:Jt(U.proxy_id),retrievedProxyIdPresent:null,retrievedProxyIdSuffix:null,retrievedProxyIdMatchesRequested:null,defaultProxyDisabled:u,defaultProxyDisableError:S,proxyResolution:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution??null,timeoutSeconds:i,stealth:typeof U.stealth=="boolean"?U.stealth:null,profilePresent:null,poolPresent:null,retrieveError:null};if(this.debugSnapshot.\u006b\u0065\u0072\u006e\u0065\u006c=o,console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_created",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,timeout_seconds:i,proxy_mode:n,proxy_id_present:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,profile_name_present:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,proxy_resolution_source:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.source})),this.debugEnabled&&await this.populate\u004b\u0065\u0072\u006e\u0065\u006cRetrieveDebug(o,t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId),this.browser=await Pn.connectOverCDP(U.cdp_ws_url),t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName||t.preserve\u004b\u0065\u0072\u006e\u0065\u006cContext)this.context=this.browser.contexts()[0]??await this.browser.newContext(Wt(t)),this.page=this.context.pages()[0]??await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{let r=await bt(this.browser);this.debugSnapshot.context.userAgent=r,this.context=await this.browser.newContext(Wt(t,r)),this.page=await this.context.newPage()}if(await this.installEsbuildHelperShims(this.context),t.preserve\u004b\u0065\u0072\u006e\u0065\u006cContext||await this.page.setViewportSize(t.viewport),(this.debugEnabled||t.requireUsEgress)&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation()),t.requireUsEgress){let r=this.debugSnapshot.networkLocation?.country??null;if(!Ln(r))throw new E(`Non-US egress (${r??"unknown"}) \u2014 retrying with a fresh session`)}return}let s={headless:t.headless,proxy:t.proxy?{server:t.proxy}:void 0,...t.browserExecutablePath?{executablePath:t.browserExecutablePath}:{},...t.ignoreDefaultArgs?{ignoreDefaultArgs:t.ignoreDefaultArgs}:{}};if(t.profileDir)this.context=await _t.launchPersistentContext(t.profileDir,{...s,...Wt(t)}),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{this.browser=await _t.launch(s);let i=await bt(this.browser);this.debugSnapshot.context.userAgent=i,this.context=await this.browser.newContext(Wt(t,i)),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage()}this.debugEnabled&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation())}async populate\u004b\u0065\u0072\u006e\u0065\u006cRetrieveDebug(t,n){if(!(!this.\u006b\u0065\u0072\u006e\u0065\u006cClient||!this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId))try{let a=await ot(this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.retrieve(this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId),5e3,`\u004b\u0065\u0072\u006e\u0065\u006c session ${this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId} retrieve`);t.retrievedProxyIdPresent=typeof a.proxy_id=="string"?!!a.proxy_id:!1,t.retrievedProxyIdSuffix=Jt(a.proxy_id),t.retrievedProxyIdMatchesRequested=n?a.proxy_id===n:!a.proxy_id,t.timeoutSeconds=typeof a.timeout_seconds=="number"?a.timeout_seconds:t.timeoutSeconds,t.stealth=typeof a.stealth=="boolean"?a.stealth:t.stealth,t.profilePresent=!!a.profile,t.poolPresent=!!a.pool}catch(a){t.retrieveError=ut(a)}}async captureBrowserNetworkLocation(){let t=(a,s="ipapi.co")=>({source:s,ip:null,city:null,region:null,country:null,org:null,timezone:null,error:a});if(!this.context)return t("browser context is not available");let n=null;try{n=await this.context.newPage();let a=await this.loadJsonInDebugPage(n,"https://ipwho.is/");if(a){let l=typeof a.connection=="object"&&a.connection!==null?a.connection:{};return{source:"ipwho.is",ip:typeof a.ip=="string"?a.ip:null,city:typeof a.city=="string"?a.city:null,region:typeof a.region=="string"?a.region:null,country:typeof a.country=="string"?a.country:null,org:typeof l.org=="string"?l.org:null,timezone:typeof a.timezone=="object"&&a.timezone!==null&&typeof a.timezone.id=="string"?a.timezone.id:null,error:null}}let s=await this.loadJsonInDebugPage(n,"https://api64.ipify.org?format=json");if(s)return{source:"api64.ipify.org",ip:typeof s.ip=="string"?s.ip:null,city:null,region:null,country:null,org:null,timezone:null,error:null};await ot(n.goto("https://ipapi.co/json/",{waitUntil:"domcontentloaded",timeout:7e3}),8e3,"browser network location navigation");let i=await n.locator("body").innerText({timeout:2e3}),d=JSON.parse(i);return{source:"ipapi.co",ip:typeof d.ip=="string"?d.ip:null,city:typeof d.city=="string"?d.city:null,region:typeof d.region=="string"?d.region:null,country:typeof d.country_name=="string"?d.country_name:typeof d.country=="string"?d.country:null,org:typeof d.org=="string"?d.org:null,timezone:typeof d.timezone=="string"?d.timezone:null,error:null}}catch(a){return t(ut(a))}finally{await n?.close().catch(()=>{})}}async loadJsonInDebugPage(t,n){try{await ot(t.goto(n,{waitUntil:"domcontentloaded",timeout:7e3}),8e3,`browser network location navigation ${n}`);let a=await t.locator("body").innerText({timeout:2e3});return JSON.parse(a)}catch{return null}}async installEsbuildHelperShims(t){await t.addInitScript(()=>{let n=globalThis;typeof n.__name!="function"&&(n.__name=a=>a),typeof n.__publicField!="function"&&(n.__publicField=(a,s,i)=>(a[s]=i,i))})}async withTemporaryPage(t,n){if(!this.context)throw new Error("Browser context is not available");return ft(this.context,async()=>{},t,n)}async navigateToSERP(t,n,a,s,i){let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n);let l="https://www.google.com/search?"+d.toString(),U=Le(l),u=i?.debug?{requestedUrl:l,finalUrl:null,title:null,bodySnippet:null,hasPaa:null,captchaDetected:null,googleSorryUrl:null,redirected:null}:null;u&&(this.debugSnapshot.serpNavigation=u);try{await this.page.goto(U,{waitUntil:"domcontentloaded",timeout:45e3})}catch(o){await this.updateSerpNavigationDebug(u,l,{hasPaa:null,captchaDetected:null});let r=await this.captureDiagnostics(l);throw new vt(`page.goto failed: ${o.message} | ${r}`)}if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());if(await this.page.waitForSelector(H.item,{timeout:4e3}).catch(()=>null))return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());for(let o=1;o<=6;o++)if(await this.page.evaluate(h=>{window.scrollTo(0,document.body.scrollHeight*h)},o/6),await this.page.waitForTimeout(600),await this.page.locator(H.item).count()>0)return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};return await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!1}),{hasPaa:!1}}async isSerpChallengeVisible(){if(!this.page)return!1;if(await this.page.locator(H.captchaMarker).count().catch(()=>0)>0)return!0;let n=this.page.url();if(/google\.[^/]+\/sorry\//i.test(n))return!0;let a=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,1e3)).catch(()=>"");return/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(a)}async updateSerpNavigationDebug(t,n,a){if(!(!t||!this.page))try{let s=this.page.url(),i=await this.page.title().catch(()=>""),d=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,500)).catch(()=>""),l=/recaptcha|unusual traffic|are you a robot/i.test(d);t.finalUrl=s,t.title=i,t.bodySnippet=d,t.hasPaa=a.hasPaa,t.captchaDetected=a.captchaDetected??l,t.googleSorryUrl=/google\.[^/]+\/sorry\//i.test(s),t.redirected=s!==n}catch(s){t.bodySnippet=`debug capture failed: ${ut(s)}`}}async captureDiagnostics(t){try{let n=this.page.url(),a=await this.page.title().catch(()=>""),s=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,400)).catch(()=>""),i=/consent\.google\./.test(n)||/before you continue/i.test(s),d=/recaptcha|unusual traffic|are you a robot/i.test(s),l=[i?"CONSENT_WALL":"",d?"BOT_CHALLENGE":"",n!==t?"REDIRECTED":""].filter(Boolean).join(",");return`intended=${t} | final=${n} | title="${a}" | flags=[${l}] | body="${s}"`}catch(n){return`diagnostics-failed: ${n.message}`}}captchaMessage(){return this.\u006b\u0065\u0072\u006e\u0065\u006cClient?"Google returned a CAPTCHA on this session \u2014 retrying with a fresh session.":Ce}async navigateTo(t){let n=process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim(),a=n?Le(t,n):t;try{if(await this.page.goto(a,{waitUntil:"domcontentloaded",timeout:45e3}),/google\.[^/]+\/search\?/i.test(t)&&await this.isSerpChallengeVisible())throw new E(this.captchaMessage())}catch(s){if(s instanceof E)throw s;let i=await this.captureDiagnostics(t);throw new vt(`page.goto failed: ${s.message} | ${i}`)}}async navigateToChannel(t){let n=Bn(t);try{await this.page.goto(n,{waitUntil:"networkidle",timeout:3e4})}catch(a){let s=await this.captureDiagnostics(n);throw new vt(`navigateToChannel failed: ${a.message} | ${s}`)}}async evaluate(t,n){return this.page.evaluate(t,n)}getPage(){return this.page}get\u004b\u0065\u0072\u006e\u0065\u006cSessionId(){return this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId}getProviderSessionMetadata(){return{provider:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId?"\u006b\u0065\u0072\u006e\u0065\u006c":"local",providerSessionId:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,disconnectObservation:null}}getDebugSnapshot(){return this.debugSnapshot}async close(){if(this.browser){let t=this.browser,n=this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,a=this.\u006b\u0065\u0072\u006e\u0065\u006cClient,s=this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs,i=this.\u006b\u0065\u0072\u006e\u0065\u006cStealth,d=this.\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed,l=this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource,U=this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType,u=this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent,S=this.getProviderSessionMetadata();if(this.browser=null,this.context=null,this.page=null,this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId=null,this.\u006b\u0065\u0072\u006e\u0065\u006cClient=null,this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=null,this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource=null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType=null,a&&n&&s!=null&&ve({\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n,source:"browser_driver",stealth:i,headlessSent:u,proxyUsed:d,proxySource:l,proxyType:U,openedAtMs:s,closedAtMs:Date.now()}),a&&n){console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_started",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n}));let o=ot(a.browsers.deleteByID(n),In,`\u004b\u0065\u0072\u006e\u0065\u006c session ${n} delete`),r=ot(t.close(),Wn,`\u004b\u0065\u0072\u006e\u0065\u006c browser ${n} close`),[h,c]=await Promise.allSettled([o,r]),C={\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!0,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:h.status==="fulfilled",\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:h.status==="rejected"?h.reason instanceof Error?h.reason.message:String(h.reason):null,browserCloseSucceeded:c.status==="fulfilled",browserCloseError:c.status==="rejected"?c.reason instanceof Error?c.reason.message:String(c.reason):null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null};return h.status==="rejected"?(console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_failed",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n,message:C.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError})),console.warn(`\u004b\u0065\u0072\u006e\u0065\u006c session cleanup failed for ${n}:`,h.reason)):console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_succeeded",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n})),c.status==="rejected"&&(console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_close_failed",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n,message:C.browserCloseError})),console.warn(`\u004b\u0065\u0072\u006e\u0065\u006c browser close failed for ${n}:`,c.reason)),C}return await t.close(),{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null}}else if(this.context){let t=this.context;return this.context=null,this.page=null,await t.close(),{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}return{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:null,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}};import{chromium as Rn}from"playwright";function Hn(e,t){if(typeof e!="string")return null;let n=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();return n?n.slice(0,t):null}function Oe(e){let t=Hn(e,320);return t?t.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization):\s*\S+/gi,"$1: [redacted]").replace(/\bBearer\s+\S+/gi,"Bearer [redacted]").slice(0,320):null}function xe(e){return Oe(e)}function Be(e){if(typeof e!="string"||e.length===0||e.length>512||e.trim()!==e||/[\s\u0000-\u001f\u007f]/.test(e)||/:\/\//.test(e))throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 did not return a valid browser session ID");return e}function zt(e,t,n=new Date().toISOString()){let a=Oe(t);return{source:e,message:a??e.replaceAll("_"," "),observedAt:n}}function Fn(e){return{provider:e.provider,providerDisconnectObserved:e.disconnectObservation!==null,providerDisconnectMessage:e.disconnectObservation?.message??null}}function He(e){return{...Fn(e),providerSessionId:e.providerSessionId}}var Gn=new Set(["image","media","font","stylesheet","ping"]),Dn=["doubleclick.net","googleadservices.com","googlesyndication.com","google-analytics.com"];function Fe(){return process.env.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_WSS?.trim()||null}function Re(){return process.env.SERP_BROWSER_PROVIDER?.trim().toLowerCase()==="\u006b\u0065\u0072\u006e\u0065\u006c"?!1:!!Fe()}async function qt(e,t=!1){let n=e.request();try{let a=await n.allHeaders(),s=await fetch(n.url(),{method:n.method(),headers:{accept:a.accept??"*/*","accept-language":a["accept-language"]??"en-US,en;q=0.9","user-agent":a["user-agent"]??"Mozilla/5.0",...t?{cookie:a.cookie??"",referer:a.referer??"https://www.google.com/",...a["x-client-data"]?{"x-client-data":a["x-client-data"]}:{}}:{}},body:n.method()==="GET"||n.method()==="HEAD"?void 0:n.postData()});if(!s.ok)return!1;let i=Buffer.from(await s.arrayBuffer());return await e.fulfill({status:s.status,contentType:s.headers.get("content-type")??"application/octet-stream",headers:{"cache-control":"public, max-age=3600"},body:i}),!0}catch{return!1}}var At=class{browser=null;context=null;page=null;cdpSession=null;providerSessionId=null;disconnectObservation=null;closing=!1;observeDisconnect(t,n){this.closing||this.disconnectObservation||(this.disconnectObservation=zt(t,n))}async launch(t){if(this.page)return;this.closing=!1,this.providerSessionId=null,this.disconnectObservation=null;let n=Fe();if(!n)throw new Error("SERP browser service is not configured");this.browser=await Rn.connectOverCDP(n),this.context=this.browser.contexts()[0]??await this.browser.newContext(),await this.context.addInitScript(()=>{let i=globalThis;typeof i.__name!="function"&&(i.__name=d=>d),typeof i.__publicField!="function"&&(i.__publicField=(d,l,U)=>(d[l]=U,U))}),this.page=this.context.pages()[0]??await this.context.newPage(),this.browser.on("disconnected",()=>this.observeDisconnect("browser_disconnected")),this.context.on("close",()=>this.observeDisconnect("context_closed")),this.page.on("close",()=>this.observeDisconnect("page_closed")),this.page.on("crash",()=>this.observeDisconnect("page_crashed")),this.cdpSession=await this.context.newCDPSession(this.page);let s=await this.cdpSession.send("Browser.getSessionId");this.providerSessionId=Be(s?.sessionId),await this.setupPage(this.page)}async setupPage(t){await t.route("**/*",async n=>{let a=n.request(),s=a.resourceType(),i;try{i=new URL(a.url())}catch{await n.continue();return}let d=i.hostname.toLowerCase();if(!(i.pathname==="/sgasync"&&await qt(n,!0))&&!(i.pathname.includes("/shared_dict/")&&await qt(n))&&!(s==="script"&&(d==="www.google.com"||d==="www.gstatic.com")&&await qt(n))){if(Gn.has(s)){await n.abort("blockedbyclient");return}if(Dn.some(l=>d===l||d.endsWith(`.${l}`))){await n.abort("blockedbyclient");return}await n.continue()}})}async withTemporaryPage(t,n){if(!this.context||this.closing)throw new Error("Browser context is not available");return ft(this.context,a=>this.setupPage(a),t,n)}async navigateToSERP(t,n,a,s,i){if(!this.page)throw new Error("Browser page is not available");let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});return i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n),await this.page.goto(`https://www.google.com/search?${d}`,{waitUntil:"load",timeout:45e3}),{hasPaa:!!await this.page.waitForSelector(H.item,{timeout:8e3}).catch(()=>null)}}async navigateTo(t){if(!this.page)throw new Error("Browser page is not available");await this.page.goto(t,{waitUntil:"load",timeout:45e3})}async navigateToChannel(){throw new Error("YouTube navigation is not supported by the SERP browser driver")}async evaluate(t,n){if(!this.page)throw new Error("Browser page is not available");return this.page.evaluate(t,n)}getPage(){if(!this.page)throw new Error("Browser page is not available");return this.page}get\u004b\u0065\u0072\u006e\u0065\u006cSessionId(){return null}getProviderSessionMetadata(){return{provider:"\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api",providerSessionId:this.providerSessionId,disconnectObservation:this.disconnectObservation}}getDebugSnapshot(){return{\u006b\u0065\u0072\u006e\u0065\u006c:null,context:null,networkLocation:null,serpNavigation:null}}async close(){this.closing=!0;let t=null;try{await this.browser?.close()}catch(a){t=xe(a instanceof Error?a.message:String(a))??"\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 browser close failed",this.disconnectObservation??=zt("browser_close_error",t)}let n=He(this.getProviderSessionMetadata());return this.browser=null,this.context=null,this.page=null,this.cdpSession=null,{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:t===null,browserCloseError:t,...n}}};function Ge(e,t){let n=!!(e?.serpIdentity||e?.\u006b\u0065\u0072\u006e\u0065\u006cProfileName);return t?.forceManaged===!0||Re()&&!n?new At:new It}var tt={austin:"Austin,Texas,United States","new york":"New York,New York,United States","new york city":"New York,New York,United States",nyc:"New York,New York,United States","los angeles":"Los Angeles,California,United States",la:"Los Angeles,California,United States",chicago:"Chicago,Illinois,United States",houston:"Houston,Texas,United States",phoenix:"Phoenix,Arizona,United States",philadelphia:"Philadelphia,Pennsylvania,United States",philly:"Philadelphia,Pennsylvania,United States","san antonio":"San Antonio,Texas,United States",dallas:"Dallas,Texas,United States",miami:"Miami,Florida,United States",seattle:"Seattle,Washington,United States",denver:"Denver,Colorado,United States",loveland:"Loveland,Colorado,United States","loveland co":"Loveland,Colorado,United States","fort collins":"Fort Collins,Colorado,United States",boulder:"Boulder,Colorado,United States","colorado springs":"Colorado Springs,Colorado,United States",boston:"Boston,Massachusetts,United States",atlanta:"Atlanta,Georgia,United States","san francisco":"San Francisco,California,United States",sf:"San Francisco,California,United States",portland:"Portland,Oregon,United States","las vegas":"Las Vegas,Nevada,United States",minneapolis:"Minneapolis,Minnesota,United States",detroit:"Detroit,Michigan,United States",nashville:"Nashville,Tennessee,United States",charlotte:"Charlotte,North Carolina,United States",orlando:"Orlando,Florida,United States","san diego":"San Diego,California,United States",baltimore:"Baltimore,Maryland,United States",sacramento:"Sacramento,California,United States",columbus:"Columbus,Ohio,United States",indianapolis:"Indianapolis,Indiana,United States","san jose":"San Jose,California,United States","fort worth":"Fort Worth,Texas,United States",jacksonville:"Jacksonville,Florida,United States",memphis:"Memphis,Tennessee,United States",louisville:"Louisville,Kentucky,United States",raleigh:"Raleigh,North Carolina,United States",richmond:"Richmond,Virginia,United States","salt lake city":"Salt Lake City,Utah,United States",toronto:"Toronto,Ontario,Canada",vancouver:"Vancouver,British Columbia,Canada",montreal:"Montreal,Quebec,Canada",calgary:"Calgary,Alberta,Canada",ottawa:"Ottawa,Ontario,Canada",london:"London,England,United Kingdom",manchester:"Manchester,England,United Kingdom",birmingham:"Birmingham,England,United Kingdom",edinburgh:"Edinburgh,Scotland,United Kingdom",glasgow:"Glasgow,Scotland,United Kingdom",leeds:"Leeds,England,United Kingdom",sydney:"Sydney,New South Wales,Australia",melbourne:"Melbourne,Victoria,Australia",brisbane:"Brisbane,Queensland,Australia",perth:"Perth,Western Australia,Australia",adelaide:"Adelaide,South Australia,Australia",dublin:"Dublin,Leinster,Ireland"};var De=`3, Nanjemoy,Maryland,United States
2
2
  Aaronsburg,Pennsylvania,United States
3
3
  Abbeville,Alabama,United States
4
4
  Abbeville,Georgia,United States
@@ -1,4 +1,4 @@
1
- import{b as O}from"./chunk-KJQXUZ4Y.js";import{s as N}from"./chunk-Z66Z2HV4.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
1
+ import{b as O}from"./chunk-KJQXUZ4Y.js";import{t as N}from"./chunk-7JAMFXNL.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
2
2
  CREATE TABLE IF NOT EXISTS location_data_imports (
3
3
  id TEXT PRIMARY KEY,
4
4
  dataset_kind TEXT NOT NULL,
@@ -1,4 +1,4 @@
1
- import{a as N}from"./chunk-YQZGZBB4.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,s as x}from"./chunk-Z66Z2HV4.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&amp;/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
1
+ import{a as N}from"./chunk-YQZGZBB4.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,t as x}from"./chunk-7JAMFXNL.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&amp;/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
2
2
  `).matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?([^"')&]+)(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?\s*\)/gi))n(l[1],`static-css-url${i}`,"image");for(let l of d.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi))n(l[1],`static-bare-url${i}`)},c=typeof e=="string"?P(e):e;return a(c),c.$("script").each((d,i)=>{let o=c.$(i).text(),h=o.replace(/\\u003[cC]/g,"<").replace(/\\u003[eE]/g,">").replace(/\\u0026/gi,"&").replace(/\\["']/g,l=>l.slice(1)).replace(/\\\//g,"/");h!==o&&/<(?:img|source|video|audio)\b/i.test(h)&&a(P(h),"-json-unescaped")}),[...r.values()]}function fe(e,t,r=100){let n=new Map;for(let a of e)A(n,a);for(let a of t?.assets??[])A(n,{...a,type:a.type});let g=new Map;for(let a of n.values()){let c=pe(a.url),d=g.get(c)??[];d.push(a),g.set(c,d)}return[...g.values()].map(a=>{let c=[...a].sort((i,o)=>S(o)-S(i)||i.url.localeCompare(o.url)),d={...c[0]};return d.discoveryMethods=[...new Set(a.flatMap(i=>i.discoveryMethods))],d.altTexts=[...new Set(a.flatMap(i=>i.altTexts))],d.contexts=[...new Set(a.flatMap(i=>i.contexts))],{...d,variants:c.slice(1).map(i=>i.url)}}).sort((a,c)=>S(c)-S(a)||a.url.localeCompare(c.url)).slice(0,Math.max(1,r))}async function ye(e,t,r,n={}){let g=Math.max(1,Math.min(n.maxBytes??52428800,104857600)),a=e,c=null;for(let p=0;p<=5;p++){let y=await L(a,{field:"media URL"});if(y.error||!y.parsed)throw new Error(y.error??"Media URL was rejected");if(c=await fetch(y.parsed.href,{signal:AbortSignal.timeout(15e3),redirect:"manual"}),c.status>=300&&c.status<400){let w=c.headers.get("location");if(!w)throw new Error(`HTTP ${c.status} redirect did not include Location`);a=new URL(w,y.parsed.href).href,c=null;continue}break}if(!c)throw new Error("Media download exceeded five redirects");if(!c.ok)throw new Error(`HTTP ${c.status}`);if(!c.body)throw new Error("Empty response body");let d=c.headers.get("content-type")?.split(";")[0].trim()??null;if(n.expectedType&&(!d||q(d)!==n.expectedType))throw new Error(`Expected ${n.expectedType} content but received ${d??"no content-type"}`);let i=Number(c.headers.get("content-length")??0);if(Number.isFinite(i)&&i>g)throw new Error(`Media asset exceeds ${g} byte limit`);let o=I(t,r);if(d&&!k(r)){let y={"image/jpeg":".jpg","image/png":".png","image/webp":".webp","image/gif":".gif","image/svg+xml":".svg","image/avif":".avif","video/mp4":".mp4","video/webm":".webm","audio/mpeg":".mp3","audio/ogg":".ogg","audio/wav":".wav"}[d];y&&(o=o+y)}let h=Z(o);await new Promise((p,y)=>{h.once("open",()=>p()),h.once("error",y)});let l=0,u=new ie({transform(p,y,w){let _=p.length;if(l+_>g){w(new Error(`Media asset exceeds ${g} byte limit`));return}if(n.consumeBytes&&!n.consumeBytes(_)){w(new Error("Media export total byte limit exceeded"));return}l+=_,w(null,p)}});try{await ne(oe.fromWeb(c.body),u,h)}catch(p){throw h.destroy(),te(o,{force:!0}),p}let{statSync:s}=await import("fs"),m=s(o).size;return{savedPath:o,sizeBytes:m,mimeType:d}}async function Le(e,t,r={}){let n=r.types??["image","video","audio"],g=new Set(n),a=he(e,t),c=Math.max(1,Math.min(r.maxAssets??100,250)),d=fe(a,r.rendered,2e3),i=d.slice(0,c),o=new Set([...a.map(f=>f.url),...(r.rendered?.assets??[]).map(f=>f.url)]).size,h=i.flatMap(f=>{let b=f.type??F(f.url);return b&&g.has(b)?[{...f,type:b}]:[]}),l=o-h.length,u=d.length>c,s=r.rendered?.exhausted===!0&&!u,m=u?"asset_limit":r.rendered?.stopReason??"render_unavailable",p=u?[`Media discovery found ${d.length} responsive families; only the requested ${c} records were retained.`]:r.rendered?r.rendered.exhausted?[]:[`Rendered media discovery stopped with ${r.rendered.stopReason}; the inventory may be incomplete.`]:["Rendered media discovery was unavailable; this manifest contains static-source evidence only."],y={pageUrl:t,staticFound:a.length,renderedFound:r.rendered?.assets.length??0,filteredCount:l,totalFound:o,completeness:s?"complete":"partial",exhausted:s,stopReason:m,scrollRounds:r.rendered?.scrollRounds??0,warnings:p,artifact:null};if(r.outputDir===null)return{...y,outputDir:null,assets:h.map((f,b)=>({...f,mimeType:null,filename:U(f.url,b),savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"not_attempted",downloadError:null,inlinePreview:null}))};let w=(()=>{try{return new URL(t).hostname.replace(/^www\./,"")}catch{return"unknown"}})(),_=new Date().toISOString().replace(/[:.]/g,"-").slice(0,19),C=r.outputDir??I(re(),"Downloads","mcp-scraper","media",`${_}-${w}`);ee(C,{recursive:!0});let T=[];return await Promise.allSettled(h.map(async(f,b)=>{let R=U(f.url,b);try{let{savedPath:v,sizeBytes:Q,mimeType:D}=await ye(f.url,C,R),G=D?q(D)??f.type:f.type;T.push({...f,type:G,mimeType:D,filename:B(v),savedPath:v,sizeBytes:Q,sha256:null,downloadStatus:"downloaded",downloadError:null,inlinePreview:null})}catch(v){T.push({...f,mimeType:null,filename:R,savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"failed",downloadError:v instanceof Error?v.message:"media_download_failed",inlinePreview:null})}})),T.sort((f,b)=>f.savedPath&&!b.savedPath?-1:!f.savedPath&&b.savedPath?1:f.url.localeCompare(b.url)),{...y,outputDir:C,assets:T}}var be=new Set(["browser_navigation_blocked","browser_navigation_failed","browser_timeout","browser_session_failed","browser_result_missing"]);function we(e){return e instanceof Error?e.code:void 0}function _e(e){return e instanceof Error?e.message:String(e)}function H(e){let t=e?.match(/^browser_http_(\d+)$/);return t?Number(t[1]):null}function ve(e){return e===404?"page_not_found":e===403?"page_forbidden":e===429?"page_rate_limited":e>=500&&e<=599?"page_server_error":"page_http_error"}function xe(e){let t=_e(e),r=we(e);if(N(e,t))return{errorCode:"vendor_unavailable",httpStatus:null,retryable:!0};let n=H(r)??H(t.match(/^(browser_http_\d+)/)?.[1]);return n!=null?{errorCode:ve(n),httpStatus:n,retryable:n===429||n>=500}:r==="browser_challenge_unresolved"||/^browser_challenge_unresolved/.test(t)?{errorCode:"bot_check_unresolved",httpStatus:null,retryable:!0}:r==="browser_response_too_large"||/exceeds \d+ byte limit/i.test(t)?{errorCode:"page_too_large",httpStatus:null,retryable:!1}:/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(t)?{errorCode:"browser_session_interrupted",httpStatus:null,retryable:!0}:be.has(r??"")||/timeout|navigation_failed|ERR_/i.test(t)?{errorCode:"page_unreachable",httpStatus:null,retryable:!0}:{errorCode:"extraction_failed",httpStatus:null,retryable:!1}}function j(e,t){switch(e){case"page_not_found":return`The page could not be found (HTTP ${t??404}). It may have been moved or deleted.`;case"page_forbidden":return`The site refused the request (HTTP ${t??403}).`;case"page_rate_limited":return"The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";case"page_server_error":return`The page's own server returned an error (HTTP ${t}). This is on the target site's side, not something a retry here can fix.`;case"page_http_error":return`The page returned an unexpected HTTP ${t} response.`;case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. A retry uses a fresh browser session.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"extraction_failed":return"The page could not be extracted. Please retry, or contact support if this persists for the same URL."}}function Ie(e,t={}){let r=j(e.errorCode,e.httpStatus);return{...$({errorCode:e.errorCode,retryable:e.retryable,chargeStatus:t.chargeStatus,details:{...t.details??{},...e.httpStatus!=null?{http_status:e.httpStatus}:{}}}),message:r,error:r}}function ke(e){return e.httpStatus===404?404:e.httpStatus===403?403:e.httpStatus===429?429:e.httpStatus!=null&&e.httpStatus>=500&&e.httpStatus<=599?500:e.httpStatus!=null?502:e.errorCode==="vendor_unavailable"?503:502}var Ee="wayback_capture_missing";function Be(e,t){if(e==null)return{failureCode:null,failureReason:null};if(e===Ee)return{failureCode:e,failureReason:t??null};let r=new Error(t??e);r.code=e;let n=xe(r);return{failureCode:n.errorCode,failureReason:j(n.errorCode,n.httpStatus)}}import{neon as Te}from"@neondatabase/serverless";var Se="jina",M=null,z=!1;function W(){return(process.env.JINA_EMBED_MODEL??"jina-embeddings-v5-omni-small").trim()}function Y(){return Number((process.env.JINA_EMBED_DIM??"1024").trim())}function J(){return!!(process.env.JINA_API_KEY?.trim()&&process.env.MEMORY_DATABASE_URL?.trim())}function E(){if(M)return M;let e=process.env.MEMORY_DATABASE_URL?.trim();if(!e)throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");return M=Te(e),M}async function K(){if(z)return;let e=Y();await E().query("CREATE EXTENSION IF NOT EXISTS vector"),await E().query(`
3
3
  CREATE TABLE IF NOT EXISTS commons_index_vectors (
4
4
  document_id TEXT PRIMARY KEY,
@@ -0,0 +1 @@
1
+ import{$ as M,$a as Ma,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Z as K,Za as Ka,_ as L,_a as La,aa as N,ab as Na,ba as O,bb as Oa,ca as P,cb as Pa,da as Q,db as Qa,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-7JAMFXNL.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,nb as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Bb as SiteAuditJobRowSchema,Cb as SiteAuditPhaseLogRowSchema,ta as advanceWorkflowRunStep,ra as appendWorkflowArtifacts,S as attachHarvestAttemptProviderSession,Ra as cancelJob,G as checkRateLimit,Ca as checkpointDurablePaaJob,Ba as checkpointJob,Aa as claimDurablePaaJob,tb as claimMonthlyFreeRefresh,za as claimPendingJob,Ta as claimPendingKpoJob,r as claimStripeEvent,Ea as completeDurablePaaJob,Ha as completeExpiredDurablePaaJob,Na as completeJob,Wa as completeKpoJob,Oa as completeRunningJob,s as completeStripeEvent,oa as completeWorkflowRunRecord,m as consumeCode,I as consumePasswordResetToken,ya as countActiveJobsForUser,xa as countActiveUsers,N as createJob,O as createJobWithId,Sa as createKpoJob,H as createPasswordResetToken,P as createRunningJob,F as createUser,ja as createWorkflowRun,ca as createWorkflowSchedule,qb as creditMc,rb as creditMcIdempotent,M as deactivateUser,ub as debitMc,vb as debitMcIdempotent,ga as deleteWorkflowSchedule,ob as expireOldLots,Fa as failDurablePaaJob,Ia as failExpiredDurablePaaJob,Pa as failJob,Xa as failKpoJob,Qa as failRunningJob,t as failStripeEvent,pa as failWorkflowRunRecord,T as finalizeExpiredHarvestAttempts,U as finishHarvestAttempt,v as generateApiKey,yb as getBillingLedgerSummary,k as getClient,fb as getConnectedAccountBillingState,e as getDb,Eb as getInboxMessage,Y as getJob,Ua as getKpoJob,Za as getKpoPhaseLog,xb as getLedger,jb as getMemoryPlan,kb as getMemoryProvisioned,Q as getOrCreateRunningJob,_ as getPageSnapshot,o as getRefresh,z as getUserByApiKey,y as getUserByEmail,A as getUserById,$a as getUserByStripeCustomerId,B as getUserMemoryCreds,E as getUserStats,va as getWorkflowArtifact,ka as getWorkflowRun,la as getWorkflowRunByIdempotencyKey,sa as getWorkflowRunStepState,da as getWorkflowSchedule,w as hashPassword,Hb as insertInboxMessage,d as inspectCurrentCoreSchema,sb as ledgerExistsForOperation,zb as ledgerExistsForStripePI,Ib as listActiveUserIds,ha as listDueWorkflowSchedules,X as listHarvestAttempts,W as listHarvestAttemptsPendingProviderTelemetry,Db as listInboxMessages,Z as listJobs,_a as listKpoJobs,Ja as listRecentTerminalDurablePaaJobs,La as listRecoverableDurablePaaJobs,ba as listRequestEvents,Ka as listStaleRunningDurablePaaJobs,L as listUsers,hb as listUsersDueForConnectedAccountBillingReconciliation,ua as listWorkflowArtifacts,ma as listWorkflowRuns,ea as listWorkflowSchedules,Ya as logKpoPhaseComplete,aa as logRequestEvent,Fb as markInboxMessageOpened,Ma as markJobBillingSettled,na as markWorkflowRunRunning,ia as markWorkflowScheduleRan,i as migrate,pb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,fa as patchWorkflowSchedule,l as putCode,n as putRefresh,Ab as reconcileBalanceMc,wa as recordWorkflowWebhookDelivery,j as registerClient,Da as renewDurablePaaLease,qa as replaceWorkflowArtifacts,Ga as requeueExpiredDurablePaaJob,mb as resolveUserByMemorySubscription,K as revokeApiKey,p as revokeRefresh,J as rotateApiKey,q as rotateRefresh,cb as setConcurrencyPackQuantity,db as setConcurrencySubId,gb as setConnectedAccountBillingState,f as setDbForTesting,bb as setExtraConcurrencySlots,ib as setMemoryPlan,lb as setMemoryProvisioned,D as setPassword,ab as setStripeCustomerId,eb as setSubscriptionTier,C as setUserMemoryCreds,wb as settleDebitMcIdempotent,Gb as softDeleteInboxMessage,R as startHarvestAttempt,u as stripeEventAlreadyProcessed,V as updateHarvestAttemptProviderTelemetry,Va as updateKpoJobState,$ as upsertPageSnapshot,x as verifyPassword,h as verifySecuredMainOAuthSchema};
@@ -1,4 +1,4 @@
1
- import{c as Je,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-K7KMLT33.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-VEJQ4XPI.js";import"./chunk-YQZGZBB4.js";import"./chunk-QPWPR5XG.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-MZN4U5BL.js";import"./chunk-HE45FFBU.js";import{s as qe}from"./chunk-Z66Z2HV4.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
1
+ import{c as Je,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-WYCXAFID.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-JJDYKM5I.js";import"./chunk-YQZGZBB4.js";import"./chunk-QPWPR5XG.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-MZN4U5BL.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-7JAMFXNL.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
2
2
 
3
3
  `).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
4
4
 
@@ -1 +1 @@
1
- import{A as d,B as e,C as f,D as g,E as h,x as a,y as b,z as c}from"./chunk-2O3UYISB.js";import"./chunk-Z27SYWYD.js";import"./chunk-HUV2WTRW.js";import"./chunk-MZN4U5BL.js";import"./chunk-D4NOJAUN.js";import"./chunk-Z66Z2HV4.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};
1
+ import{A as d,B as e,C as f,D as g,E as h,x as a,y as b,z as c}from"./chunk-2ICI4TIX.js";import"./chunk-NGQYSQQA.js";import"./chunk-HUV2WTRW.js";import"./chunk-MZN4U5BL.js";import"./chunk-53HIX3X7.js";import"./chunk-7JAMFXNL.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- import{u as z}from"./chunk-J7EVLE6H.js";import"./chunk-QPWPR5XG.js";import"./chunk-4FROKQJN.js";import{h as M}from"./chunk-D4NOJAUN.js";import"./chunk-3GP5CYZX.js";import"./chunk-Z66Z2HV4.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
1
+ import{u as z}from"./chunk-SEHBXTMX.js";import"./chunk-QPWPR5XG.js";import"./chunk-4FROKQJN.js";import{h as M}from"./chunk-53HIX3X7.js";import"./chunk-3GP5CYZX.js";import"./chunk-7JAMFXNL.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
2
2
 
3
3
  IMPORTANT TECHNICAL CONTEXT: Clip 2 will be generated using image-to-video, meaning it will start from the exact last frame of Clip 1. The two clips will be visually seamless \u2014 same location, same characters, continuous motion. You must write prompts that make this feel like one uninterrupted 16-second video.
4
4
 
@@ -1 +1 @@
1
- import{A,B,C,D,E,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z}from"./chunk-EN66HBAW.js";import"./chunk-Z66Z2HV4.js";export{a as LEAD_LIST_MAX_DISPATCH_ATTEMPTS,c as LEAD_LIST_RUNNING_STALE_MINUTES,b as LEAD_LIST_START_DEADLINE_MINUTES,s as cancelPendingLeadListEnrichmentRows,C as claimLeadListEnrichmentOutbox,n as claimLeadListEnrichmentRow,u as completeLeadListEnrichmentJob,j as createOrGetLeadListEnrichmentJob,g as ensureLeadListEnrichmentRepositorySchema,z as failExpiredLeadListEnrichmentStart,v as failLeadListEnrichmentJob,B as failStaleRunningLeadListEnrichmentJob,h as getLeadListEnrichmentJob,i as getLeadListEnrichmentJobByIdempotencyKey,o as getLeadListEnrichmentRow,e as leadListEnrichmentFingerprint,f as leadListRowInputDigest,y as listExpiredLeadListEnrichmentStarts,x as listLeadListEnrichmentJobsNeedingSettlement,p as listLeadListEnrichmentRows,A as listStaleRunningLeadListEnrichmentJobs,l as markLeadListEnrichmentJobQueued,m as markLeadListEnrichmentJobRunning,D as markLeadListEnrichmentOutboxDispatched,E as markLeadListEnrichmentOutboxFailed,w as recordLeadListEnrichmentSettlement,r as requestLeadListEnrichmentCancellation,t as saveLeadListEnrichmentDeliverable,q as saveLeadListEnrichmentRowResult,k as setInitializingLeadListEnrichmentConcurrencyLock,d as stableCanonicalJson};
1
+ import{A,B,C,D,E,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z}from"./chunk-RZSJC7FX.js";import"./chunk-7JAMFXNL.js";export{a as LEAD_LIST_MAX_DISPATCH_ATTEMPTS,c as LEAD_LIST_RUNNING_STALE_MINUTES,b as LEAD_LIST_START_DEADLINE_MINUTES,s as cancelPendingLeadListEnrichmentRows,C as claimLeadListEnrichmentOutbox,n as claimLeadListEnrichmentRow,u as completeLeadListEnrichmentJob,j as createOrGetLeadListEnrichmentJob,g as ensureLeadListEnrichmentRepositorySchema,z as failExpiredLeadListEnrichmentStart,v as failLeadListEnrichmentJob,B as failStaleRunningLeadListEnrichmentJob,h as getLeadListEnrichmentJob,i as getLeadListEnrichmentJobByIdempotencyKey,o as getLeadListEnrichmentRow,e as leadListEnrichmentFingerprint,f as leadListRowInputDigest,y as listExpiredLeadListEnrichmentStarts,x as listLeadListEnrichmentJobsNeedingSettlement,p as listLeadListEnrichmentRows,A as listStaleRunningLeadListEnrichmentJobs,l as markLeadListEnrichmentJobQueued,m as markLeadListEnrichmentJobRunning,D as markLeadListEnrichmentOutboxDispatched,E as markLeadListEnrichmentOutboxFailed,w as recordLeadListEnrichmentSettlement,r as requestLeadListEnrichmentCancellation,t as saveLeadListEnrichmentDeliverable,q as saveLeadListEnrichmentRowResult,k as setInitializingLeadListEnrichmentConcurrencyLock,d as stableCanonicalJson};
@@ -1 +1 @@
1
- import{a,b,c,d,e,f,g,h,i,j,k,l,m,n}from"./chunk-DMCYIWB3.js";import"./chunk-KJQXUZ4Y.js";import"./chunk-Z66Z2HV4.js";export{b as HOSTED_CENSUS_DATASET_PREFIX,a as HOSTED_ZIP_DATASET_KIND,c as MIN_NATIONWIDE_ZIP_COUNT,d as REQUIRED_LOCATION_STATE_CODES,g as ensureHostedLocationDataSchema,h as getActiveHostedDataset,i as getActiveHostedLocationDataset,k as getHostedZipGroups,m as importHostedCensusPlacesCsv,l as importHostedZipGroupsCsv,j as listActiveHostedLocationDatasets,e as parseHostedCensusPlacesCsv,f as parseHostedZipGroupsCsv,n as queryHostedLocationMarkets};
1
+ import{a,b,c,d,e,f,g,h,i,j,k,l,m,n}from"./chunk-UB5N2ICD.js";import"./chunk-KJQXUZ4Y.js";import"./chunk-7JAMFXNL.js";export{b as HOSTED_CENSUS_DATASET_PREFIX,a as HOSTED_ZIP_DATASET_KIND,c as MIN_NATIONWIDE_ZIP_COUNT,d as REQUIRED_LOCATION_STATE_CODES,g as ensureHostedLocationDataSchema,h as getActiveHostedDataset,i as getActiveHostedLocationDataset,k as getHostedZipGroups,m as importHostedCensusPlacesCsv,l as importHostedZipGroupsCsv,j as listActiveHostedLocationDatasets,e as parseHostedCensusPlacesCsv,f as parseHostedZipGroupsCsv,n as queryHostedLocationMarkets};