mcp-scraper 0.89.1 → 0.89.2

package/dist/{chunk-6QO4F2KC.js → chunk-J7EVLE6H.js}

@@ -1,4 +1,4 @@
-import{f as q}from"./chunk-QPWPR5XG.js";import{a as ye}from"./chunk-4FROKQJN.js";import{a as ke,b as pe,c as Nt,d as me,g as ve}from"./chunk-RYNHNEF3.js";import{a as ce,c as we,d as Ut,g as ge,l as Ne}from"./chunk-3GP5CYZX.js";import{a as Ce,i as Me,j as E,k as vt,m as _,n as ht,o as Kt}from"./chunk-63T4W2LX.js";import{chromium as _t}from"playwright-extra";import fn from"puppeteer-extra-plugin-stealth";import{chromium as Pn}from"playwright";var H={container:".eJH8qe.adDDi",dataInitq:"[data-initq]",item:".related-question-pair",itemDataQ:"data-q",itemDataInitQ:"data-initq",itemQuestionEl:".JlqpRe",answerContainer:".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",aiAnswerBody:".pWvJNd",aiCitation:"a.PMDqCb[href]",aiSourceCard:"a.vIWmYe[href], a.Zbfntb[href]",sourceTitle:"h3",sourceSite:".VuuXrf",sourceCite:"cite",clickTarget:".JlqpRe",expandedClass:"aoRk1c",captchaMarker:'#captcha-form, #recaptcha, form[action*="/sorry/"], .g-recaptcha, [data-sitekey]'},Te={container:'div[jscontroller="HWk0Gf"]',sectionHeading:'.mgAbYb[role="heading"]',item:"a.rIRoqf"},Tt={udm:"39",item:"a.rIRoqf",durationPattern:/^\d+:\d+$/,platforms:["YouTube","TikTok","Instagram","Facebook","X"]},fe={section:".ULSxyf",item:"a.KYg7td.INpicf",title:".hyYc0c",source:".K4ETW"},Pe={sectionTag:"g-section-with-header",sectionHeadingText:"What people are saying",card:'.dRzkFf[role="listitem"]',cardLink:'a.WlydOe[jsname="YKoRaf"]',titleH1:"h1.WQWxe",titleDiv:".eAaXgc",popularCommentLabel:".qgdis",source:".sTl1Td",platformBadge:".appd0, .KrMNbf",ytChannel:".sjVJQd",ytDate:".PLq9Je",authorNote:".nDgy9d"},be={root:"[data-lhcontainer][data-streaming-container][eid]",legacyRoot:'[data-hveid="CBMQAA"]',wrapper:".Fgyi2e",controller:'[jscontroller="AkrxPe"]',contentSubtree:'[data-subtree="mfc"]',header:".heWuVc",heading:".Fzsovc.cwYVJe.RJPOee",showMoreButton:'[aria-label="Show more AI Overview"]',sourcesPanel:".OZ9ddf.WAUd4",disclaimer:".DuQANe.MSJHRb"},We={root:'[data-hveid="CAUQAA"]',wrapper:".Fgyi2e",citations:'.Fgyi2e [data-hveid] a[jsname="pxBnId"]'},Ie={result:".wHYlTd.tF2Cxc",title:"h3.LC20lb",siteName:".VuuXrf",cite:"cite.tjvcx",snippet:".VwiC3b",redditCite:"cite.qLRx3b",ratingWrap:".Y0A0hc",ratingValue:".yi40Hd",reviewCount:".RDApEe"},Ae={headingText:"Businesses",card:".w7Dbne",name:".OSrXXb",ratingValue:".yi40Hd",reviewCount:".RDApEe"};var _a={ratingAndCount:"div.F7nice",hoursTable:"table.eK4R0e",reviewScrollPane:'div.m6QErb[tabindex="-1"]',reviewScrollPaneFallback:'[role="main"] div[tabindex="-1"]',reviewCardAuthor:"div.d4r55, span.d4r55, span.RPZfBb",reviewCardDate:"span.rsqaWe",reviewCardText:"span.wiI7pd",reviewCardOwnerBlock:"div.CDe7pd",hoursTableAlt:'table[aria-label*="Hour"]',reviewCard:"[data-review-id]",reviewStars:'[role="img"][aria-label*="star"]',reviewTab:'button[role="tab"][aria-label*="Review"]',aboutTab:'button[role="tab"][aria-label*="About"]',expandReview:'[data-review-id] button[aria-label*="See more"], [data-review-id] button.w8nwRe'};async function ft(e,t,n,a){let s,i=!1,d,l=()=>{},U=()=>a?.reason instanceof DOMException&&a.reason.name==="TimeoutError"?a.reason:new _,u=()=>l(U()),S=new Promise((r,h)=>{l=h}),o=async r=>{let h;try{await Promise.race([r.close().catch(()=>{}),new Promise(c=>{h=setTimeout(c,2e3)})])}finally{h&&clearTimeout(h)}};if(a?.aborted)throw U();a?.addEventListener("abort",u,{once:!0}),d=setTimeout(()=>l(new DOMException("Optional pagination timed out","TimeoutError")),3e4);try{return await Promise.race([(async()=>{let r=await e.newPage();if(i)throw await o(r),new _;if(s=r,await t(r),i)throw new _;return n(r)})(),S])}finally{i=!0,d&&clearTimeout(d),a?.removeEventListener("abort",u),s&&await o(s)}}var Tn=/\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;async function Pt(e){try{let t=await e.evaluate(()=>navigator.userAgent);return typeof t=="string"&&Tn.test(t)?t:null}catch{return null}}async function bt(e){let t=null;try{let n=e.contexts().flatMap(a=>a.pages())[0];return n||(t=await e.newContext(),n=t.pages()[0]??await t.newPage()),await Pt(n)}catch{return null}finally{t&&await t.close().catch(()=>{})}}function ja(e,t){return t?{...e,userAgent:t}:e}_t.use(fn());var bn=180,Wn=3e3,In=5e3;function An(e,t){let n=process.env[e];if(!n)return t;let a=Number(n);return Number.isInteger(a)&&a>0?a:t}function Jt(e){return e?e.slice(-6):null}function Le(e,t){let n=(t??process.env.SERP_REDIRECT_BASE)?.trim().replace(/\/+$/,"");return n?`${n}/g/${Buffer.from(e,"utf8").toString("base64url")}`:e}function Ln(e){return e?/united states|u\.?s\.?a?\.?|^us$/i.test(e.trim()):!1}function ut(e){return e instanceof Error?e.message:String(e)}function On(e){return/\b409\b|conflict|already exists/i.test(ut(e))}async function xn(e,t){try{await e.profiles.create({name:t})}catch(n){if(!On(n))throw n}}function Wt(e,t){return{viewport:e.viewport,locale:e.locale,...t?{userAgent:t}:{},...e.deviceScaleFactor?{deviceScaleFactor:e.deviceScaleFactor}:{},...e.isMobile!==void 0?{isMobile:e.isMobile}:{},...e.hasTouch!==void 0?{hasTouch:e.hasTouch}:{}}}async function ot(e,t,n){let a;try{return await Promise.race([e,new Promise((s,i)=>{a=setTimeout(()=>i(new Error(`${n} timed out after ${t}ms`)),t)})])}finally{a&&clearTimeout(a)}}function Bn(e){let t=e.trim();if(!t)throw new Error("channelHandle is required");if(/^https?:\/\//i.test(t)||/^(www\.|m\.)?youtube\.com\//i.test(t)){let d=new URL(/^https?:\/\//i.test(t)?t:`https://${t}`);if(d.hostname.replace(/^www\./,"").replace(/^m\./,"").toLowerCase()!=="youtube.com")throw new Error("channel URL must be on youtube.com");let U=d.pathname.split("/").filter(Boolean),u=U[0]??"",S=U[1]??"";if(u.startsWith("@"))return`https://www.youtube.com/${u}/videos`;if(u==="channel"&&S)return`https://www.youtube.com/channel/${S}/videos`;if((u==="c"||u==="user")&&S)return`https://www.youtube.com/${u}/${S}/videos`;throw new Error("channel URL must be a YouTube handle, /channel/UC..., /c/..., or /user/... URL")}let s=t.replace(/^\/+/,"").replace(/\/+$/,"").replace(/\/videos$/i,"");if(/^UC[\w-]{20,}$/.test(s))return`https://www.youtube.com/channel/${s}/videos`;let i=s.startsWith("@")?s:`@${s}`;if(!/^@[\w.-]+$/.test(i))throw new Error("channelHandle must be an @handle, UC channel ID, or YouTube channel URL");return`https://www.youtube.com/${i}/videos`}var It=class{browser=null;context=null;page=null;kernelClient=null;kernelSessionId=null;kernelOpenedAtMs=null;kernelStealth=null;kernelProxyUsed=null;kernelProxySource=null;kernelProxyType=null;kernelHeadlessSent=null;debugEnabled=!1;debugSnapshot={kernel:null,context:null,networkLocation:null,serpNavigation:null};async launch(t){this.debugEnabled=t.debug===!0;let n=t.proxyMode??(t.kernelProxyId?"configured":"none"),a=t.isMobile?"mobile":"desktop";if(this.debugSnapshot={kernel:null,context:{viewport:t.viewport,locale:t.locale,device:a,userAgent:null,deviceScaleFactor:t.deviceScaleFactor??null,isMobile:t.isMobile===!0,hasTouch:t.hasTouch===!0},networkLocation:null,serpNavigation:null},t.kernelApiKey){this.kernelClient=new q({apiKey:t.kernelApiKey});let i=Number.isInteger(t.kernelTimeoutSeconds)&&Number(t.kernelTimeoutSeconds)>0?Number(t.kernelTimeoutSeconds):An("KERNEL_BROWSER_TIMEOUT_SECONDS",bn);t.kernelProfileName&&t.kernelProfileSaveChanges===!0&&await xn(this.kernelClient,t.kernelProfileName);let d=Nt(),l;t.headlessMode==="headful"||d?.forceHeadful===!0?l=!1:t.headlessMode==="headless"?l=!0:l=d?.forceHeadless===!0||process.env.KERNEL_HEADLESS_DISABLED!=="true"&&!!d?.op&&me.has(d.op),this.kernelHeadlessSent=l?!0:null,d?.headlessSentOut&&(d.headlessSentOut.value=this.kernelHeadlessSent);let U=await this.kernelClient.browsers.create({stealth:t.kernelStealth===!0,timeout_seconds:i,...l?{headless:!0}:{},...t.kernelProxyId?{proxy_id:t.kernelProxyId}:{},...t.kernelProfileName?{profile:{name:t.kernelProfileName,...typeof t.kernelProfileSaveChanges=="boolean"?{save_changes:t.kernelProfileSaveChanges}:{}}}:{}});this.kernelSessionId=U.session_id,this.kernelOpenedAtMs=Date.now(),this.kernelStealth=typeof U.stealth=="boolean"?U.stealth:null,this.kernelProxyUsed=typeof U.proxy_id=="string"?!!U.proxy_id:!!t.kernelProxyId,this.kernelProxySource=t.kernelProxyResolution?.source??null,this.kernelProxyType=t.kernelProxyResolution?.proxyType??null;let u=null,S=null;if(!t.kernelProxyId&&!t.keepDefaultProxy)try{await ot(this.kernelClient.browsers.update(this.kernelSessionId,{disable_default_proxy:!0}),5e3,`Kernel session ${this.kernelSessionId} disable default proxy`),u=!0}catch(r){u=!1,S=ut(r)}let o={sessionId:this.kernelSessionId,proxyMode:n,requestedProxyIdPresent:!!t.kernelProxyId,requestedProxyIdSuffix:Jt(t.kernelProxyId),createdProxyIdPresent:typeof U.proxy_id=="string"?!!U.proxy_id:null,createdProxyIdSuffix:Jt(U.proxy_id),retrievedProxyIdPresent:null,retrievedProxyIdSuffix:null,retrievedProxyIdMatchesRequested:null,defaultProxyDisabled:u,defaultProxyDisableError:S,proxyResolution:t.kernelProxyResolution??null,timeoutSeconds:i,stealth:typeof U.stealth=="boolean"?U.stealth:null,profilePresent:null,poolPresent:null,retrieveError:null};if(this.debugSnapshot.kernel=o,console.info(JSON.stringify({event:"kernel_browser_created",kernel_session_id:this.kernelSessionId,timeout_seconds:i,proxy_mode:n,proxy_id_present:!!t.kernelProxyId,profile_name_present:!!t.kernelProfileName,proxy_resolution_source:t.kernelProxyResolution?.source})),this.debugEnabled&&await this.populateKernelRetrieveDebug(o,t.kernelProxyId),this.browser=await Pn.connectOverCDP(U.cdp_ws_url),t.kernelProfileName||t.preserveKernelContext)this.context=this.browser.contexts()[0]??await this.browser.newContext(Wt(t)),this.page=this.context.pages()[0]??await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{let r=await bt(this.browser);this.debugSnapshot.context.userAgent=r,this.context=await this.browser.newContext(Wt(t,r)),this.page=await this.context.newPage()}if(await this.installEsbuildHelperShims(this.context),t.preserveKernelContext||await this.page.setViewportSize(t.viewport),(this.debugEnabled||t.requireUsEgress)&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation()),t.requireUsEgress){let r=this.debugSnapshot.networkLocation?.country??null;if(!Ln(r))throw new E(`Non-US egress (${r??"unknown"}) \u2014 retrying with a fresh session`)}return}let s={headless:t.headless,proxy:t.proxy?{server:t.proxy}:void 0,...t.browserExecutablePath?{executablePath:t.browserExecutablePath}:{},...t.ignoreDefaultArgs?{ignoreDefaultArgs:t.ignoreDefaultArgs}:{}};if(t.profileDir)this.context=await _t.launchPersistentContext(t.profileDir,{...s,...Wt(t)}),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{this.browser=await _t.launch(s);let i=await bt(this.browser);this.debugSnapshot.context.userAgent=i,this.context=await this.browser.newContext(Wt(t,i)),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage()}this.debugEnabled&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation())}async populateKernelRetrieveDebug(t,n){if(!(!this.kernelClient||!this.kernelSessionId))try{let a=await ot(this.kernelClient.browsers.retrieve(this.kernelSessionId),5e3,`Kernel session ${this.kernelSessionId} retrieve`);t.retrievedProxyIdPresent=typeof a.proxy_id=="string"?!!a.proxy_id:!1,t.retrievedProxyIdSuffix=Jt(a.proxy_id),t.retrievedProxyIdMatchesRequested=n?a.proxy_id===n:!a.proxy_id,t.timeoutSeconds=typeof a.timeout_seconds=="number"?a.timeout_seconds:t.timeoutSeconds,t.stealth=typeof a.stealth=="boolean"?a.stealth:t.stealth,t.profilePresent=!!a.profile,t.poolPresent=!!a.pool}catch(a){t.retrieveError=ut(a)}}async captureBrowserNetworkLocation(){let t=(a,s="ipapi.co")=>({source:s,ip:null,city:null,region:null,country:null,org:null,timezone:null,error:a});if(!this.context)return t("browser context is not available");let n=null;try{n=await this.context.newPage();let a=await this.loadJsonInDebugPage(n,"https://ipwho.is/");if(a){let l=typeof a.connection=="object"&&a.connection!==null?a.connection:{};return{source:"ipwho.is",ip:typeof a.ip=="string"?a.ip:null,city:typeof a.city=="string"?a.city:null,region:typeof a.region=="string"?a.region:null,country:typeof a.country=="string"?a.country:null,org:typeof l.org=="string"?l.org:null,timezone:typeof a.timezone=="object"&&a.timezone!==null&&typeof a.timezone.id=="string"?a.timezone.id:null,error:null}}let s=await this.loadJsonInDebugPage(n,"https://api64.ipify.org?format=json");if(s)return{source:"api64.ipify.org",ip:typeof s.ip=="string"?s.ip:null,city:null,region:null,country:null,org:null,timezone:null,error:null};await ot(n.goto("https://ipapi.co/json/",{waitUntil:"domcontentloaded",timeout:7e3}),8e3,"browser network location navigation");let i=await n.locator("body").innerText({timeout:2e3}),d=JSON.parse(i);return{source:"ipapi.co",ip:typeof d.ip=="string"?d.ip:null,city:typeof d.city=="string"?d.city:null,region:typeof d.region=="string"?d.region:null,country:typeof d.country_name=="string"?d.country_name:typeof d.country=="string"?d.country:null,org:typeof d.org=="string"?d.org:null,timezone:typeof d.timezone=="string"?d.timezone:null,error:null}}catch(a){return t(ut(a))}finally{await n?.close().catch(()=>{})}}async loadJsonInDebugPage(t,n){try{await ot(t.goto(n,{waitUntil:"domcontentloaded",timeout:7e3}),8e3,`browser network location navigation ${n}`);let a=await t.locator("body").innerText({timeout:2e3});return JSON.parse(a)}catch{return null}}async installEsbuildHelperShims(t){await t.addInitScript(()=>{let n=globalThis;typeof n.__name!="function"&&(n.__name=a=>a),typeof n.__publicField!="function"&&(n.__publicField=(a,s,i)=>(a[s]=i,i))})}async withTemporaryPage(t,n){if(!this.context)throw new Error("Browser context is not available");return ft(this.context,async()=>{},t,n)}async navigateToSERP(t,n,a,s,i){let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n);let l="https://www.google.com/search?"+d.toString(),U=Le(l),u=i?.debug?{requestedUrl:l,finalUrl:null,title:null,bodySnippet:null,hasPaa:null,captchaDetected:null,googleSorryUrl:null,redirected:null}:null;u&&(this.debugSnapshot.serpNavigation=u);try{await this.page.goto(U,{waitUntil:"domcontentloaded",timeout:45e3})}catch(o){await this.updateSerpNavigationDebug(u,l,{hasPaa:null,captchaDetected:null});let r=await this.captureDiagnostics(l);throw new vt(`page.goto failed: ${o.message} | ${r}`)}if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());if(await this.page.waitForSelector(H.item,{timeout:4e3}).catch(()=>null))return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());for(let o=1;o<=6;o++)if(await this.page.evaluate(h=>{window.scrollTo(0,document.body.scrollHeight*h)},o/6),await this.page.waitForTimeout(600),await this.page.locator(H.item).count()>0)return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};return await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!1}),{hasPaa:!1}}async isSerpChallengeVisible(){if(!this.page)return!1;if(await this.page.locator(H.captchaMarker).count().catch(()=>0)>0)return!0;let n=this.page.url();if(/google\.[^/]+\/sorry\//i.test(n))return!0;let a=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,1e3)).catch(()=>"");return/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(a)}async updateSerpNavigationDebug(t,n,a){if(!(!t||!this.page))try{let s=this.page.url(),i=await this.page.title().catch(()=>""),d=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,500)).catch(()=>""),l=/recaptcha|unusual traffic|are you a robot/i.test(d);t.finalUrl=s,t.title=i,t.bodySnippet=d,t.hasPaa=a.hasPaa,t.captchaDetected=a.captchaDetected??l,t.googleSorryUrl=/google\.[^/]+\/sorry\//i.test(s),t.redirected=s!==n}catch(s){t.bodySnippet=`debug capture failed: ${ut(s)}`}}async captureDiagnostics(t){try{let n=this.page.url(),a=await this.page.title().catch(()=>""),s=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,400)).catch(()=>""),i=/consent\.google\./.test(n)||/before you continue/i.test(s),d=/recaptcha|unusual traffic|are you a robot/i.test(s),l=[i?"CONSENT_WALL":"",d?"BOT_CHALLENGE":"",n!==t?"REDIRECTED":""].filter(Boolean).join(",");return`intended=${t} | final=${n} | title="${a}" | flags=[${l}] | body="${s}"`}catch(n){return`diagnostics-failed: ${n.message}`}}captchaMessage(){return this.kernelClient?"Google returned a CAPTCHA on this session \u2014 retrying with a fresh session.":Ce}async navigateTo(t){let n=process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim(),a=n?Le(t,n):t;try{if(await this.page.goto(a,{waitUntil:"domcontentloaded",timeout:45e3}),/google\.[^/]+\/search\?/i.test(t)&&await this.isSerpChallengeVisible())throw new E(this.captchaMessage())}catch(s){if(s instanceof E)throw s;let i=await this.captureDiagnostics(t);throw new vt(`page.goto failed: ${s.message} | ${i}`)}}async navigateToChannel(t){let n=Bn(t);try{await this.page.goto(n,{waitUntil:"networkidle",timeout:3e4})}catch(a){let s=await this.captureDiagnostics(n);throw new vt(`navigateToChannel failed: ${a.message} | ${s}`)}}async evaluate(t,n){return this.page.evaluate(t,n)}getPage(){return this.page}getKernelSessionId(){return this.kernelSessionId}getProviderSessionMetadata(){return{provider:this.kernelSessionId?"kernel":"local",providerSessionId:this.kernelSessionId,disconnectObservation:null}}getDebugSnapshot(){return this.debugSnapshot}async close(){if(this.browser){let t=this.browser,n=this.kernelSessionId,a=this.kernelClient,s=this.kernelOpenedAtMs,i=this.kernelStealth,d=this.kernelProxyUsed,l=this.kernelProxySource,U=this.kernelProxyType,u=this.kernelHeadlessSent,S=this.getProviderSessionMetadata();if(this.browser=null,this.context=null,this.page=null,this.kernelSessionId=null,this.kernelClient=null,this.kernelOpenedAtMs=null,this.kernelHeadlessSent=null,this.kernelProxySource=null,this.kernelProxyType=null,a&&n&&s!=null&&ve({kernelSessionId:n,source:"browser_driver",stealth:i,headlessSent:u,proxyUsed:d,proxySource:l,proxyType:U,openedAtMs:s,closedAtMs:Date.now()}),a&&n){console.info(JSON.stringify({event:"kernel_browser_delete_started",kernel_session_id:n}));let o=ot(a.browsers.deleteByID(n),In,`Kernel session ${n} delete`),r=ot(t.close(),Wn,`Kernel browser ${n} close`),[h,c]=await Promise.allSettled([o,r]),C={kernelSessionId:n,kernelDeleteStarted:!0,kernelDeleteSucceeded:h.status==="fulfilled",kernelDeleteError:h.status==="rejected"?h.reason instanceof Error?h.reason.message:String(h.reason):null,browserCloseSucceeded:c.status==="fulfilled",browserCloseError:c.status==="rejected"?c.reason instanceof Error?c.reason.message:String(c.reason):null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null};return h.status==="rejected"?(console.warn(JSON.stringify({event:"kernel_browser_delete_failed",kernel_session_id:n,message:C.kernelDeleteError})),console.warn(`Kernel session cleanup failed for ${n}:`,h.reason)):console.info(JSON.stringify({event:"kernel_browser_delete_succeeded",kernel_session_id:n})),c.status==="rejected"&&(console.warn(JSON.stringify({event:"kernel_browser_close_failed",kernel_session_id:n,message:C.browserCloseError})),console.warn(`Kernel browser close failed for ${n}:`,c.reason)),C}return await t.close(),{kernelSessionId:null,kernelDeleteStarted:!1,kernelDeleteSucceeded:null,kernelDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null}}else if(this.context){let t=this.context;return this.context=null,this.page=null,await t.close(),{kernelSessionId:null,kernelDeleteStarted:!1,kernelDeleteSucceeded:null,kernelDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}return{kernelSessionId:null,kernelDeleteStarted:!1,kernelDeleteSucceeded:null,kernelDeleteError:null,browserCloseSucceeded:null,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}};import{chromium as Rn}from"playwright";function Hn(e,t){if(typeof e!="string")return null;let n=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();return n?n.slice(0,t):null}function Oe(e){let t=Hn(e,320);return t?t.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization):\s*\S+/gi,"$1: [redacted]").replace(/\bBearer\s+\S+/gi,"Bearer [redacted]").slice(0,320):null}function xe(e){return Oe(e)}function Be(e){if(typeof e!="string"||e.length===0||e.length>512||e.trim()!==e||/[\s\u0000-\u001f\u007f]/.test(e)||/:\/\//.test(e))throw new Error("Bright Data did not return a valid browser session ID");return e}function zt(e,t,n=new Date().toISOString()){let a=Oe(t);return{source:e,message:a??e.replaceAll("_"," "),observedAt:n}}function Fn(e){return{provider:e.provider,providerDisconnectObserved:e.disconnectObservation!==null,providerDisconnectMessage:e.disconnectObservation?.message??null}}function He(e){return{...Fn(e),providerSessionId:e.providerSessionId}}var Gn=new Set(["image","media","font","stylesheet","ping"]),Dn=["doubleclick.net","googleadservices.com","googlesyndication.com","google-analytics.com"];function Fe(){return process.env.BRIGHTDATA_BROWSER_WSS?.trim()||null}function Re(){return process.env.SERP_BROWSER_PROVIDER?.trim().toLowerCase()==="kernel"?!1:!!Fe()}async function qt(e,t=!1){let n=e.request();try{let a=await n.allHeaders(),s=await fetch(n.url(),{method:n.method(),headers:{accept:a.accept??"*/*","accept-language":a["accept-language"]??"en-US,en;q=0.9","user-agent":a["user-agent"]??"Mozilla/5.0",...t?{cookie:a.cookie??"",referer:a.referer??"https://www.google.com/",...a["x-client-data"]?{"x-client-data":a["x-client-data"]}:{}}:{}},body:n.method()==="GET"||n.method()==="HEAD"?void 0:n.postData()});if(!s.ok)return!1;let i=Buffer.from(await s.arrayBuffer());return await e.fulfill({status:s.status,contentType:s.headers.get("content-type")??"application/octet-stream",headers:{"cache-control":"public, max-age=3600"},body:i}),!0}catch{return!1}}var At=class{browser=null;context=null;page=null;cdpSession=null;providerSessionId=null;disconnectObservation=null;closing=!1;observeDisconnect(t,n){this.closing||this.disconnectObservation||(this.disconnectObservation=zt(t,n))}async launch(t){if(this.page)return;this.closing=!1,this.providerSessionId=null,this.disconnectObservation=null;let n=Fe();if(!n)throw new Error("SERP browser service is not configured");this.browser=await Rn.connectOverCDP(n),this.context=this.browser.contexts()[0]??await this.browser.newContext(),await this.context.addInitScript(()=>{let i=globalThis;typeof i.__name!="function"&&(i.__name=d=>d),typeof i.__publicField!="function"&&(i.__publicField=(d,l,U)=>(d[l]=U,U))}),this.page=this.context.pages()[0]??await this.context.newPage(),this.browser.on("disconnected",()=>this.observeDisconnect("browser_disconnected")),this.context.on("close",()=>this.observeDisconnect("context_closed")),this.page.on("close",()=>this.observeDisconnect("page_closed")),this.page.on("crash",()=>this.observeDisconnect("page_crashed")),this.cdpSession=await this.context.newCDPSession(this.page);let s=await this.cdpSession.send("Browser.getSessionId");this.providerSessionId=Be(s?.sessionId),await this.setupPage(this.page)}async setupPage(t){await t.route("**/*",async n=>{let a=n.request(),s=a.resourceType(),i;try{i=new URL(a.url())}catch{await n.continue();return}let d=i.hostname.toLowerCase();if(!(i.pathname==="/sgasync"&&await qt(n,!0))&&!(i.pathname.includes("/shared_dict/")&&await qt(n))&&!(s==="script"&&(d==="www.google.com"||d==="www.gstatic.com")&&await qt(n))){if(Gn.has(s)){await n.abort("blockedbyclient");return}if(Dn.some(l=>d===l||d.endsWith(`.${l}`))){await n.abort("blockedbyclient");return}await n.continue()}})}async withTemporaryPage(t,n){if(!this.context||this.closing)throw new Error("Browser context is not available");return ft(this.context,a=>this.setupPage(a),t,n)}async navigateToSERP(t,n,a,s,i){if(!this.page)throw new Error("Browser page is not available");let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});return i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n),await this.page.goto(`https://www.google.com/search?${d}`,{waitUntil:"load",timeout:45e3}),{hasPaa:!!await this.page.waitForSelector(H.item,{timeout:8e3}).catch(()=>null)}}async navigateTo(t){if(!this.page)throw new Error("Browser page is not available");await this.page.goto(t,{waitUntil:"load",timeout:45e3})}async navigateToChannel(){throw new Error("YouTube navigation is not supported by the SERP browser driver")}async evaluate(t,n){if(!this.page)throw new Error("Browser page is not available");return this.page.evaluate(t,n)}getPage(){if(!this.page)throw new Error("Browser page is not available");return this.page}getKernelSessionId(){return null}getProviderSessionMetadata(){return{provider:"bright_data_browser_api",providerSessionId:this.providerSessionId,disconnectObservation:this.disconnectObservation}}getDebugSnapshot(){return{kernel:null,context:null,networkLocation:null,serpNavigation:null}}async close(){this.closing=!0;let t=null;try{await this.browser?.close()}catch(a){t=xe(a instanceof Error?a.message:String(a))??"Bright Data browser close failed",this.disconnectObservation??=zt("browser_close_error",t)}let n=He(this.getProviderSessionMetadata());return this.browser=null,this.context=null,this.page=null,this.cdpSession=null,{kernelSessionId:null,kernelDeleteStarted:!1,kernelDeleteSucceeded:null,kernelDeleteError:null,browserCloseSucceeded:t===null,browserCloseError:t,...n}}};function Ge(e,t){let n=!!(e?.serpIdentity||e?.kernelProfileName);return t?.forceManaged===!0||Re()&&!n?new At:new It}var tt={austin:"Austin,Texas,United States","new york":"New York,New York,United States","new york city":"New York,New York,United States",nyc:"New York,New York,United States","los angeles":"Los Angeles,California,United States",la:"Los Angeles,California,United States",chicago:"Chicago,Illinois,United States",houston:"Houston,Texas,United States",phoenix:"Phoenix,Arizona,United States",philadelphia:"Philadelphia,Pennsylvania,United States",philly:"Philadelphia,Pennsylvania,United States","san antonio":"San Antonio,Texas,United States",dallas:"Dallas,Texas,United States",miami:"Miami,Florida,United States",seattle:"Seattle,Washington,United States",denver:"Denver,Colorado,United States",loveland:"Loveland,Colorado,United States","loveland co":"Loveland,Colorado,United States","fort collins":"Fort Collins,Colorado,United States",boulder:"Boulder,Colorado,United States","colorado springs":"Colorado Springs,Colorado,United States",boston:"Boston,Massachusetts,United States",atlanta:"Atlanta,Georgia,United States","san francisco":"San Francisco,California,United States",sf:"San Francisco,California,United States",portland:"Portland,Oregon,United States","las vegas":"Las Vegas,Nevada,United States",minneapolis:"Minneapolis,Minnesota,United States",detroit:"Detroit,Michigan,United States",nashville:"Nashville,Tennessee,United States",charlotte:"Charlotte,North Carolina,United States",orlando:"Orlando,Florida,United States","san diego":"San Diego,California,United States",baltimore:"Baltimore,Maryland,United States",sacramento:"Sacramento,California,United States",columbus:"Columbus,Ohio,United States",indianapolis:"Indianapolis,Indiana,United States","san jose":"San Jose,California,United States","fort worth":"Fort Worth,Texas,United States",jacksonville:"Jacksonville,Florida,United States",memphis:"Memphis,Tennessee,United States",louisville:"Louisville,Kentucky,United States",raleigh:"Raleigh,North Carolina,United States",richmond:"Richmond,Virginia,United States","salt lake city":"Salt Lake City,Utah,United States",toronto:"Toronto,Ontario,Canada",vancouver:"Vancouver,British Columbia,Canada",montreal:"Montreal,Quebec,Canada",calgary:"Calgary,Alberta,Canada",ottawa:"Ottawa,Ontario,Canada",london:"London,England,United Kingdom",manchester:"Manchester,England,United Kingdom",birmingham:"Birmingham,England,United Kingdom",edinburgh:"Edinburgh,Scotland,United Kingdom",glasgow:"Glasgow,Scotland,United Kingdom",leeds:"Leeds,England,United Kingdom",sydney:"Sydney,New South Wales,Australia",melbourne:"Melbourne,Victoria,Australia",brisbane:"Brisbane,Queensland,Australia",perth:"Perth,Western Australia,Australia",adelaide:"Adelaide,South Australia,Australia",dublin:"Dublin,Leinster,Ireland"};var De=`3, Nanjemoy,Maryland,United States
+import{f as q}from"./chunk-QPWPR5XG.js";import{a as ye}from"./chunk-4FROKQJN.js";import{a as ke,b as pe,c as Nt,d as me,g as ve}from"./chunk-D4NOJAUN.js";import{a as ce,c as we,d as Ut,g as ge,l as Ne}from"./chunk-3GP5CYZX.js";import{a as Ce,i as Me,j as E,k as vt,m as _,n as ht,o as Kt}from"./chunk-Z66Z2HV4.js";import{chromium as _t}from"playwright-extra";import fn from"puppeteer-extra-plugin-stealth";import{chromium as Pn}from"playwright";var H={container:".eJH8qe.adDDi",dataInitq:"[data-initq]",item:".related-question-pair",itemDataQ:"data-q",itemDataInitQ:"data-initq",itemQuestionEl:".JlqpRe",answerContainer:".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",aiAnswerBody:".pWvJNd",aiCitation:"a.PMDqCb[href]",aiSourceCard:"a.vIWmYe[href], a.Zbfntb[href]",sourceTitle:"h3",sourceSite:".VuuXrf",sourceCite:"cite",clickTarget:".JlqpRe",expandedClass:"aoRk1c",captchaMarker:'#captcha-form, #recaptcha, form[action*="/sorry/"], .g-recaptcha, [data-sitekey]'},Te={container:'div[jscontroller="HWk0Gf"]',sectionHeading:'.mgAbYb[role="heading"]',item:"a.rIRoqf"},Tt={udm:"39",item:"a.rIRoqf",durationPattern:/^\d+:\d+$/,platforms:["YouTube","TikTok","Instagram","Facebook","X"]},fe={section:".ULSxyf",item:"a.KYg7td.INpicf",title:".hyYc0c",source:".K4ETW"},Pe={sectionTag:"g-section-with-header",sectionHeadingText:"What people are saying",card:'.dRzkFf[role="listitem"]',cardLink:'a.WlydOe[jsname="YKoRaf"]',titleH1:"h1.WQWxe",titleDiv:".eAaXgc",popularCommentLabel:".qgdis",source:".sTl1Td",platformBadge:".appd0, .KrMNbf",ytChannel:".sjVJQd",ytDate:".PLq9Je",authorNote:".nDgy9d"},be={root:"[data-lhcontainer][data-streaming-container][eid]",legacyRoot:'[data-hveid="CBMQAA"]',wrapper:".Fgyi2e",controller:'[jscontroller="AkrxPe"]',contentSubtree:'[data-subtree="mfc"]',header:".heWuVc",heading:".Fzsovc.cwYVJe.RJPOee",showMoreButton:'[aria-label="Show more AI Overview"]',sourcesPanel:".OZ9ddf.WAUd4",disclaimer:".DuQANe.MSJHRb"},We={root:'[data-hveid="CAUQAA"]',wrapper:".Fgyi2e",citations:'.Fgyi2e [data-hveid] a[jsname="pxBnId"]'},Ie={result:".wHYlTd.tF2Cxc",title:"h3.LC20lb",siteName:".VuuXrf",cite:"cite.tjvcx",snippet:".VwiC3b",redditCite:"cite.qLRx3b",ratingWrap:".Y0A0hc",ratingValue:".yi40Hd",reviewCount:".RDApEe"},Ae={headingText:"Businesses",card:".w7Dbne",name:".OSrXXb",ratingValue:".yi40Hd",reviewCount:".RDApEe"};var _a={ratingAndCount:"div.F7nice",hoursTable:"table.eK4R0e",reviewScrollPane:'div.m6QErb[tabindex="-1"]',reviewScrollPaneFallback:'[role="main"] div[tabindex="-1"]',reviewCardAuthor:"div.d4r55, span.d4r55, span.RPZfBb",reviewCardDate:"span.rsqaWe",reviewCardText:"span.wiI7pd",reviewCardOwnerBlock:"div.CDe7pd",hoursTableAlt:'table[aria-label*="Hour"]',reviewCard:"[data-review-id]",reviewStars:'[role="img"][aria-label*="star"]',reviewTab:'button[role="tab"][aria-label*="Review"]',aboutTab:'button[role="tab"][aria-label*="About"]',expandReview:'[data-review-id] button[aria-label*="See more"], [data-review-id] button.w8nwRe'};async function ft(e,t,n,a){let s,i=!1,d,l=()=>{},U=()=>a?.reason instanceof DOMException&&a.reason.name==="TimeoutError"?a.reason:new _,u=()=>l(U()),S=new Promise((r,h)=>{l=h}),o=async r=>{let h;try{await Promise.race([r.close().catch(()=>{}),new Promise(c=>{h=setTimeout(c,2e3)})])}finally{h&&clearTimeout(h)}};if(a?.aborted)throw U();a?.addEventListener("abort",u,{once:!0}),d=setTimeout(()=>l(new DOMException("Optional pagination timed out","TimeoutError")),3e4);try{return await Promise.race([(async()=>{let r=await e.newPage();if(i)throw await o(r),new _;if(s=r,await t(r),i)throw new _;return n(r)})(),S])}finally{i=!0,d&&clearTimeout(d),a?.removeEventListener("abort",u),s&&await o(s)}}var Tn=/\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;async function Pt(e){try{let t=await e.evaluate(()=>navigator.userAgent);return typeof t=="string"&&Tn.test(t)?t:null}catch{return null}}async function bt(e){let t=null;try{let n=e.contexts().flatMap(a=>a.pages())[0];return n||(t=await e.newContext(),n=t.pages()[0]??await t.newPage()),await Pt(n)}catch{return null}finally{t&&await t.close().catch(()=>{})}}function ja(e,t){return t?{...e,userAgent:t}:e}_t.use(fn());var bn=180,Wn=3e3,In=5e3;function An(e,t){let n=process.env[e];if(!n)return t;let a=Number(n);return Number.isInteger(a)&&a>0?a:t}function Jt(e){return e?e.slice(-6):null}function Le(e,t){let n=(t??process.env.SERP_REDIRECT_BASE)?.trim().replace(/\/+$/,"");return n?`${n}/g/${Buffer.from(e,"utf8").toString("base64url")}`:e}function Ln(e){return e?/united states|u\.?s\.?a?\.?|^us$/i.test(e.trim()):!1}function ut(e){return e instanceof Error?e.message:String(e)}function On(e){return/\b409\b|conflict|already exists/i.test(ut(e))}async function xn(e,t){try{await e.profiles.create({name:t})}catch(n){if(!On(n))throw n}}function Wt(e,t){return{viewport:e.viewport,locale:e.locale,...t?{userAgent:t}:{},...e.deviceScaleFactor?{deviceScaleFactor:e.deviceScaleFactor}:{},...e.isMobile!==void 0?{isMobile:e.isMobile}:{},...e.hasTouch!==void 0?{hasTouch:e.hasTouch}:{}}}async function ot(e,t,n){let a;try{return await Promise.race([e,new Promise((s,i)=>{a=setTimeout(()=>i(new Error(`${n} timed out after ${t}ms`)),t)})])}finally{a&&clearTimeout(a)}}function Bn(e){let t=e.trim();if(!t)throw new Error("channelHandle is required");if(/^https?:\/\//i.test(t)||/^(www\.|m\.)?youtube\.com\//i.test(t)){let d=new URL(/^https?:\/\//i.test(t)?t:`https://${t}`);if(d.hostname.replace(/^www\./,"").replace(/^m\./,"").toLowerCase()!=="youtube.com")throw new Error("channel URL must be on youtube.com");let U=d.pathname.split("/").filter(Boolean),u=U[0]??"",S=U[1]??"";if(u.startsWith("@"))return`https://www.youtube.com/${u}/videos`;if(u==="channel"&&S)return`https://www.youtube.com/channel/${S}/videos`;if((u==="c"||u==="user")&&S)return`https://www.youtube.com/${u}/${S}/videos`;throw new Error("channel URL must be a YouTube handle, /channel/UC..., /c/..., or /user/... URL")}let s=t.replace(/^\/+/,"").replace(/\/+$/,"").replace(/\/videos$/i,"");if(/^UC[\w-]{20,}$/.test(s))return`https://www.youtube.com/channel/${s}/videos`;let i=s.startsWith("@")?s:`@${s}`;if(!/^@[\w.-]+$/.test(i))throw new Error("channelHandle must be an @handle, UC channel ID, or YouTube channel URL");return`https://www.youtube.com/${i}/videos`}var It=class{browser=null;context=null;page=null;\u006b\u0065\u0072\u006e\u0065\u006cClient=null;\u006b\u0065\u0072\u006e\u0065\u006cSessionId=null;\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=null;\u006b\u0065\u0072\u006e\u0065\u006cStealth=null;\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed=null;\u006b\u0065\u0072\u006e\u0065\u006cProxySource=null;\u006b\u0065\u0072\u006e\u0065\u006cProxyType=null;\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=null;debugEnabled=!1;debugSnapshot={\u006b\u0065\u0072\u006e\u0065\u006c:null,context:null,networkLocation:null,serpNavigation:null};async launch(t){this.debugEnabled=t.debug===!0;let n=t.proxyMode??(t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId?"configured":"none"),a=t.isMobile?"mobile":"desktop";if(this.debugSnapshot={\u006b\u0065\u0072\u006e\u0065\u006c:null,context:{viewport:t.viewport,locale:t.locale,device:a,userAgent:null,deviceScaleFactor:t.deviceScaleFactor??null,isMobile:t.isMobile===!0,hasTouch:t.hasTouch===!0},networkLocation:null,serpNavigation:null},t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey){this.\u006b\u0065\u0072\u006e\u0065\u006cClient=new q({apiKey:t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey});let i=Number.isInteger(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds)&&Number(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds)>0?Number(t.\u006b\u0065\u0072\u006e\u0065\u006cTimeoutSeconds):An("\u004b\u0045\u0052\u004e\u0045\u004c_BROWSER_TIMEOUT_SECONDS",bn);t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName&&t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges===!0&&await xn(this.\u006b\u0065\u0072\u006e\u0065\u006cClient,t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName);let d=Nt(),l;t.headlessMode==="headful"||d?.forceHeadful===!0?l=!1:t.headlessMode==="headless"?l=!0:l=d?.forceHeadless===!0||process.env.\u004b\u0045\u0052\u004e\u0045\u004c_HEADLESS_DISABLED!=="true"&&!!d?.op&&me.has(d.op),this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=l?!0:null,d?.headlessSentOut&&(d.headlessSentOut.value=this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent);let U=await this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.create({stealth:t.\u006b\u0065\u0072\u006e\u0065\u006cStealth===!0,timeout_seconds:i,...l?{headless:!0}:{},...t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId?{proxy_id:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId}:{},...t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName?{profile:{name:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,...typeof t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges=="boolean"?{save_changes:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges}:{}}}:{}});this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId=U.session_id,this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=Date.now(),this.\u006b\u0065\u0072\u006e\u0065\u006cStealth=typeof U.stealth=="boolean"?U.stealth:null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed=typeof U.proxy_id=="string"?!!U.proxy_id:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource=t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.source??null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType=t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.proxyType??null;let u=null,S=null;if(!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId&&!t.keepDefaultProxy)try{await ot(this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.update(this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,{disable_default_proxy:!0}),5e3,`\u004b\u0065\u0072\u006e\u0065\u006c session ${this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId} disable default proxy`),u=!0}catch(r){u=!1,S=ut(r)}let o={sessionId:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,proxyMode:n,requestedProxyIdPresent:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,requestedProxyIdSuffix:Jt(t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId),createdProxyIdPresent:typeof U.proxy_id=="string"?!!U.proxy_id:null,createdProxyIdSuffix:Jt(U.proxy_id),retrievedProxyIdPresent:null,retrievedProxyIdSuffix:null,retrievedProxyIdMatchesRequested:null,defaultProxyDisabled:u,defaultProxyDisableError:S,proxyResolution:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution??null,timeoutSeconds:i,stealth:typeof U.stealth=="boolean"?U.stealth:null,profilePresent:null,poolPresent:null,retrieveError:null};if(this.debugSnapshot.\u006b\u0065\u0072\u006e\u0065\u006c=o,console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_created",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,timeout_seconds:i,proxy_mode:n,proxy_id_present:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,profile_name_present:!!t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,proxy_resolution_source:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution?.source})),this.debugEnabled&&await this.populate\u004b\u0065\u0072\u006e\u0065\u006cRetrieveDebug(o,t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId),this.browser=await Pn.connectOverCDP(U.cdp_ws_url),t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName||t.preserve\u004b\u0065\u0072\u006e\u0065\u006cContext)this.context=this.browser.contexts()[0]??await this.browser.newContext(Wt(t)),this.page=this.context.pages()[0]??await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{let r=await bt(this.browser);this.debugSnapshot.context.userAgent=r,this.context=await this.browser.newContext(Wt(t,r)),this.page=await this.context.newPage()}if(await this.installEsbuildHelperShims(this.context),t.preserve\u004b\u0065\u0072\u006e\u0065\u006cContext||await this.page.setViewportSize(t.viewport),(this.debugEnabled||t.requireUsEgress)&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation()),t.requireUsEgress){let r=this.debugSnapshot.networkLocation?.country??null;if(!Ln(r))throw new E(`Non-US egress (${r??"unknown"}) \u2014 retrying with a fresh session`)}return}let s={headless:t.headless,proxy:t.proxy?{server:t.proxy}:void 0,...t.browserExecutablePath?{executablePath:t.browserExecutablePath}:{},...t.ignoreDefaultArgs?{ignoreDefaultArgs:t.ignoreDefaultArgs}:{}};if(t.profileDir)this.context=await _t.launchPersistentContext(t.profileDir,{...s,...Wt(t)}),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage(),this.debugSnapshot.context.userAgent=await Pt(this.page);else{this.browser=await _t.launch(s);let i=await bt(this.browser);this.debugSnapshot.context.userAgent=i,this.context=await this.browser.newContext(Wt(t,i)),await this.installEsbuildHelperShims(this.context),this.page=await this.context.newPage()}this.debugEnabled&&(this.debugSnapshot.networkLocation=await this.captureBrowserNetworkLocation())}async populate\u004b\u0065\u0072\u006e\u0065\u006cRetrieveDebug(t,n){if(!(!this.\u006b\u0065\u0072\u006e\u0065\u006cClient||!this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId))try{let a=await ot(this.\u006b\u0065\u0072\u006e\u0065\u006cClient.browsers.retrieve(this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId),5e3,`\u004b\u0065\u0072\u006e\u0065\u006c session ${this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId} retrieve`);t.retrievedProxyIdPresent=typeof a.proxy_id=="string"?!!a.proxy_id:!1,t.retrievedProxyIdSuffix=Jt(a.proxy_id),t.retrievedProxyIdMatchesRequested=n?a.proxy_id===n:!a.proxy_id,t.timeoutSeconds=typeof a.timeout_seconds=="number"?a.timeout_seconds:t.timeoutSeconds,t.stealth=typeof a.stealth=="boolean"?a.stealth:t.stealth,t.profilePresent=!!a.profile,t.poolPresent=!!a.pool}catch(a){t.retrieveError=ut(a)}}async captureBrowserNetworkLocation(){let t=(a,s="ipapi.co")=>({source:s,ip:null,city:null,region:null,country:null,org:null,timezone:null,error:a});if(!this.context)return t("browser context is not available");let n=null;try{n=await this.context.newPage();let a=await this.loadJsonInDebugPage(n,"https://ipwho.is/");if(a){let l=typeof a.connection=="object"&&a.connection!==null?a.connection:{};return{source:"ipwho.is",ip:typeof a.ip=="string"?a.ip:null,city:typeof a.city=="string"?a.city:null,region:typeof a.region=="string"?a.region:null,country:typeof a.country=="string"?a.country:null,org:typeof l.org=="string"?l.org:null,timezone:typeof a.timezone=="object"&&a.timezone!==null&&typeof a.timezone.id=="string"?a.timezone.id:null,error:null}}let s=await this.loadJsonInDebugPage(n,"https://api64.ipify.org?format=json");if(s)return{source:"api64.ipify.org",ip:typeof s.ip=="string"?s.ip:null,city:null,region:null,country:null,org:null,timezone:null,error:null};await ot(n.goto("https://ipapi.co/json/",{waitUntil:"domcontentloaded",timeout:7e3}),8e3,"browser network location navigation");let i=await n.locator("body").innerText({timeout:2e3}),d=JSON.parse(i);return{source:"ipapi.co",ip:typeof d.ip=="string"?d.ip:null,city:typeof d.city=="string"?d.city:null,region:typeof d.region=="string"?d.region:null,country:typeof d.country_name=="string"?d.country_name:typeof d.country=="string"?d.country:null,org:typeof d.org=="string"?d.org:null,timezone:typeof d.timezone=="string"?d.timezone:null,error:null}}catch(a){return t(ut(a))}finally{await n?.close().catch(()=>{})}}async loadJsonInDebugPage(t,n){try{await ot(t.goto(n,{waitUntil:"domcontentloaded",timeout:7e3}),8e3,`browser network location navigation ${n}`);let a=await t.locator("body").innerText({timeout:2e3});return JSON.parse(a)}catch{return null}}async installEsbuildHelperShims(t){await t.addInitScript(()=>{let n=globalThis;typeof n.__name!="function"&&(n.__name=a=>a),typeof n.__publicField!="function"&&(n.__publicField=(a,s,i)=>(a[s]=i,i))})}async withTemporaryPage(t,n){if(!this.context)throw new Error("Browser context is not available");return ft(this.context,async()=>{},t,n)}async navigateToSERP(t,n,a,s,i){let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n);let l="https://www.google.com/search?"+d.toString(),U=Le(l),u=i?.debug?{requestedUrl:l,finalUrl:null,title:null,bodySnippet:null,hasPaa:null,captchaDetected:null,googleSorryUrl:null,redirected:null}:null;u&&(this.debugSnapshot.serpNavigation=u);try{await this.page.goto(U,{waitUntil:"domcontentloaded",timeout:45e3})}catch(o){await this.updateSerpNavigationDebug(u,l,{hasPaa:null,captchaDetected:null});let r=await this.captureDiagnostics(l);throw new vt(`page.goto failed: ${o.message} | ${r}`)}if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());if(await this.page.waitForSelector(H.item,{timeout:4e3}).catch(()=>null))return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};if(await this.isSerpChallengeVisible())throw await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!0}),new E(this.captchaMessage());for(let o=1;o<=6;o++)if(await this.page.evaluate(h=>{window.scrollTo(0,document.body.scrollHeight*h)},o/6),await this.page.waitForTimeout(600),await this.page.locator(H.item).count()>0)return await this.updateSerpNavigationDebug(u,l,{hasPaa:!0,captchaDetected:!1}),{hasPaa:!0};return await this.updateSerpNavigationDebug(u,l,{hasPaa:!1,captchaDetected:!1}),{hasPaa:!1}}async isSerpChallengeVisible(){if(!this.page)return!1;if(await this.page.locator(H.captchaMarker).count().catch(()=>0)>0)return!0;let n=this.page.url();if(/google\.[^/]+\/sorry\//i.test(n))return!0;let a=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,1e3)).catch(()=>"");return/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(a)}async updateSerpNavigationDebug(t,n,a){if(!(!t||!this.page))try{let s=this.page.url(),i=await this.page.title().catch(()=>""),d=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,500)).catch(()=>""),l=/recaptcha|unusual traffic|are you a robot/i.test(d);t.finalUrl=s,t.title=i,t.bodySnippet=d,t.hasPaa=a.hasPaa,t.captchaDetected=a.captchaDetected??l,t.googleSorryUrl=/google\.[^/]+\/sorry\//i.test(s),t.redirected=s!==n}catch(s){t.bodySnippet=`debug capture failed: ${ut(s)}`}}async captureDiagnostics(t){try{let n=this.page.url(),a=await this.page.title().catch(()=>""),s=await this.page.evaluate(()=>(document.body?.innerText??"").replace(/\s+/g," ").trim().slice(0,400)).catch(()=>""),i=/consent\.google\./.test(n)||/before you continue/i.test(s),d=/recaptcha|unusual traffic|are you a robot/i.test(s),l=[i?"CONSENT_WALL":"",d?"BOT_CHALLENGE":"",n!==t?"REDIRECTED":""].filter(Boolean).join(",");return`intended=${t} | final=${n} | title="${a}" | flags=[${l}] | body="${s}"`}catch(n){return`diagnostics-failed: ${n.message}`}}captchaMessage(){return this.\u006b\u0065\u0072\u006e\u0065\u006cClient?"Google returned a CAPTCHA on this session \u2014 retrying with a fresh session.":Ce}async navigateTo(t){let n=process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim(),a=n?Le(t,n):t;try{if(await this.page.goto(a,{waitUntil:"domcontentloaded",timeout:45e3}),/google\.[^/]+\/search\?/i.test(t)&&await this.isSerpChallengeVisible())throw new E(this.captchaMessage())}catch(s){if(s instanceof E)throw s;let i=await this.captureDiagnostics(t);throw new vt(`page.goto failed: ${s.message} | ${i}`)}}async navigateToChannel(t){let n=Bn(t);try{await this.page.goto(n,{waitUntil:"networkidle",timeout:3e4})}catch(a){let s=await this.captureDiagnostics(n);throw new vt(`navigateToChannel failed: ${a.message} | ${s}`)}}async evaluate(t,n){return this.page.evaluate(t,n)}getPage(){return this.page}get\u004b\u0065\u0072\u006e\u0065\u006cSessionId(){return this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId}getProviderSessionMetadata(){return{provider:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId?"\u006b\u0065\u0072\u006e\u0065\u006c":"local",providerSessionId:this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,disconnectObservation:null}}getDebugSnapshot(){return this.debugSnapshot}async close(){if(this.browser){let t=this.browser,n=this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,a=this.\u006b\u0065\u0072\u006e\u0065\u006cClient,s=this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs,i=this.\u006b\u0065\u0072\u006e\u0065\u006cStealth,d=this.\u006b\u0065\u0072\u006e\u0065\u006cProxyUsed,l=this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource,U=this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType,u=this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent,S=this.getProviderSessionMetadata();if(this.browser=null,this.context=null,this.page=null,this.\u006b\u0065\u0072\u006e\u0065\u006cSessionId=null,this.\u006b\u0065\u0072\u006e\u0065\u006cClient=null,this.\u006b\u0065\u0072\u006e\u0065\u006cOpenedAtMs=null,this.\u006b\u0065\u0072\u006e\u0065\u006cHeadlessSent=null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxySource=null,this.\u006b\u0065\u0072\u006e\u0065\u006cProxyType=null,a&&n&&s!=null&&ve({\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n,source:"browser_driver",stealth:i,headlessSent:u,proxyUsed:d,proxySource:l,proxyType:U,openedAtMs:s,closedAtMs:Date.now()}),a&&n){console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_started",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n}));let o=ot(a.browsers.deleteByID(n),In,`\u004b\u0065\u0072\u006e\u0065\u006c session ${n} delete`),r=ot(t.close(),Wn,`\u004b\u0065\u0072\u006e\u0065\u006c browser ${n} close`),[h,c]=await Promise.allSettled([o,r]),C={\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!0,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:h.status==="fulfilled",\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:h.status==="rejected"?h.reason instanceof Error?h.reason.message:String(h.reason):null,browserCloseSucceeded:c.status==="fulfilled",browserCloseError:c.status==="rejected"?c.reason instanceof Error?c.reason.message:String(c.reason):null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null};return h.status==="rejected"?(console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_failed",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n,message:C.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError})),console.warn(`\u004b\u0065\u0072\u006e\u0065\u006c session cleanup failed for ${n}:`,h.reason)):console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_delete_succeeded",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n})),c.status==="rejected"&&(console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_browser_close_failed",\u006b\u0065\u0072\u006e\u0065\u006c_session_id:n,message:C.browserCloseError})),console.warn(`\u004b\u0065\u0072\u006e\u0065\u006c browser close failed for ${n}:`,c.reason)),C}return await t.close(),{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:S.provider,providerSessionId:S.providerSessionId,providerDisconnectObserved:!1,providerDisconnectMessage:null}}else if(this.context){let t=this.context;return this.context=null,this.page=null,await t.close(),{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:!0,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}return{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:null,browserCloseError:null,provider:"local",providerSessionId:null,providerDisconnectObserved:!1,providerDisconnectMessage:null}}};import{chromium as Rn}from"playwright";function Hn(e,t){if(typeof e!="string")return null;let n=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();return n?n.slice(0,t):null}function Oe(e){let t=Hn(e,320);return t?t.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization):\s*\S+/gi,"$1: [redacted]").replace(/\bBearer\s+\S+/gi,"Bearer [redacted]").slice(0,320):null}function xe(e){return Oe(e)}function Be(e){if(typeof e!="string"||e.length===0||e.length>512||e.trim()!==e||/[\s\u0000-\u001f\u007f]/.test(e)||/:\/\//.test(e))throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 did not return a valid browser session ID");return e}function zt(e,t,n=new Date().toISOString()){let a=Oe(t);return{source:e,message:a??e.replaceAll("_"," "),observedAt:n}}function Fn(e){return{provider:e.provider,providerDisconnectObserved:e.disconnectObservation!==null,providerDisconnectMessage:e.disconnectObservation?.message??null}}function He(e){return{...Fn(e),providerSessionId:e.providerSessionId}}var Gn=new Set(["image","media","font","stylesheet","ping"]),Dn=["doubleclick.net","googleadservices.com","googlesyndication.com","google-analytics.com"];function Fe(){return process.env.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_WSS?.trim()||null}function Re(){return process.env.SERP_BROWSER_PROVIDER?.trim().toLowerCase()==="\u006b\u0065\u0072\u006e\u0065\u006c"?!1:!!Fe()}async function qt(e,t=!1){let n=e.request();try{let a=await n.allHeaders(),s=await fetch(n.url(),{method:n.method(),headers:{accept:a.accept??"*/*","accept-language":a["accept-language"]??"en-US,en;q=0.9","user-agent":a["user-agent"]??"Mozilla/5.0",...t?{cookie:a.cookie??"",referer:a.referer??"https://www.google.com/",...a["x-client-data"]?{"x-client-data":a["x-client-data"]}:{}}:{}},body:n.method()==="GET"||n.method()==="HEAD"?void 0:n.postData()});if(!s.ok)return!1;let i=Buffer.from(await s.arrayBuffer());return await e.fulfill({status:s.status,contentType:s.headers.get("content-type")??"application/octet-stream",headers:{"cache-control":"public, max-age=3600"},body:i}),!0}catch{return!1}}var At=class{browser=null;context=null;page=null;cdpSession=null;providerSessionId=null;disconnectObservation=null;closing=!1;observeDisconnect(t,n){this.closing||this.disconnectObservation||(this.disconnectObservation=zt(t,n))}async launch(t){if(this.page)return;this.closing=!1,this.providerSessionId=null,this.disconnectObservation=null;let n=Fe();if(!n)throw new Error("SERP browser service is not configured");this.browser=await Rn.connectOverCDP(n),this.context=this.browser.contexts()[0]??await this.browser.newContext(),await this.context.addInitScript(()=>{let i=globalThis;typeof i.__name!="function"&&(i.__name=d=>d),typeof i.__publicField!="function"&&(i.__publicField=(d,l,U)=>(d[l]=U,U))}),this.page=this.context.pages()[0]??await this.context.newPage(),this.browser.on("disconnected",()=>this.observeDisconnect("browser_disconnected")),this.context.on("close",()=>this.observeDisconnect("context_closed")),this.page.on("close",()=>this.observeDisconnect("page_closed")),this.page.on("crash",()=>this.observeDisconnect("page_crashed")),this.cdpSession=await this.context.newCDPSession(this.page);let s=await this.cdpSession.send("Browser.getSessionId");this.providerSessionId=Be(s?.sessionId),await this.setupPage(this.page)}async setupPage(t){await t.route("**/*",async n=>{let a=n.request(),s=a.resourceType(),i;try{i=new URL(a.url())}catch{await n.continue();return}let d=i.hostname.toLowerCase();if(!(i.pathname==="/sgasync"&&await qt(n,!0))&&!(i.pathname.includes("/shared_dict/")&&await qt(n))&&!(s==="script"&&(d==="www.google.com"||d==="www.gstatic.com")&&await qt(n))){if(Gn.has(s)){await n.abort("blockedbyclient");return}if(Dn.some(l=>d===l||d.endsWith(`.${l}`))){await n.abort("blockedbyclient");return}await n.continue()}})}async withTemporaryPage(t,n){if(!this.context||this.closing)throw new Error("Browser context is not available");return ft(this.context,a=>this.setupPage(a),t,n)}async navigateToSERP(t,n,a,s,i){if(!this.page)throw new Error("Browser page is not available");let d=new URLSearchParams({q:t,gl:a,hl:s,pws:"0"});return i?.tbs&&d.set("tbs",i.tbs),n&&d.set("uule",n),await this.page.goto(`https://www.google.com/search?${d}`,{waitUntil:"load",timeout:45e3}),{hasPaa:!!await this.page.waitForSelector(H.item,{timeout:8e3}).catch(()=>null)}}async navigateTo(t){if(!this.page)throw new Error("Browser page is not available");await this.page.goto(t,{waitUntil:"load",timeout:45e3})}async navigateToChannel(){throw new Error("YouTube navigation is not supported by the SERP browser driver")}async evaluate(t,n){if(!this.page)throw new Error("Browser page is not available");return this.page.evaluate(t,n)}getPage(){if(!this.page)throw new Error("Browser page is not available");return this.page}get\u004b\u0065\u0072\u006e\u0065\u006cSessionId(){return null}getProviderSessionMetadata(){return{provider:"\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api",providerSessionId:this.providerSessionId,disconnectObservation:this.disconnectObservation}}getDebugSnapshot(){return{\u006b\u0065\u0072\u006e\u0065\u006c:null,context:null,networkLocation:null,serpNavigation:null}}async close(){this.closing=!0;let t=null;try{await this.browser?.close()}catch(a){t=xe(a instanceof Error?a.message:String(a))??"\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 browser close failed",this.disconnectObservation??=zt("browser_close_error",t)}let n=He(this.getProviderSessionMetadata());return this.browser=null,this.context=null,this.page=null,this.cdpSession=null,{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:t===null,browserCloseError:t,...n}}};function Ge(e,t){let n=!!(e?.serpIdentity||e?.\u006b\u0065\u0072\u006e\u0065\u006cProfileName);return t?.forceManaged===!0||Re()&&!n?new At:new It}var tt={austin:"Austin,Texas,United States","new york":"New York,New York,United States","new york city":"New York,New York,United States",nyc:"New York,New York,United States","los angeles":"Los Angeles,California,United States",la:"Los Angeles,California,United States",chicago:"Chicago,Illinois,United States",houston:"Houston,Texas,United States",phoenix:"Phoenix,Arizona,United States",philadelphia:"Philadelphia,Pennsylvania,United States",philly:"Philadelphia,Pennsylvania,United States","san antonio":"San Antonio,Texas,United States",dallas:"Dallas,Texas,United States",miami:"Miami,Florida,United States",seattle:"Seattle,Washington,United States",denver:"Denver,Colorado,United States",loveland:"Loveland,Colorado,United States","loveland co":"Loveland,Colorado,United States","fort collins":"Fort Collins,Colorado,United States",boulder:"Boulder,Colorado,United States","colorado springs":"Colorado Springs,Colorado,United States",boston:"Boston,Massachusetts,United States",atlanta:"Atlanta,Georgia,United States","san francisco":"San Francisco,California,United States",sf:"San Francisco,California,United States",portland:"Portland,Oregon,United States","las vegas":"Las Vegas,Nevada,United States",minneapolis:"Minneapolis,Minnesota,United States",detroit:"Detroit,Michigan,United States",nashville:"Nashville,Tennessee,United States",charlotte:"Charlotte,North Carolina,United States",orlando:"Orlando,Florida,United States","san diego":"San Diego,California,United States",baltimore:"Baltimore,Maryland,United States",sacramento:"Sacramento,California,United States",columbus:"Columbus,Ohio,United States",indianapolis:"Indianapolis,Indiana,United States","san jose":"San Jose,California,United States","fort worth":"Fort Worth,Texas,United States",jacksonville:"Jacksonville,Florida,United States",memphis:"Memphis,Tennessee,United States",louisville:"Louisville,Kentucky,United States",raleigh:"Raleigh,North Carolina,United States",richmond:"Richmond,Virginia,United States","salt lake city":"Salt Lake City,Utah,United States",toronto:"Toronto,Ontario,Canada",vancouver:"Vancouver,British Columbia,Canada",montreal:"Montreal,Quebec,Canada",calgary:"Calgary,Alberta,Canada",ottawa:"Ottawa,Ontario,Canada",london:"London,England,United Kingdom",manchester:"Manchester,England,United Kingdom",birmingham:"Birmingham,England,United Kingdom",edinburgh:"Edinburgh,Scotland,United Kingdom",glasgow:"Glasgow,Scotland,United Kingdom",leeds:"Leeds,England,United Kingdom",sydney:"Sydney,New South Wales,Australia",melbourne:"Melbourne,Victoria,Australia",brisbane:"Brisbane,Queensland,Australia",perth:"Perth,Western Australia,Australia",adelaide:"Adelaide,South Australia,Australia",dublin:"Dublin,Leinster,Ireland"};var De=`3, Nanjemoy,Maryland,United States
 Aaronsburg,Pennsylvania,United States
 Abbeville,Alabama,United States
 Abbeville,Georgia,United States
@@ -21701,10 +21701,10 @@ Zwolle,Louisiana,United States`,Ve={ad:"Andorra",ae:"United Arab Emirates",af:"A
 
 `).trim();return!L||/not available|try again|can't generate/i.test(L)?null:L}function u(g){if(!g||g.startsWith("javascript:"))return null;try{let p=new URL(g,window.location.href);return!["http:","https:"].includes(p.protocol)||p.username||p.password?null:{href:p.href,rawHref:p.href}}catch{return null}}function S(g){if(!g)return[];let p=new Set,w=[];for(let T of Array.from(g.querySelectorAll("a[href]"))){let y=u(T.getAttribute("href")??"");if(!y||p.has(y.rawHref))continue;p.add(y.rawHref);let M="";try{M=new URL(y.href).hostname.replace(/^www\./,"")}catch{}w.push({text:s(T)||M||y.href,href:y.href,rawHref:y.rawHref})}return w}async function o(g){let p=g.querySelector(t.aio.showMoreButton);if(!p||p.getAttribute("aria-expanded")!=="false")return!1;p.click();let w=t.expandWaitMs??1500;return w>0&&await new Promise(T=>setTimeout(T,w)),!0}let r=l(),h=null,c=[],C=!1,O=!1,P=[];if(r){C=await o(r);let g=r.querySelector(t.aio.controller),p=r.querySelector(t.aio.contentSubtree),w=r.querySelector(t.aio.showMoreButton);O=g?.getAttribute("data-trnct")==="false"||w?.getAttribute("aria-expanded")==="true"||!w,h=U(p??g??r),P=(h??"").split(`
 `).map(T=>T.trim()).filter(T=>/^\d+\.\s+.+/.test(T)),c=S(r)}let W=document.querySelector(t.aim.root),b=a==="aim"&&!!W,x=W?.closest(t.aim.wrapper)??W,m=b?U(x):null,k=b&&i(m),A=k?m:null,v=k?S(x):[],N=!!r&&i(h);return{surface:a,aiOverview:{detected:N,text:N?h:null,citations:N?c:[],expanded:N?C:!1,fullyExpanded:N?O:!1,sections:N?P:[]},aiMode:{detected:k&&A!==null,text:A,citations:v}}}import{createHash as ea}from"crypto";var Ze=8e3,Xe=3e3,tn=1e4,na=8;function j(e){return/(^|\.)google\.(?:com|[a-z]{2,3}(?:\.[a-z]{2})?)$/i.test(e)}function gt(e){if(!e||e.length>Ze)return null;try{let t=new URL(e);return t.protocol!=="http:"&&t.protocol!=="https:"||t.username||t.password?null:t}catch{return null}}function aa(e,t){if(!e||e.length>Ze||/^javascript:/i.test(e))return null;try{let n=new URL(e,t);return gt(n.href)}catch{return null}}function wt(e){let t=gt(e);return!t||t.protocol!=="https:"||!j(t.hostname)||t.pathname!=="/goto"?null:t.searchParams.get("url")?t:null}function Xt(e){if(e.linkType!=="google_goto_redirect"||e.resolutionStatus!=="resolved"||!wt(e.rawUrl)||!e.resolvedUrl)return!1;let t=gt(e.resolvedUrl);return!!(t&&!j(t.hostname)&&t.href===e.resolvedUrl&&e.url===t.href)}function et(e){let t=e.normalize("NFKC").toLocaleLowerCase("en-US").replace(/[^\p{L}\p{N}\s]/gu,"").replace(/\s+/g," ").trim();return`paa_${ea("sha256").update(t).digest("hex").slice(0,16)}`}function Ot(e,t){let n=aa(e,t);if(!n)return null;let a=n.href;if(!j(n.hostname))return{url:a,rawUrl:a,resolvedUrl:a,linkType:"plain",resolutionStatus:"not_needed"};if(n.pathname==="/url"){let s=gt(n.searchParams.get("q")??n.searchParams.get("url")??"");return!s||j(s.hostname)?null:{url:s.href,rawUrl:a,resolvedUrl:s.href,linkType:"google_url_redirect",resolutionStatus:"resolved"}}return n.pathname==="/goto"&&n.searchParams.get("url")?{url:a,rawUrl:a,resolvedUrl:null,linkType:"google_goto_redirect",resolutionStatus:"unresolved"}:null}function te(e){return wt(e)?.searchParams.get("url")??null}function en(e,t){try{let n=new URL(e),a=gt(new URL(t,n).href);return!a||j(a.hostname)?null:(!a.hash&&n.hash&&(a.hash=n.hash),a.href)}catch{return null}}async function nn(e,t={}){let n=[...new Set([...e].flatMap(o=>wt(o)?[o]:[]))],a=new Map(n.map(o=>[o,{rawUrl:o,resolvedUrl:null,resolutionStatus:"unresolved"}])),s=typeof globalThis.fetch=="function"?globalThis.fetch.bind(globalThis):null,i=t.fetchImpl??s;if(n.length===0||!i)return{resolutions:[...a.values()],nonGoogleRequestsObserved:0};let d=Math.max(1,t.timeoutMs??Xe),l=Date.now()+Math.max(1,t.totalBudgetMs??tn),U=Math.max(1,Math.min(16,Math.floor(t.concurrency??na))),u=0,S=async()=>{for(;u<n.length;){let o=n[u++],r=wt(o),h=r?.searchParams.get("url");if(!r||!h){a.set(o,{rawUrl:o,resolvedUrl:null,resolutionStatus:"rejected"});continue}r.hash="";let c=r.href;for(let C=0;C<2;C++){let O=l-Date.now();if(O<=0)return;let P=new AbortController,W=setTimeout(()=>P.abort(),Math.min(d,O)),b;try{b=await i(c,{method:"GET",redirect:"manual",headers:{accept:"*/*","user-agent":"Mozilla/5.0"},signal:P.signal})}catch{break}finally{clearTimeout(W)}if(b.status<300||b.status>=400)break;let x=b.headers.get("location");if(!x)break;let m=en(o,x);if(m){a.set(o,{rawUrl:o,resolvedUrl:m,resolutionStatus:"resolved"});break}let k;try{k=new URL(x,c)}catch{a.set(o,{rawUrl:o,resolvedUrl:null,resolutionStatus:"rejected"});break}if(!j(k.hostname)||te(k.href)!==h){a.set(o,{rawUrl:o,resolvedUrl:null,resolutionStatus:"rejected"});break}c=k.href}}};return await Promise.all(Array.from({length:Math.min(U,n.length)},()=>S())),{resolutions:[...a.values()],nonGoogleRequestsObserved:0}}function Qe(e,t){try{return new URL(e.resolvedUrl??e.url).hostname.replace(/^www\./,"")}catch{return t}}async function an(e,t={}){let n=[],a=o=>{for(let r of o)n.push(r),a(r.children??[])};a(e.tree??[]);let s=[...e.flat.flatMap(o=>o.sources??[]),...n.flatMap(o=>o.sources),...e.organicResults??[],...(e.aiOverview?.citations??[]).map(({text:o,href:r,...h})=>({...h,url:r})),...(e.aiMode?.citations??[]).map(({text:o,href:r,...h})=>({...h,url:r}))],i=new Map(s.filter(o=>!!o?.rawUrl).map(o=>[o.rawUrl,o])),d=[...i.values()].filter(o=>o.linkType==="google_goto_redirect"&&o.resolutionStatus!=="resolved").map(o=>o.rawUrl);if(d.length>0){let o=await nn(d,t);for(let r of o.resolutions){let h=i.get(r.rawUrl);h&&i.set(r.rawUrl,r.resolutionStatus==="resolved"&&r.resolvedUrl?{...h,url:r.resolvedUrl,resolvedUrl:r.resolvedUrl,resolutionStatus:"resolved"}:{...h,resolutionStatus:r.resolutionStatus})}}let l=o=>({...o,...i.get(o.rawUrl)??o}),U=o=>o.map(r=>{let h=l(r);return{...h,site:Qe(h,r.site)}});for(let o of e.flat){o.sources=U(o.sources??[]);let r=o.sources[0];o.source_title=r?.title??o.source_title,o.source_site=r?.site??o.source_site,o.source_cite=r?.url??o.source_cite}for(let o of n){o.sources=U(o.sources??[]);let r=o.sources[0];o.sourceTitle=r?.title??o.sourceTitle,o.sourceSite=r?.site??o.sourceSite,o.sourceCite=r?.url??o.sourceCite}for(let o=0;o<(e.organicResults?.length??0);o++){let r=l(e.organicResults[o]);e.organicResults[o]={...r,domain:Qe(r,e.organicResults[o].domain)??""}}let u=o=>{let r=l({url:o.href,rawUrl:o.rawUrl,resolvedUrl:o.resolvedUrl,linkType:o.linkType,resolutionStatus:o.resolutionStatus}),{url:h,...c}=r;return{...o,...c,href:h}};e.aiOverview&&(e.aiOverview.citations=(e.aiOverview.citations??[]).map(u)),e.aiMode&&(e.aiMode.citations=(e.aiMode.citations??[]).map(u));let S=[...i.values()];return e.diagnostics.links={linkCandidatesObserved:i.size,plainLinksObserved:S.filter(o=>o.linkType==="plain").length,googleUrlRedirectsObserved:S.filter(o=>o.linkType==="google_url_redirect").length,googleGotoRedirectsObserved:S.filter(o=>o.linkType==="google_goto_redirect").length,redirectsResolved:S.filter(o=>o.linkType!=="plain"&&o.resolutionStatus==="resolved").length,redirectsUnresolved:S.filter(o=>o.resolutionStatus==="unresolved").length,redirectsRejected:S.filter(o=>o.resolutionStatus==="rejected").length,resolverNonGoogleRequestsObserved:0,paaSourcesPreserved:e.flat.reduce((o,r)=>o+r.sources.length,0),aiOverviewLabelValidated:e.aiOverview?.detected??!1},e}async function sn(e,t,n={}){let a=new Map;for(let r of t){let h=a.get(r.rawUrl);h&&Xt(h)||a.set(r.rawUrl,r.linkType==="google_goto_redirect"&&!Xt(r)?{...r,url:r.rawUrl,resolvedUrl:null,resolutionStatus:"unresolved"}:r)}let s=[...a.values()].filter(r=>r.linkType==="google_goto_redirect"&&!Xt(r));if(s.length===0)return a;let i=Date.now()+(n.totalBudgetMs??tn),d=await nn(s.map(r=>r.rawUrl),n);for(let r of d.resolutions){let h=a.get(r.rawUrl);h&&a.set(r.rawUrl,r.resolutionStatus==="resolved"&&r.resolvedUrl?{...h,url:r.resolvedUrl,resolvedUrl:r.resolvedUrl,resolutionStatus:"resolved"}:{...h,resolutionStatus:r.resolutionStatus})}let l=s.filter(r=>a.get(r.rawUrl)?.resolutionStatus==="unresolved");if(l.length===0||n.browserFallback===!1||Date.now()>=i)return n.onNetworkAudit?.({nonGoogleRequestsObserved:0}),a;let U=n.timeoutMs??Xe,u=await e.context().newPage().catch(()=>null);if(!u)return n.onNetworkAudit?.({nonGoogleRequestsObserved:0}),a;let S=null,o=0;try{await u.route("**/*",async r=>{let h=r.request().url();try{j(new URL(h).hostname)||(o+=1)}catch{}let c=te(h);if(!S||c!==S.token){await r.abort("blockedbyclient").catch(()=>{});return}let C=S;try{let O=await r.fetch({maxRedirects:0,timeout:C.timeoutMs});C.location=O.headers().location??null}finally{await r.abort("blockedbyclient").catch(()=>{}),C.finish()}});for(let r of l){let h=i-Date.now();if(h<=0)break;let c=wt(r.rawUrl),C=c?.searchParams.get("url");if(!c||!C){a.set(r.rawUrl,{...r,resolutionStatus:"rejected"});continue}let O=Math.min(U,h);c.hash="";let P=c.href,W=!1;for(let b=0;b<2;b++){let x,m=new Promise(g=>{x=g}),k=`mcp-goto-${Math.random().toString(36).slice(2)}`;S={token:C,timeoutMs:O,location:null,finish:x},await u.evaluate(({url:g,id:p})=>{let w=document.createElement("iframe");w.id=p,w.hidden=!0,w.src=g,document.body.append(w)},{url:P,id:k}).catch(()=>null),await Promise.race([m,new Promise(g=>setTimeout(g,O))]);let A=S.location;if(S=null,await u.evaluate(g=>document.getElementById(g)?.remove(),k).catch(()=>null),!A)break;let v=en(r.rawUrl,A);if(v){a.set(r.rawUrl,{...r,url:v,resolvedUrl:v,resolutionStatus:"resolved"});break}let N;try{N=new URL(A,P)}catch{W=!0;break}if(!j(N.hostname)||te(N.href)!==C){W=!0;break}P=N.href}W&&a.get(r.rawUrl)?.resolutionStatus!=="resolved"&&a.set(r.rawUrl,{...r,resolutionStatus:"rejected"})}}finally{S=null,await u.close().catch(()=>{}),n.onNetworkAudit?.({nonGoogleRequestsObserved:o})}return a}var on=String.raw`^(?:an error has occurred(?:[.!…]\s*)?(?:please try again later[.!…]?)?|loading|please wait|try again|something went wrong)[.!…\s]*$`,ia=700,sa=1e3,oa=1400,dn=900,da=3,rn=250,xt=class{constructor(t,n,a,s){this.driver=t;this.reporter=n;this.onPaaProgress=a;this.onProviderSession=s}driver;reporter;onPaaProgress;onProviderSession;completeness={paaWithoutAnswer:0,paaWithoutSource:0,paaAnswersRecovered:0,aioShareCaptured:null};aioShareUrlEarly=null;linkDiagnostics=this.emptyLinkDiagnostics();collectedRows=[];partialContext=null;progressSequence=0;clickedQuestions=new Set;paaLifecycle=this.emptyPaaLifecycle(0);checkpointMaterial=null;interactionObservations=[];emptyPaaLifecycle(t){return{requestedQuestions:t,questionsFound:0,targetStatus:"not_met",discoveryStatus:"in_progress",materialStatus:"partial",completionReason:"PAA expansion is still running.",automaticRetries:0,knownQuestions:0,processedQuestions:0,failedInteractions:0,noGrowthConfirmations:0,visibleControls:0,eligibleControls:0,dispatchedInteractions:0,confirmedInteractions:0,unconfirmedInteractions:0}}emptyLinkDiagnostics(){return{linkCandidatesObserved:0,plainLinksObserved:0,googleUrlRedirectsObserved:0,googleGotoRedirectsObserved:0,redirectsResolved:0,redirectsUnresolved:0,redirectsRejected:0,resolverNonGoogleRequestsObserved:0,paaSourcesPreserved:0,aiOverviewLabelValidated:!1}}mergePaaSources(t=[],n=[]){let a=new Map;for(let s of[...t,...n]){let i=a.get(s.rawUrl);if(!i){a.set(s.rawUrl,s);continue}let d=s.resolutionStatus==="resolved"&&i.resolutionStatus!=="resolved"?s:i;a.set(s.rawUrl,{...d,title:d.title??i.title??s.title,site:d.site??i.site??s.site})}return[...a.values()]}normalizeAICitations(t,n){let a=new Set;return n.flatMap(s=>{let i=Ot(s.rawHref??s.href,t);if(!i||a.has(i.rawUrl))return[];a.add(i.rawUrl);let{url:d,...l}=i;return[{...l,text:s.text,href:d}]})}async resolveMaterialLinks(t,n,a,s,i){let d=[...n.flatMap(h=>h.sources??[]),...a,...s.citations.map(({text:h,href:c,...C})=>({...C,url:c})),...i.citations.map(({text:h,href:c,...C})=>({...C,url:c}))],l=new Map(d.map(h=>[h.rawUrl,h])),U=0,u=await sn(t,d,{onNetworkAudit:h=>{U=h.nonGoogleRequestsObserved}}),S=h=>({...h,...u.get(h.rawUrl)??h});for(let h of n){h.sources=(h.sources??[]).map(S);let c=h.sources[0];h.source_title=c?.title??h.source_title,h.source_site=c?.site??h.source_site,h.source_cite=c?.url??h.source_cite}for(let h=0;h<a.length;h++){let c=S(a[h]);if(c.resolvedUrl)try{c.domain=new URL(c.resolvedUrl).hostname.replace(/^www\./,"")}catch{}a[h]=c}let o=h=>{let c=S({url:h.href,rawUrl:h.rawUrl,resolvedUrl:h.resolvedUrl,linkType:h.linkType,resolutionStatus:h.resolutionStatus}),{url:C,...O}=c;return{...O,text:h.text,href:C}};s.citations=s.citations.map(o),i.citations=i.citations.map(o);let r=[...u.values()];this.linkDiagnostics={linkCandidatesObserved:l.size,plainLinksObserved:r.filter(h=>h.linkType==="plain").length,googleUrlRedirectsObserved:r.filter(h=>h.linkType==="google_url_redirect").length,googleGotoRedirectsObserved:r.filter(h=>h.linkType==="google_goto_redirect").length,redirectsResolved:r.filter(h=>h.linkType!=="plain"&&h.resolutionStatus==="resolved").length,redirectsUnresolved:r.filter(h=>h.resolutionStatus==="unresolved").length,redirectsRejected:r.filter(h=>h.resolutionStatus==="rejected").length,resolverNonGoogleRequestsObserved:U,paaSourcesPreserved:n.reduce((h,c)=>h+(c.sources??[]).length,0),aiOverviewLabelValidated:s.detected}}normalizeQuestion(t){return t.toLowerCase().replace(/[^\w\s]/g,"").replace(/\s+/g," ").trim()}throwIfAborted(t){if(t?.aborted)throw t.reason instanceof DOMException&&t.reason.name==="TimeoutError"?t.reason:new _}async throwIfCaptcha(t,n){if(await t.locator(H.captchaMarker).count().catch(()=>0)>0)throw new E(`${n} returned a CAPTCHA \u2014 retrying with a fresh session.`)}async extractVisibleItems(t,n){let a=H,s=await t.evaluate(({selectors:d,onlyQuestion:l,answerPlaceholderPattern:U})=>{function u(o){if(!o)return"";let r=[];for(let h of o.childNodes)if(h.nodeType===Node.TEXT_NODE){let c=h.textContent?.trim();c&&r.push(c)}else{if(h.tagName==="STYLE"||h.tagName==="SCRIPT")continue;{let c=u(h);c&&r.push(c)}}return r.join(" ").replace(/\s+/g," ").trim()}function S(o){return(o??"").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i,"").replace(/\s*-\s*Opens in new tab.*$/i,"").trim()}return Array.from(document.querySelectorAll(d.item)).filter(o=>!l||(o.getAttribute(d.itemDataQ)||o.getAttribute(d.itemDataInitQ)||o.querySelector(d.itemQuestionEl)?.innerText?.trim()||"")===l).map(o=>{let r=o.querySelector(d.clickTarget),h=o.classList.contains(d.expandedClass)||r?.getAttribute("aria-expanded")==="true",c=o.querySelector(d.answerContainer),O=c?.querySelector(d.aiAnswerBody)??c,P=u(O),W=new RegExp(U,"i").test(P),b=[...Array.from((O??o).querySelectorAll("a[href]")),...Array.from(o.querySelectorAll(d.aiCitation)),...Array.from(o.querySelectorAll(d.aiSourceCard))],x=!W&&(h&&P.length>=10||P.length>30&&b.some(v=>/^https?:\/\//.test(v.href))),m=x&&o.querySelector(d.sourceTitle)?.innerText?.trim()||void 0,k=x&&o.querySelector(d.sourceSite)?.innerText?.trim()||void 0,A=[];if(x){let v=new Set;for(let g of b)!/^https?:\/\//.test(g.href)||v.has(g.href)||(v.add(g.href),A.push({title:S(g.getAttribute("aria-label"))||(g.textContent||"").trim(),rawHref:g.href}));let N=A.find(g=>g.title.length>2)??A[0];N&&(m=m||N.title||void 0)}return{googleLinkId:o.getAttribute("data-lk")||o.querySelector("[data-lk]")?.getAttribute("data-lk")||null,googleEvidenceId:o.getAttribute("data-ved")||o.querySelector("[data-ved]")?.getAttribute("data-ved")||null,question:o.getAttribute(d.itemDataQ)||o.getAttribute(d.itemDataInitQ)||o.querySelector(d.itemQuestionEl)?.innerText?.trim()||"",answer:x&&P||void 0,sourceTitle:m,sourceSite:k,sources:A}})},{selectors:a,onlyQuestion:n,answerPlaceholderPattern:on}),i=typeof t.url=="function"?t.url():"https://www.google.com/";return s.flatMap(d=>{let l=[],U=new Set;for(let r of d.sources){let h=Ot(r.rawHref,i);if(!h||U.has(h.rawUrl))continue;U.add(h.rawUrl);let c=d.sourceSite??null;try{c||=new URL(h.resolvedUrl??h.rawUrl).hostname.replace(/^www\./,"")}catch{}l.push({...h,title:r.title||d.sourceTitle||null,site:c})}let u=l[0],S={questionId:et(d.question),googleLinkId:d.googleLinkId,googleEvidenceId:d.googleEvidenceId,question:d.question,answer:$e(d.answer,d.question,d.sourceTitle),sourceTitle:u?.title??d.sourceTitle,sourceSite:u?.site??d.sourceSite,sourceCite:u?.url,sources:l},o=ge.safeParse(S);return o.success?[o.data]:(console.warn("[PAAExtractor] item parse failed:",d.question,o.error.issues[0]?.message),[])})}jitteredDelayMs(t,n){return t+Math.floor(Math.random()*(n-t+1))}async clickItem(t,n){try{await t.locator(`${H.item}[data-q="${n}"], ${H.item}[data-initq="${n}"]`).first().click()}catch{}}toFlatRow(t,n,a,s){return{seed_query:s,question_id:t.questionId??et(t.question),google_link_id:t.googleLinkId??"",google_evidence_id:t.googleEvidenceId??"",question:t.question,answer:t.answer??"",source_title:t.sourceTitle??"",source_site:t.sourceSite??"",source_cite:t.sourceCite??"",sources:t.sources??[],depth:n,parent_question:a??"",extracted_at:new Date().toISOString()}}upsertCollectedRow(t){let n=this.normalizeQuestion(t.question),a=this.collectedRows.findIndex(l=>this.normalizeQuestion(l.question)===n);if(a<0)return this.collectedRows.push({...t}),!0;let s=this.collectedRows[a],i={...s,...t,answer:t.answer||s.answer,source_title:t.source_title||s.source_title,source_site:t.source_site||s.source_site,source_cite:t.source_cite||s.source_cite,sources:this.mergePaaSources(s.sources,t.sources),extracted_at:t.extracted_at||s.extracted_at},d=i.answer!==s.answer||i.source_title!==s.source_title||i.source_site!==s.source_site||i.source_cite!==s.source_cite||JSON.stringify(i.sources)!==JSON.stringify(s.sources);return d&&(this.collectedRows[a]=i),d}async emitProgress(t){if(!this.onPaaProgress||this.collectedRows.length===0&&!this.checkpointMaterial)return;let n={sequence:++this.progressSequence,observedAt:new Date().toISOString(),clickedQuestions:[...this.clickedQuestions],phase:t,records:this.collectedRows.map(a=>({...a})),lifecycle:{...this.paaLifecycle},...this.checkpointMaterial?{material:structuredClone(this.checkpointMaterial)}:{},interactions:this.interactionObservations.map(a=>({...a,confirmationSignals:[...a.confirmationSignals]}))};try{await this.onPaaProgress(n)}catch(a){console.warn(JSON.stringify({event:"paa_progress_sink_failed",sequence:n.sequence,phase:n.phase,message:a instanceof Error?a.message:String(a)}))}}getPartialResult(){let t=this.partialContext;if(!t||this.collectedRows.length===0&&!this.checkpointMaterial)return null;let n=this.collectedRows.map(l=>({...l})),a=this.checkpointMaterial,s=t.questionsOnly?0:n.filter(l=>!l.answer?.trim()).length,i=t.questionsOnly?0:n.filter(l=>(l.sources??[]).length===0).length,d={seed:t.seed,totalQuestions:n.length,maxDepthReached:n.reduce((l,U)=>Math.max(l,U.depth),0),durationMs:Date.now()-t.startMs,errorCount:0};return{seed:t.seed,location:t.location,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"paa_found",problem:null,...a?.pagination?{pagination:{...a.pagination}}:{},resultQuality:"partial",degradedResult:!1,retryRecommended:!0,paaLifecycle:{...this.paaLifecycle,questionsFound:n.length,targetStatus:n.length>=(t.maxQuestions??0)?"met":"not_met",discoveryStatus:"interrupted",materialStatus:s>0||i>0?"partial":"complete",completionReason:"The browser attempt ended before frontier exhaustion could be proven."},completeness:{...this.completeness,paaWithoutAnswer:s,paaWithoutSource:i,paaRequested:t.maxQuestions,paaReturned:n.length,paaUnique:n.length},warnings:[{code:"paa_partial",surface:"paa",message:`Harvest stopped before finishing; returning the ${n.length} question(s) collected up to that point.`,retryable:!0}]},totalQuestions:n.length,surface:a?.surface??"web",aiOverview:a?.aiOverview??{detected:!1,text:null,citations:[]},aiMode:a?.aiMode??{detected:!1,text:null,citations:[]},whatPeopleSaying:a?.whatPeopleSaying??[],tree:this.buildTree(n,t.seed),flat:n,videos:a?.videos??[],forums:a?.forums??[],organicResults:a?.organicResults??[],localPack:a?.localPack??[],entityIds:a?.entityIds??{entities:[],kgIds:[],cids:[],gcids:[]},stats:d}}async runBFS(t,n,a){await t.waitForTimeout(700);let s=new Set,i=new Set,d=[];this.clickedQuestions=new Set,this.interactionObservations=[];let l=new Set,U=new Map,u=new Set,S=new Set,o=new Map,r=2,h=3,c=0,C={sel:H.item,dataQ:H.itemDataQ,dataInitQ:H.itemDataInitQ,questionEl:H.itemQuestionEl,answerSel:H.answerContainer,expandedClass:H.expandedClass,clickTarget:H.clickTarget,answerPlaceholderPattern:on,questionsOnly:n.questionsOnly,sourceSel:"a[href]"},O=()=>t.evaluate(({sel:k,dataQ:A,dataInitQ:v,questionEl:N,expandedClass:g,clickTarget:p,answerSel:w,answerPlaceholderPattern:T,questionsOnly:y})=>{let M=L=>{if(!L)return"";let f=[];for(let I of L.childNodes){if(I.nodeType===Node.TEXT_NODE){let F=I.textContent?.trim();F&&f.push(F);continue}if(I.nodeType!==Node.ELEMENT_NODE)continue;let R=I;if(R.tagName==="STYLE"||R.tagName==="SCRIPT"||R.getAttribute("aria-hidden")==="true")continue;let Y=M(R);Y&&f.push(Y)}return f.join(" ").replace(/\s+/g," ").trim()};return Array.from(document.querySelectorAll(k)).map((L,f)=>{let I=M(L.querySelector(w)).replace(/An AI Overview is not available for this search/gi," ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi," ").replace(/\bAI Overview\b/gi," ").replace(/\s+/g," ").trim(),R=L.classList.contains(g)||L.querySelector(p)?.getAttribute("aria-expanded")==="true",Y=I.length>=10&&!new RegExp(T,"i").test(I);return{q:L.getAttribute(A)||L.getAttribute(v)||L.querySelector(N)?.innerText?.trim()||"",done:R&&(y||Y),expanded:R,index:f}}).filter(L=>L.q)},C),P=async(k,A,v,N,g,p)=>{let w=Date.now(),T={id:`${p}:${et(k)}:${g}`,questionId:et(k),question:k,batchNumber:p,controlAttempt:g,observedAt:new Date(w).toISOString(),beforeQuestionCount:A,expandedBefore:N},y=v>=0?t.locator(C.sel).nth(v).locator(C.clickTarget):null,M=null,L=y?await(async()=>(N&&(await y.click({timeout:dn,force:!0}),await t.waitForTimeout(120)),await y.click({timeout:dn,force:!0}),!0))().catch(D=>(M=D instanceof Error&&(D.name==="TimeoutError"||/timeout/i.test(D.message))?"click_ack_timeout":"click_failed",!1)):!1,f=Date.now()-w;if(!L&&M!=="click_ack_timeout")return{dispatched:!1,confirmed:!1,observation:{...T,dispatched:!1,outcome:"dispatch_failed",confirmationSignals:[],confirmationCheckpointMs:null,confirmedAtMs:null,dispatchDurationMs:f,totalDurationMs:f,afterQuestionCount:A,newQuestionCount:0,expandedAfter:N,answerObserved:!1,sourceCount:0,errorCode:v<0?"control_missing":M??"click_failed"}};let I={exists:!0,expanded:N,answerReady:!1,sourceCount:0,questionCount:A},R=[],Y=null,F=null,G=!1;for(let D of[ia,sa,oa]){let at=w+D-Date.now();if(at>0&&await t.waitForTimeout(at),I=await t.evaluate(({sel:lt,dataQ:X,dataInitQ:yt,questionEl:it,expandedClass:Ct,clickTarget:St,answerSel:Vt,answerPlaceholderPattern:Mt,sourceSel:mn,target:vn})=>{let re=st=>{if(!st)return"";let Et=[];for(let pt of st.childNodes){if(pt.nodeType===Node.TEXT_NODE){let ue=pt.textContent?.trim();ue&&Et.push(ue);continue}if(pt.nodeType!==Node.ELEMENT_NODE)continue;let mt=pt;if(mt.tagName==="STYLE"||mt.tagName==="SCRIPT"||mt.getAttribute("aria-hidden")==="true")continue;let he=re(mt);he&&Et.push(he)}return Et.join(" ").replace(/\s+/g," ").trim()},Yt=Array.from(document.querySelectorAll(lt)),kt=Yt.find(st=>(st.getAttribute(X)||st.getAttribute(yt)||st.querySelector(it)?.innerText?.trim())===vn);if(!kt)return{exists:!1,expanded:!1,answerReady:!1,sourceCount:0,questionCount:Yt.length};let Nn=kt.classList.contains(Ct)||kt.querySelector(St)?.getAttribute("aria-expanded")==="true",le=kt.querySelector(Vt),Se=re(le).replace(/An AI Overview is not available for this search/gi," ").replace(/Can't generate an AI overview right now\.?\s*Try again later\.?/gi," ").replace(/\bAI Overview\b/gi," ").replace(/\s+/g," ").trim(),Ue=Se.length>=10&&!new RegExp(Mt,"i").test(Se);return{exists:!0,expanded:Nn,answerReady:Ue,sourceCount:Ue?le?.querySelectorAll(mn).length??0:0,questionCount:Yt.length}},{...C,target:k}).catch(()=>I),R=[...I.expanded?["expanded"]:[],...I.answerReady?["answer"]:[],...I.sourceCount>0?["source"]:[],...I.questionCount>A?["question_growth"]:[]],I.answerReady||I.sourceCount>0||I.questionCount>A){G=!0,Y=D,F=Date.now()-w;break}}let J=Date.now()-w,B={...T,dispatched:L,outcome:G?"confirmed":L?"unconfirmed":"dispatch_failed",confirmationSignals:R,confirmationCheckpointMs:Y,confirmedAtMs:F,dispatchDurationMs:f,totalDurationMs:J,afterQuestionCount:I.questionCount,newQuestionCount:Math.max(0,I.questionCount-A),expandedAfter:I.expanded,answerObserved:I.answerReady,sourceCount:I.sourceCount,errorCode:L?G?null:I.exists?"confirmation_timeout":"control_missing":M};return{dispatched:L,confirmed:G,observation:B}},W=0;t:for(;;){if(n.softDeadlineMs&&Date.now()>=n.softDeadlineMs)throw this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"interrupted",completionReason:"The execution deadline arrived before frontier exhaustion could be proven."},new ht(this.paaLifecycle.completionReason);this.throwIfAborted(a);let k=await O(),A=!1;for(let w of k)i.has(w.q)||(i.add(w.q),d.push(w.q),d.length<=n.maxQuestions&&(A=this.upsertCollectedRow(this.toFlatRow({questionId:et(w.q),googleLinkId:null,googleEvidenceId:null,question:w.q,sources:[]},1,null,n.query))||A)),w.done&&u.add(w.q);if(this.paaLifecycle={...this.paaLifecycle,questionsFound:Math.min(d.length,n.maxQuestions),targetStatus:d.length>=n.maxQuestions?"met":"not_met",knownQuestions:d.length,processedQuestions:u.size,failedInteractions:S.size,visibleControls:k.length},A&&await this.emitProgress("discovered"),n.questionsOnly&&i.size>=n.maxQuestions){this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"target_reached",targetStatus:"met",completionReason:"The requested unique-question count was observed."};break}if(!n.questionsOnly&&i.size>=n.maxQuestions){let w=!1,T=new Set(d.slice(0,n.maxQuestions));for(let y of await this.extractVisibleItems(t)){if(!T.has(y.question))continue;let M=o.get(y.question),L={...y,answer:y.answer??M?.answer,sourceTitle:y.sourceTitle??M?.sourceTitle,sourceSite:y.sourceSite??M?.sourceSite,sourceCite:y.sourceCite??M?.sourceCite,sources:this.mergePaaSources(M?.sources,y.sources)};o.set(y.question,L),w=this.upsertCollectedRow(this.toFlatRow(L,1,null,n.query))||w}if(w&&await this.emitProgress("answer_captured"),[...T].every(y=>{let M=o.get(y);return!!(M?.answer?.trim()&&M.sources.length>0)})){S.clear(),this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"target_reached",targetStatus:"met",failedInteractions:0,completionReason:"The requested unique-question count and its answer/source material were observed."};break}}let v=new Set(d.slice(0,n.maxQuestions)),N=k.filter(w=>v.has(w.q)&&!w.done&&(U.get(w.q)??0)<r&&!l.has(w.q)),g=N.slice(-da);if(this.paaLifecycle={...this.paaLifecycle,eligibleControls:N.length},g.length===0){if(i.size>=n.maxQuestions){this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"target_reached",targetStatus:"met",completionReason:"The requested unique-question count was observed."};break}let w=new Set(k.map(y=>y.q)),T=d.slice(0,n.maxQuestions).filter(y=>!u.has(y)&&(S.has(y)||!w.has(y)));if(T.length>0)throw this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"interrupted",failedInteractions:new Set([...S,...T]).size,completionReason:"At least one known PAA control failed or disappeared before it could be confirmed."},await this.emitProgress("expansion_finished"),new ht(this.paaLifecycle.completionReason);for(let y=1;y<=h;y++){this.throwIfAborted(a),await this.throwIfCaptcha(t,"Google PAA exhaustion confirmation"),await t.mouse.wheel(0,320);let M=await t.waitForFunction(({sel:f,min:I})=>document.querySelectorAll(f).length>I,{sel:H.item,min:k.length},{timeout:1500,polling:100}).catch(()=>null);if(this.paaLifecycle={...this.paaLifecycle,noGrowthConfirmations:y},M||(await O()).some(f=>!i.has(f.q)))continue t}this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"frontier_exhausted",targetStatus:"not_met",failedInteractions:0,noGrowthConfirmations:h,completionReason:"Every known eligible PAA control was processed and three healthy no-growth confirmations found no additional questions."};break}await this.throwIfCaptcha(t,"Google PAA expansion"),c+=1;for(let w of g)this.reporter.onDepth(++W),(U.get(w.q)??0)>0&&(this.paaLifecycle={...this.paaLifecycle,automaticRetries:this.paaLifecycle.automaticRetries+1}),U.set(w.q,(U.get(w.q)??0)+1);let p=await Promise.all(g.map(async w=>({target:w,result:await P(w.q,k.length,w.index,w.expanded,U.get(w.q)??1,c)})));for(let{target:w,result:T}of p)(T.dispatched||T.confirmed)&&this.clickedQuestions.add(w.q),this.interactionObservations.push(T.observation);if(this.interactionObservations.length>rn&&this.interactionObservations.splice(0,this.interactionObservations.length-rn),!n.questionsOnly){let w=new Set(g.map(T=>T.q));for(let T of await this.extractVisibleItems(t))w.has(T.question)&&(o.set(T.question,T),this.upsertCollectedRow(this.toFlatRow(T,1,null,n.query))&&await this.emitProgress("answer_captured"))}for(let{target:w,result:T}of p){let y=o.get(w.q);T.confirmed||y?.answer?.trim()?(u.add(w.q),S.delete(w.q)):(U.get(w.q)??0)>=r&&(S.add(w.q),l.add(w.q))}this.paaLifecycle={...this.paaLifecycle,processedQuestions:u.size,failedInteractions:S.size,noGrowthConfirmations:0,dispatchedInteractions:this.interactionObservations.filter(w=>w.dispatched).length,confirmedInteractions:this.interactionObservations.filter(w=>w.outcome==="confirmed").length,unconfirmedInteractions:this.interactionObservations.filter(w=>w.outcome!=="confirmed").length},await this.emitProgress("click_observed")}let b=n.questionsOnly?new Map:new Map(o);n.questionsOnly||await this.fillIncompleteItems(t,d,b,n,l);let x=[];for(let k of d){if(x.length>=n.maxQuestions)break;let A=this.normalizeQuestion(k);if(s.has(A))continue;s.add(A);let v=b.get(k);if(v){let N=this.toFlatRow(v,1,null,n.query);this.upsertCollectedRow(N),x.push(N),this.reporter.onQuestion({questionId:v.questionId,googleLinkId:v.googleLinkId,googleEvidenceId:v.googleEvidenceId,question:v.question,answer:v.answer??null,sourceTitle:v.sourceTitle??null,sourceSite:v.sourceSite??null,sourceCite:v.sourceCite??null,sources:v.sources,depth:1,parentQuestion:null,children:[]})}else{let N=this.toFlatRow({questionId:et(k),googleLinkId:null,googleEvidenceId:null,question:k,answer:void 0,sourceTitle:void 0,sourceSite:void 0,sourceCite:void 0,sources:[]},1,null,n.query);this.upsertCollectedRow(N),x.push(N)}}this.completeness.paaWithoutAnswer=n.questionsOnly?0:x.filter(k=>!k.answer).length,this.completeness.paaWithoutSource=n.questionsOnly?0:x.filter(k=>(k.sources??[]).length===0).length,this.completeness.paaRequested=n.maxQuestions,this.completeness.paaReturned=x.length,this.completeness.paaUnique=s.size;let m=!n.questionsOnly&&(this.completeness.paaWithoutAnswer>0||this.completeness.paaWithoutSource>0);return this.paaLifecycle={...this.paaLifecycle,questionsFound:x.length,targetStatus:x.length>=n.maxQuestions?"met":"not_met",materialStatus:m?"partial":"complete",processedQuestions:u.size,failedInteractions:S.size},await this.emitProgress("fill_incomplete"),await this.emitProgress("expansion_finished"),x}async fillIncompleteItems(t,n,a,s,i){let d=new Set(n.slice(0,s.maxQuestions)),l=S=>!S?.answer||(S.sources??[]).length===0,U=new Set([...d].filter(S=>l(a.get(S)))),u=new Set([...U].filter(S=>!a.get(S)?.answer));if(U.size!==0){for(let S of U){if(s.softDeadlineMs&&Date.now()>=s.softDeadlineMs)break;let o=(await this.extractVisibleItems(t,S))[0];if(!o)continue;let r=a.get(S);a.set(S,{...o,answer:o.answer??r?.answer,sourceTitle:o.sourceTitle??r?.sourceTitle,sourceSite:o.sourceSite??r?.sourceSite,sourceCite:o.sourceCite??r?.sourceCite,sources:this.mergePaaSources(r?.sources,o.sources)});let h=a.get(S);h&&this.upsertCollectedRow(this.toFlatRow(h,1,null,s.query))&&await this.emitProgress("fill_incomplete")}this.completeness.paaAnswersRecovered=[...u].filter(S=>a.get(S)?.answer).length}}async reExpandCollapsedItems(t){let n=`${H.item}:not(.${H.expandedClass}) ${H.clickTarget}`,a=await t.locator(n).count().catch(()=>0);if(a===0)return!1;for(let s=0;s<a;s++)try{let i=t.locator(n).first();await i.scrollIntoViewIfNeeded(),await i.click({force:!0}),await t.waitForTimeout(this.jitteredDelayMs(300,500))}catch{}return!0}async extractVideos(t){let n=Te;return t.evaluate(a=>{let s=[],i=Array.from(document.querySelectorAll(a.container));for(let d of i){let u=(d.querySelector(a.sectionHeading)?.textContent?.trim()??"").toLowerCase().includes("short")?"short_video":"video",S=Array.from(d.querySelectorAll(a.item));for(let o of S){let r=o.href;if(!r||!r.includes("youtube")&&!r.includes("youtu.be"))continue;let h=o.textContent?.trim()??"",c=h.indexOf("YouTube");if(c===-1)continue;let C=h.slice(0,c).trim(),P=h.slice(c+7).replace(/^[·\s·]+/,"").match(/^([^·\n]+)/),W=P?P[1].trim():"";C&&s.push({type:u,title:C,channel:W,platform:"YouTube",duration:"",url:r})}}return s},n)}async extractForums(t){let n=fe;return t.evaluate(a=>{let s=[],d=Array.from(document.querySelectorAll(a.section)).find(U=>U.textContent?.includes("Discussions"));if(!d)return s;let l=Array.from(d.querySelectorAll(a.item));for(let U of l){let u=U.href;if(!u)continue;let S=U.querySelector(a.title),o=U.querySelector(a.source),r=S?.textContent?.trim()??"",h=o?.textContent?.trim()??"";r&&s.push({title:r,source:h,url:u})}return s},n)}async extractShortVideos(t,n){try{await t.goto(n,{waitUntil:"domcontentloaded"}),await t.waitForTimeout(1500)}catch{return[]}await this.throwIfCaptcha(t,"Google short video search");let a={item:Tt.item,platforms:[...Tt.platforms]};return(await t.evaluate(i=>{let d=new Set,l=[],U=Array.from(document.querySelectorAll(i.item)),u=["youtube.com","youtu.be","tiktok.com","instagram.com","facebook.com","fb.watch"],S=new Map;for(let o of U){let r=o.href;if(!r||!u.some(c=>r.includes(c)))continue;let h=o.textContent?.trim()??"";S.has(r)||S.set(r,[]),S.get(r).push(h)}for(let[o,r]of S.entries()){if(d.has(o))continue;d.add(o);let h=r.find(W=>/^\d+:\d+$/.test(W))??"",c=r.find(W=>!/^\d+:\d+$/.test(W)&&W.length>5)??"";if(!c)continue;let C=c,O="",P="";for(let W of i.platforms){let b=-1,x=0;for(;;){let N=c.indexOf(W,x);if(N===-1)break;b=N,x=N+1}if(b===-1)continue;let m=c.slice(b+W.length);if(!(/^[\s·]/.test(m)||m.trim()===""))continue;C=c.slice(0,b).trim(),O=W;let A=m.replace(/^[\s·]+/,""),v=A.indexOf("\xB7");P=(v===-1?A:A.slice(0,v)).trim();break}C&&l.push({title:C,channel:P,platform:O,duration:h,url:o})}return l},a)).map(i=>({type:"short_video",...i}))}async extractWhatPeopleSaying(t){let n=Pe;return t.evaluate(a=>{let s=Array.from(document.querySelectorAll(a.sectionTag)).find(i=>i.textContent?.includes(a.sectionHeadingText))??document.querySelector(".yG4QQe.TBC9ub.NbhJ1c");return s?Array.from(s.querySelectorAll(a.card)).map(i=>{let l=i.querySelector(a.cardLink)?.href??"",U=i.querySelector(a.titleH1)?.textContent?.trim(),u=i.querySelector(a.titleDiv)?.textContent?.trim(),S=U??u??"",o=i.querySelector(a.source)?.textContent?.trim()??"",h=i.querySelector(a.platformBadge)?.textContent?.trim()??"",c=i.querySelector(a.ytChannel)?.textContent?.trim()??"",C=i.querySelector(a.ytDate)?.textContent?.trim()??"",O=i.querySelector(a.authorNote)?.textContent?.trim()??null,P=i.querySelector(a.popularCommentLabel),W=null;if(P){let M=P.nextSibling;for(;M;){let L=M.textContent?.trim();if(L){W=L;break}M=M.nextSibling}}let b=Array.from(i.querySelectorAll("span")),x=b.find(M=>/^\d+:\d+$/.test(M.textContent?.trim()??""))?.textContent?.trim()??null,k=b.map(M=>M.textContent?.trim()??"").filter(M=>/\d/.test(M)&&(M.includes("comment")||M.includes("reaction")||M.includes("view")||M.includes("like")||M.includes("share")))[0]??"",A=b.map(M=>M.textContent?.trim()??"").filter(M=>/\d+ (day|week|month|year|hour)s? ago|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/.test(M)),v=C||(A[0]??""),N=h||(c?"YouTube":""),g=c||o,p="unknown",w=N.toLowerCase(),T=g.toLowerCase(),y=o.toLowerCase();return w.includes("reddit")||T.startsWith("r/")?p="reddit":w.includes("facebook")||y.includes("facebook")?p="facebook":w.includes("instagram")||y.includes("instagram")?p="instagram":w.includes("tiktok")||y.includes("tiktok")?p="tiktok":w.includes("youtube")||c?p="youtube":p="news",{type:p,title:S,url:l,source:g,platform:N,popularComment:W,engagement:k,date:v,duration:x,authorNote:O}}):[]},n)}async extractOrganicResults(t){let n=Ie,a=await t.evaluate(i=>{let d=[],l=0;return document.querySelectorAll(i.result).forEach(U=>{let u=U.querySelector(i.title);if(!u)return;let S=u.textContent?.trim()??"",r=u.closest("a")?.href??"";if(!S||!r)return;l++;let h=U.querySelector(i.cite)?.textContent?.trim()??null,c=U.querySelector(i.snippet)?.textContent?.trim()??null,C=!!U.querySelector(i.redditCite),O=U.querySelector(i.ratingWrap),P=O?{value:O.querySelector(i.ratingValue)?.textContent?.trim()??"",count:O.querySelector(i.reviewCount)?.textContent?.trim()??""}:null,W="";try{W=new URL(r).hostname.replace(/^www\./,"")}catch{W=U.querySelector(i.siteName)?.textContent?.trim()??""}d.push({position:l,title:S,url:r,domain:W,cite:h,snippet:c,isRedditStyle:C,inlineRating:P})}),d},n),s=typeof t.url=="function"?t.url():"https://www.google.com/";return a.flatMap(i=>{let d=Ot(i.url,s);if(!d)return[];let l=i.domain;try{l=new URL(d.resolvedUrl??d.rawUrl).hostname.replace(/^www\./,"")}catch{}return[{...i,...d,domain:l}]})}async extractLocalPack(t){let n=Ae;return t.evaluate(a=>{let s=[],i=null;return document.querySelectorAll('[role="heading"]').forEach(d=>{!i&&d.textContent?.includes(a.headingText)&&(i=d.closest("[data-hveid]"))}),i&&i.querySelectorAll(a.card).forEach((d,l)=>{let U=d.querySelector(a.name)?.textContent?.trim()??"";if(!U)return;let u=d.querySelector(a.ratingValue)?.textContent?.trim()??null,S=d.querySelector(a.reviewCount)?.textContent?.trim()??null,o=S?S.replace(/[()]/g,"").trim():null,r=d.querySelector("a[data-cid]")?.getAttribute("data-cid")??null;if(!r)for(let P of Array.from(d.querySelectorAll("a[href]"))){let W=P.href.match(/[?&]cid=(\d+)/);if(W){r=W[1];break}let b=P.href.match(/!1s0x[0-9a-f]+:0x([0-9a-f]+)/i);if(b){try{r=BigInt("0x"+b[1]).toString()}catch{}if(r)break}}let h=[];d.querySelectorAll("div, span").forEach(P=>{let W=Array.from(P.childNodes).filter(b=>b.nodeType===3).map(b=>b.textContent?.trim()??"").filter(b=>b.length>1&&b.length<120).join(" ");W&&!h.includes(W)&&h.push(W)});let c=Array.from(d.querySelectorAll("a[href]")),C=c.find(P=>P.href.includes("google.com/maps"))?.href??null,O=c.find(P=>!P.href.includes("google.com")&&P.href.startsWith("http"))?.href??null;s.push({position:l+1,name:U,cid:r,rating:u,reviewCount:o,metadata:h,websiteUrl:O,directionsUrl:C})}),s},n)}async extractEntityIds(t){return t.evaluate(()=>{let n=new Set,a=new Set,s=new Set,i=new Map;function d(U){let u=[".OSrXXb",".dbg0pd",".tzt0oe",'[role="heading"]',"h3"];for(let S of u){let o=U.querySelector(S);if(o?.textContent?.trim())return o.textContent.trim()}return""}document.querySelectorAll('[id^="pv-/g/"]').forEach(U=>{let u=U.getAttribute("id");if(!u)return;let S=u.replace("pv-","");n.add(S);let o=d(U),h=U.querySelector("a[data-cid]")?.getAttribute("data-cid")??null;h&&a.add(h),o&&i.set(S,{name:o,kgId:S,cid:h,gcid:null})}),document.querySelectorAll("[data-mid]").forEach(U=>{let u=U.getAttribute("data-mid");if(u?.startsWith("/g/")&&(n.add(u),!i.has(u))){let S=d(U);S&&i.set(u,{name:S,kgId:u,cid:null,gcid:null})}}),document.querySelectorAll(".w7Dbne").forEach(U=>{let S=U.querySelector("a[data-cid]")?.getAttribute("data-cid")??null;if(!S)return;a.add(S);let o=U.querySelector(".OSrXXb")?.textContent?.trim()??"";if(!o)return;let r=U.querySelector('[id^="pv-/g/"]'),h=r?r.getAttribute("id").replace("pv-",""):null,c=h??`cid:${S}`;if(i.has(c)){let C=i.get(c);C.cid||i.set(c,{...C,cid:S})}else i.set(c,{name:o,kgId:h,cid:S,gcid:null})}),document.querySelectorAll("a[data-cid]").forEach(U=>{let u=U.getAttribute("data-cid");if(!u)return;if(a.add(u),![...i.values()].some(o=>o.cid===u)){let o=U.parentElement,r="";for(let h=0;h<8&&o;h++){let c=o.querySelector('.OSrXXb, .dbg0pd, [role="heading"], h3');if(c?.textContent?.trim()){r=c.textContent.trim();break}o=o.parentElement}r&&i.set(`cid:${u}`,{name:r,kgId:null,cid:u,gcid:null})}});let l=Array.from(document.querySelectorAll("script:not([src])")).map(U=>U.textContent??"").filter(U=>U.length>1e4).join(`
-`);for(let U of l.matchAll(/\/g\/[a-zA-Z0-9_-]{5,20}/g))n.add(U[0]);for(let U of l.matchAll(/gcid:[a-zA-Z0-9_]+/g))s.add(U[0]);for(let U of l.matchAll(/0x[0-9a-f]+:0x([0-9a-f]+)/gi))try{a.add(BigInt("0x"+U[1]).toString())}catch{}return{entities:[...i.values()],kgIds:[...n],cids:[...a],gcids:[...s]}})}mergeLocalPackIntoEntities(t,n){let a=new Set(t.cids),s=t.entities.map(i=>({...i}));for(let i of n){if(!i.cid)continue;a.add(i.cid);let d=i.name.toLowerCase().trim(),l=s.find(U=>U.name.toLowerCase().trim()===d);l?l.cid||(l.cid=i.cid):s.find(U=>U.cid===i.cid)||s.push({name:i.name,kgId:null,cid:i.cid,gcid:null})}return{...t,entities:s,cids:[...a]}}async extractAISurfaces(t,n){let a=await t.evaluate(je,{aio:be,aim:We,expandWaitMs:1500}),s=typeof t.url=="function"?t.url():"https://www.google.com/",i={surface:a.surface,aiOverview:{...a.aiOverview,citations:this.normalizeAICitations(s,a.aiOverview.citations)},aiMode:{...a.aiMode,citations:this.normalizeAICitations(s,a.aiMode.citations)}};if(!i.aiOverview.detected)return this.completeness.aioShareCaptured=null,i;let d=this.aioShareUrlEarly??await this.captureAIOverviewShareUrl(t,n).catch(()=>null);return this.completeness.aioShareCaptured=d!==null,{...i,aiOverview:{...i.aiOverview,shareUrl:d}}}async captureAIOverviewShareUrl(t,n){if(n?.softDeadlineMs&&Date.now()>=n.softDeadlineMs||!await t.evaluate(()=>{let i=Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter(d=>(d.textContent||"").trim()==="AI Overview");for(let d of i){let l=d;for(let U=0;U<8&&l;U++){let u=l.querySelector('[aria-label="Share"]');if(u)return u.scrollIntoView({block:"center"}),u.click(),!0;l=l.parentElement}}return!1}).catch(()=>!1))return null;let s=null;try{s=await(await t.waitForFunction(()=>{let d=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT);for(;d.nextNode();){let l=(d.currentNode.textContent||"").match(/https:\/\/share\.google\/aimode\/\S+/);if(l)return l[0]}return!1},void 0,{timeout:4500,polling:250})).jsonValue()}catch{s=null}return await t.keyboard.press("Escape").catch(()=>{}),s}buildTree(t,n){let a=[],s=new Map;for(let i of t){let d={questionId:i.question_id,googleLinkId:i.google_link_id||null,googleEvidenceId:i.google_evidence_id||null,question:i.question,answer:i.answer||null,sourceTitle:i.source_title||null,sourceSite:i.source_site||null,sourceCite:i.source_cite||null,sources:i.sources??[],depth:i.depth,parentQuestion:i.parent_question||null,children:[]};s.set(i.question,d)}for(let i of s.values())i.parentQuestion&&s.has(i.parentQuestion)?s.get(i.parentQuestion).children.push(i):a.push(i);return a}getBrowserDebugSnapshot(){return this.driver.getDebugSnapshot()}buildHarvestDebugSnapshot(t,n,a,s,i){if(t.debug)return{enabled:!0,request:{query:t.query,locationInput:t.location??null,canonicalLocation:n,locationResolved:i?i.matched:null,locationMatchSource:i?i.source:null,locationAmbiguousCandidates:i?i.ambiguousCandidates:null,uule:a,gl:t.gl,hl:t.hl,device:t.device,proxyMode:t.proxyMode,proxyZip:t.proxyZip??null,serpIdentity:t.serpIdentity??null,serpOnly:t.serpOnly,pages:t.pages??1},browser:this.getBrowserDebugSnapshot(),...s?{locationEvidence:s}:{}}}async captureSecondOrganicPage(t,n,a){if(this.throwIfAborted(a),!this.driver.withTemporaryPage)return{organic:[],status:"failed",failureCode:"unsupported_driver"};let s;try{let i=new URL(t.url()),d=u=>{try{let S=new URL(u,i);return i.origin==="https://www.google.com"&&S.origin===i.origin&&!S.username&&!S.password&&S.pathname==="/search"&&S.searchParams.getAll("q").length===1&&S.searchParams.get("q")===n.query&&S.searchParams.getAll("start").length===1&&S.searchParams.get("start")==="10"&&(!S.searchParams.has("num")||S.searchParams.get("num")==="10")&&["gl","hl"].every(o=>!S.searchParams.has(o)||S.searchParams.getAll(o).length===1&&S.searchParams.get(o)===n[o])&&["uule","tbs","udm","tbm","pws"].every(o=>S.searchParams.getAll(o).length<=1&&S.searchParams.get(o)===i.searchParams.get(o))}catch{return!1}},l=await t.evaluate(()=>document.querySelector('a#pnnext, a[rel="next"]')?.getAttribute("href")??null);if(!l)return{organic:[],status:"unavailable",failureCode:"missing_next"};if(!d(l))return{organic:[],status:"unavailable",failureCode:"invalid_next"};let U=await this.driver.withTemporaryPage(async u=>{if(await u.route("**/*",async S=>{let o=S.request();o.isNavigationRequest()&&o.frame()===u.mainFrame()&&!d(o.url())?(s=/^https:\/\/www\.google\.com\/sorry(?:\/|\?)/.test(o.url())?"captcha":"invalid_next",await S.abort("blockedbyclient")):await S.fallback()}),await u.goto(new URL(l,i).href,{waitUntil:"domcontentloaded",timeout:2e4}),this.throwIfAborted(a),await this.throwIfCaptcha(u,"Google SERP page 2"),!d(u.url()))throw s="invalid_next",new Error("Invalid pagination landing");return this.extractOrganicResults(u)},a);return this.throwIfAborted(a),U.length>0?{organic:U.map(u=>({...u,position:u.position+10})),status:"captured"}:{organic:[],status:"unavailable",failureCode:"empty_page"}}catch(i){if(this.throwIfAborted(a),i instanceof _)throw i;return{organic:[],status:"failed",failureCode:s??(i instanceof E?"captcha":i instanceof Error&&(i.name==="TimeoutError"||/timed? ?out/i.test(i.message))?"timeout":"navigation_error")}}}async extract(t,n){let a=Date.now();this.completeness={paaWithoutAnswer:0,paaWithoutSource:0,paaAnswersRecovered:0,aioShareCaptured:null},this.linkDiagnostics=this.emptyLinkDiagnostics(),this.aioShareUrlEarly=null,this.collectedRows=[],this.progressSequence=0,this.checkpointMaterial=null,this.clickedQuestions=new Set,this.paaLifecycle=this.emptyPaaLifecycle(t.maxQuestions),this.partialContext={seed:t.query,location:t.location??null,startMs:a,maxQuestions:t.maxQuestions,questionsOnly:t.questionsOnly};let s=t.device==="mobile",i={headless:t.headless,profileDir:t.profileDir,proxy:t.proxy,kernelApiKey:t.kernelApiKey,kernelProxyId:t.kernelProxyId,kernelProfileName:t.kernelProfileName,kernelProfileSaveChanges:t.kernelProfileSaveChanges,kernelStealth:t.kernelStealth,kernelProxyResolution:t.kernelProxyResolution,proxyMode:t.proxyMode,keepDefaultProxy:t.keepDefaultProxy,requireUsEgress:t.requireUsEgress,viewport:s?{width:390,height:844}:{width:1280,height:800},locale:`${t.hl}-${t.gl.toUpperCase()}`,deviceScaleFactor:s?3:1,isMobile:s,hasTouch:s,debug:t.debug},d=0,l=[];try{this.throwIfAborted(n),await this.driver.launch(i);let U=this.driver.getProviderSessionMetadata?.();if(U?.providerSessionId&&this.onProviderSession)try{await this.onProviderSession(U)}catch(B){console.warn(JSON.stringify({event:"paa_provider_session_sink_failed",provider:U.provider,message:B instanceof Error?B.message:String(B)}))}this.throwIfAborted(n);let u=t.location?Qt(t.location):null,S=t.location?u?.canonical??t.location:null,o=S?Ke(S):null,r=t,h=t.includeAllSerpFeatures,c=h||t.includeLocalPack,C=h||t.includeForums,O=h||t.includeVideos,P=h||t.includeAiOverview,W=h||t.includeWhatPeopleSaying,b={surface:"web",aiOverview:{detected:!1,text:null,citations:[]},aiMode:{detected:!1,text:null,citations:[]}},{hasPaa:x}=await this.driver.navigateToSERP(r.query,o,t.gl,t.hl,{...Ut(t.recency)?{tbs:Ut(t.recency)}:{},debug:t.debug});this.throwIfAborted(n);let m=this.driver.getPage();if(await this.throwIfCaptcha(m,"Google SERP"),x||(this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"no_paa_observed",materialStatus:"complete",completionReason:"A readable Google SERP was observed without a People Also Ask block."}),t.questionsOnly){let B=x?await this.runBFS(m,r,n):[];await this.resolveMaterialLinks(m,B,[],b.aiOverview,b.aiMode);let D={seed:r.query,totalQuestions:B.length,maxDepthReached:B.reduce((at,lt)=>Math.max(at,lt.depth),0),durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(D),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:x?"paa_found":"no_paa",...x?{}:{noPaaObserved:!0},problem:null,paaLifecycle:{...this.paaLifecycle},completeness:{...this.completeness},links:{...this.linkDiagnostics},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,void 0,u)}:{}},totalQuestions:B.length,surface:"web",aiOverview:{detected:!1,text:null,citations:[]},aiMode:{detected:!1,text:null,citations:[]},whatPeopleSaying:[],tree:this.buildTree(B,r.query),flat:B,videos:[],forums:[],organicResults:[],localPack:[],entityIds:{entities:[],kgIds:[],cids:[],gcids:[]},stats:D}}if(P&&(this.aioShareUrlEarly=await this.captureAIOverviewShareUrl(m,t).catch(()=>null)),t.serpOnly){let[B,D,at]=await Promise.all([this.extractOrganicResults(m),c?this.extractLocalPack(m):Promise.resolve([]),this.extractEntityIds(m)]),lt=this.mergeLocalPackIntoEntities(at,D),X=P?await this.extractAISurfaces(m,t):b,yt=t.debug?ct(S,B,D):void 0,it=B;if((t.pages??1)>=2){let St=new URLSearchParams({q:r.query,gl:t.gl,hl:t.hl,pws:"0",start:"10"});Ut(t.recency)&&St.set("tbs",Ut(t.recency)),o&&St.set("uule",o),await this.driver.navigateTo("https://www.google.com/search?"+St.toString()),await this.throwIfCaptcha(m,"Google SERP page 2");let Vt=await this.extractOrganicResults(m);it=[...B,...Vt.map(Mt=>({...Mt,position:Mt.position+10}))],t.debug&&(yt=ct(S,it,D))}await this.resolveMaterialLinks(m,[],it,X.aiOverview,X.aiMode);let Ct={seed:r.query,totalQuestions:0,maxDepthReached:0,durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(Ct),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"serp_only",problem:null,completeness:{...this.completeness},links:{...this.linkDiagnostics},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,yt,u)}:{}},totalQuestions:0,surface:X.surface,aiOverview:X.aiOverview,aiMode:X.aiMode,whatPeopleSaying:[],tree:[],flat:[],videos:[],forums:[],organicResults:it,localPack:D,entityIds:lt,stats:Ct}}let[k,A,v,N,g,p]=await Promise.all([O?this.extractVideos(m):Promise.resolve([]),C?this.extractForums(m):Promise.resolve([]),W?this.extractWhatPeopleSaying(m):Promise.resolve([]),this.extractEntityIds(m),this.extractOrganicResults(m),c?this.extractLocalPack(m):Promise.resolve([])]),w=this.mergeLocalPackIntoEntities(N,p),T=t.debug?ct(S,g,p):void 0;this.reporter.onVideos(k),this.reporter.onForums(A);let y=P?await this.extractAISurfaces(m,t):b,M={requestedPages:(t.pages??1)>=2?2:1,capturedPages:1,page2Status:(t.pages??1)>=2?"not_attempted":"not_requested",page1OrganicCount:g.length,page2OrganicCount:0},L=g;if(this.checkpointMaterial={surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,whatPeopleSaying:v,videos:k,forums:A,organicResults:L,localPack:p,entityIds:w,pagination:M},await this.emitProgress("serp_captured"),M.requestedPages===2){let B=await this.captureSecondOrganicPage(m,r,n);L=[...g,...B.organic],M={...M,capturedPages:B.status==="captured"?2:1,page2Status:B.status,page2OrganicCount:B.organic.length,...B.failureCode?{failureCode:B.failureCode}:{}},this.checkpointMaterial={...this.checkpointMaterial,organicResults:L,pagination:M},await this.emitProgress("serp_captured"),await this.resolveMaterialLinks(m,[],L,y.aiOverview,y.aiMode),this.checkpointMaterial={...this.checkpointMaterial,organicResults:L},await this.emitProgress("serp_captured")}let f=t.debug?ct(S,L,p):T;if(!x){await this.resolveMaterialLinks(m,[],L,y.aiOverview,y.aiMode),this.checkpointMaterial={...this.checkpointMaterial,organicResults:L},await this.emitProgress("serp_captured");let B={seed:r.query,totalQuestions:0,maxDepthReached:0,durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(B),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"no_paa",noPaaObserved:!0,problem:null,pagination:M,paaLifecycle:{...this.paaLifecycle},completeness:{...this.completeness},links:{...this.linkDiagnostics},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,f,u)}:{}},totalQuestions:0,surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,whatPeopleSaying:v,tree:[],flat:[],videos:k,forums:A,organicResults:L,localPack:p,entityIds:w,stats:B}}let I=await this.runBFS(m,r,n);this.throwIfAborted(n);let R=new URLSearchParams({q:r.query,gl:t.gl,hl:t.hl,pws:"0",udm:Tt.udm});o&&R.set("uule",o);let Y=[];if(t.includeShortVideos!==!1)try{Y=await this.extractShortVideos(m,"https://www.google.com/search?"+R.toString())}catch(B){if(!(B instanceof E))throw B;d++,l.push({code:"short_videos_captcha_skipped",surface:"short_videos",message:B.message,retryable:!0})}this.reporter.onVideos(Y),await this.resolveMaterialLinks(m,I,L,y.aiOverview,y.aiMode),this.checkpointMaterial={...this.checkpointMaterial,surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,videos:[...k,...Y],organicResults:L},await this.emitProgress("expansion_finished");let F=[...k,...Y],G=this.buildTree(I,r.query),J={seed:r.query,totalQuestions:I.length,maxDepthReached:I.reduce((B,D)=>Math.max(B,D.depth),0),durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(J),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"paa_found",problem:null,pagination:M,paaLifecycle:{...this.paaLifecycle},completeness:{...this.completeness},links:{...this.linkDiagnostics},...l.length>0?{warnings:l}:{},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,f,u)}:{}},totalQuestions:I.length,surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,whatPeopleSaying:v,tree:G,flat:I,videos:F,forums:A,organicResults:L,localPack:p,entityIds:w,stats:J}}catch(U){throw d++,this.paaLifecycle.discoveryStatus==="in_progress"&&(this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"interrupted",completionReason:"The browser attempt ended before frontier exhaustion could be proven."}),await this.emitProgress("attempt_ending").catch(()=>{}),this.reporter.onError(U instanceof Error?U:new Error(String(U))),U}}};import{promises as K}from"fs";import nt from"path";import dt from"papaparse";var Bt=class{async writeJSON(t,n){await K.mkdir(n,{recursive:!0});let s=`${t.seed.toLowerCase().replace(/\W+/g,"-").slice(0,40)}-${Date.now()}.json`,i=nt.join(n,s);return await K.writeFile(i,JSON.stringify(t,null,2),"utf8"),await Ne({title:`${t.seed} scrape`,content:JSON.stringify(t),source:`mcp-scraper:${i}`}),i}async writeCSV(t,n){await K.mkdir(n,{recursive:!0});let s=(t[0]?.seed_query??"paa").toLowerCase().replace(/\W+/g,"-").slice(0,40),i=dt.unparse(t.map(({sources:U,...u})=>({...u,sources_json:JSON.stringify(U)})),{header:!0}),d=`${s}-${Date.now()}.csv`,l=nt.join(n,d);return await K.writeFile(l,i,"utf8"),l}async writeVideoCSV(t,n,a){await K.mkdir(a,{recursive:!0});let s=n.toLowerCase().replace(/\W+/g,"-").slice(0,40),i=dt.unparse(t,{header:!0}),d=`${s}-videos-${Date.now()}.csv`,l=nt.join(a,d);return await K.writeFile(l,i,"utf8"),l}async writeForumCSV(t,n,a){await K.mkdir(a,{recursive:!0});let s=n.toLowerCase().replace(/\W+/g,"-").slice(0,40),i=dt.unparse(t,{header:!0}),d=`${s}-forums-${Date.now()}.csv`,l=nt.join(a,d);return await K.writeFile(l,i,"utf8"),l}async writeAIOverviewCSV(t,n,a,s){await K.mkdir(s,{recursive:!0});let i=a.toLowerCase().replace(/\W+/g,"-").slice(0,40),d=t.map((S,o)=>({seed_query:a,response_text:o===0?n??"":"",citation_text:S.text,citation_href:S.href})),l=dt.unparse(d,{header:!0}),U=`${i}-ai-overview-${Date.now()}.csv`,u=nt.join(s,U);return await K.writeFile(u,l,"utf8"),u}async writeAIModeCSV(t,n,a,s){await K.mkdir(s,{recursive:!0});let i=a.toLowerCase().replace(/\W+/g,"-").slice(0,40),d=t.map((S,o)=>({seed_query:a,response_text:o===0?n??"":"",citation_text:S.text,citation_href:S.href})),l=dt.unparse(d,{header:!0}),U=`${i}-ai-mode-${Date.now()}.csv`,u=nt.join(s,U);return await K.writeFile(u,l,"utf8"),u}async writeWhatPeopleSayingCSV(t,n,a){await K.mkdir(a,{recursive:!0});let s=n.toLowerCase().replace(/\W+/g,"-").slice(0,40),i=t.map(u=>({seed_query:n,...u})),d=dt.unparse(i,{header:!0}),l=`${s}-what-people-saying-${Date.now()}.csv`,U=nt.join(a,l);return await K.writeFile(U,d,"utf8"),U}};var Ht=class{onQuestion(t){process.stdout.write(JSON.stringify({event:"question",depth:t.depth,question:t.question})+`
+`);for(let U of l.matchAll(/\/g\/[a-zA-Z0-9_-]{5,20}/g))n.add(U[0]);for(let U of l.matchAll(/gcid:[a-zA-Z0-9_]+/g))s.add(U[0]);for(let U of l.matchAll(/0x[0-9a-f]+:0x([0-9a-f]+)/gi))try{a.add(BigInt("0x"+U[1]).toString())}catch{}return{entities:[...i.values()],kgIds:[...n],cids:[...a],gcids:[...s]}})}mergeLocalPackIntoEntities(t,n){let a=new Set(t.cids),s=t.entities.map(i=>({...i}));for(let i of n){if(!i.cid)continue;a.add(i.cid);let d=i.name.toLowerCase().trim(),l=s.find(U=>U.name.toLowerCase().trim()===d);l?l.cid||(l.cid=i.cid):s.find(U=>U.cid===i.cid)||s.push({name:i.name,kgId:null,cid:i.cid,gcid:null})}return{...t,entities:s,cids:[...a]}}async extractAISurfaces(t,n){let a=await t.evaluate(je,{aio:be,aim:We,expandWaitMs:1500}),s=typeof t.url=="function"?t.url():"https://www.google.com/",i={surface:a.surface,aiOverview:{...a.aiOverview,citations:this.normalizeAICitations(s,a.aiOverview.citations)},aiMode:{...a.aiMode,citations:this.normalizeAICitations(s,a.aiMode.citations)}};if(!i.aiOverview.detected)return this.completeness.aioShareCaptured=null,i;let d=this.aioShareUrlEarly??await this.captureAIOverviewShareUrl(t,n).catch(()=>null);return this.completeness.aioShareCaptured=d!==null,{...i,aiOverview:{...i.aiOverview,shareUrl:d}}}async captureAIOverviewShareUrl(t,n){if(n?.softDeadlineMs&&Date.now()>=n.softDeadlineMs||!await t.evaluate(()=>{let i=Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter(d=>(d.textContent||"").trim()==="AI Overview");for(let d of i){let l=d;for(let U=0;U<8&&l;U++){let u=l.querySelector('[aria-label="Share"]');if(u)return u.scrollIntoView({block:"center"}),u.click(),!0;l=l.parentElement}}return!1}).catch(()=>!1))return null;let s=null;try{s=await(await t.waitForFunction(()=>{let d=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT);for(;d.nextNode();){let l=(d.currentNode.textContent||"").match(/https:\/\/share\.google\/aimode\/\S+/);if(l)return l[0]}return!1},void 0,{timeout:4500,polling:250})).jsonValue()}catch{s=null}return await t.keyboard.press("Escape").catch(()=>{}),s}buildTree(t,n){let a=[],s=new Map;for(let i of t){let d={questionId:i.question_id,googleLinkId:i.google_link_id||null,googleEvidenceId:i.google_evidence_id||null,question:i.question,answer:i.answer||null,sourceTitle:i.source_title||null,sourceSite:i.source_site||null,sourceCite:i.source_cite||null,sources:i.sources??[],depth:i.depth,parentQuestion:i.parent_question||null,children:[]};s.set(i.question,d)}for(let i of s.values())i.parentQuestion&&s.has(i.parentQuestion)?s.get(i.parentQuestion).children.push(i):a.push(i);return a}getBrowserDebugSnapshot(){return this.driver.getDebugSnapshot()}buildHarvestDebugSnapshot(t,n,a,s,i){if(t.debug)return{enabled:!0,request:{query:t.query,locationInput:t.location??null,canonicalLocation:n,locationResolved:i?i.matched:null,locationMatchSource:i?i.source:null,locationAmbiguousCandidates:i?i.ambiguousCandidates:null,uule:a,gl:t.gl,hl:t.hl,device:t.device,proxyMode:t.proxyMode,proxyZip:t.proxyZip??null,serpIdentity:t.serpIdentity??null,serpOnly:t.serpOnly,pages:t.pages??1},browser:this.getBrowserDebugSnapshot(),...s?{locationEvidence:s}:{}}}async captureSecondOrganicPage(t,n,a){if(this.throwIfAborted(a),!this.driver.withTemporaryPage)return{organic:[],status:"failed",failureCode:"unsupported_driver"};let s;try{let i=new URL(t.url()),d=u=>{try{let S=new URL(u,i);return i.origin==="https://www.google.com"&&S.origin===i.origin&&!S.username&&!S.password&&S.pathname==="/search"&&S.searchParams.getAll("q").length===1&&S.searchParams.get("q")===n.query&&S.searchParams.getAll("start").length===1&&S.searchParams.get("start")==="10"&&(!S.searchParams.has("num")||S.searchParams.get("num")==="10")&&["gl","hl"].every(o=>!S.searchParams.has(o)||S.searchParams.getAll(o).length===1&&S.searchParams.get(o)===n[o])&&["uule","tbs","udm","tbm","pws"].every(o=>S.searchParams.getAll(o).length<=1&&S.searchParams.get(o)===i.searchParams.get(o))}catch{return!1}},l=await t.evaluate(()=>document.querySelector('a#pnnext, a[rel="next"]')?.getAttribute("href")??null);if(!l)return{organic:[],status:"unavailable",failureCode:"missing_next"};if(!d(l))return{organic:[],status:"unavailable",failureCode:"invalid_next"};let U=await this.driver.withTemporaryPage(async u=>{if(await u.route("**/*",async S=>{let o=S.request();o.isNavigationRequest()&&o.frame()===u.mainFrame()&&!d(o.url())?(s=/^https:\/\/www\.google\.com\/sorry(?:\/|\?)/.test(o.url())?"captcha":"invalid_next",await S.abort("blockedbyclient")):await S.fallback()}),await u.goto(new URL(l,i).href,{waitUntil:"domcontentloaded",timeout:2e4}),this.throwIfAborted(a),await this.throwIfCaptcha(u,"Google SERP page 2"),!d(u.url()))throw s="invalid_next",new Error("Invalid pagination landing");return this.extractOrganicResults(u)},a);return this.throwIfAborted(a),U.length>0?{organic:U.map(u=>({...u,position:u.position+10})),status:"captured"}:{organic:[],status:"unavailable",failureCode:"empty_page"}}catch(i){if(this.throwIfAborted(a),i instanceof _)throw i;return{organic:[],status:"failed",failureCode:s??(i instanceof E?"captcha":i instanceof Error&&(i.name==="TimeoutError"||/timed? ?out/i.test(i.message))?"timeout":"navigation_error")}}}async extract(t,n){let a=Date.now();this.completeness={paaWithoutAnswer:0,paaWithoutSource:0,paaAnswersRecovered:0,aioShareCaptured:null},this.linkDiagnostics=this.emptyLinkDiagnostics(),this.aioShareUrlEarly=null,this.collectedRows=[],this.progressSequence=0,this.checkpointMaterial=null,this.clickedQuestions=new Set,this.paaLifecycle=this.emptyPaaLifecycle(t.maxQuestions),this.partialContext={seed:t.query,location:t.location??null,startMs:a,maxQuestions:t.maxQuestions,questionsOnly:t.questionsOnly};let s=t.device==="mobile",i={headless:t.headless,profileDir:t.profileDir,proxy:t.proxy,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey,\u006b\u0065\u0072\u006e\u0065\u006cProxyId:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,\u006b\u0065\u0072\u006e\u0065\u006cProfileName:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileName,\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges:t.\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges,\u006b\u0065\u0072\u006e\u0065\u006cStealth:t.\u006b\u0065\u0072\u006e\u0065\u006cStealth,\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution:t.\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution,proxyMode:t.proxyMode,keepDefaultProxy:t.keepDefaultProxy,requireUsEgress:t.requireUsEgress,viewport:s?{width:390,height:844}:{width:1280,height:800},locale:`${t.hl}-${t.gl.toUpperCase()}`,deviceScaleFactor:s?3:1,isMobile:s,hasTouch:s,debug:t.debug},d=0,l=[];try{this.throwIfAborted(n),await this.driver.launch(i);let U=this.driver.getProviderSessionMetadata?.();if(U?.providerSessionId&&this.onProviderSession)try{await this.onProviderSession(U)}catch(B){console.warn(JSON.stringify({event:"paa_provider_session_sink_failed",provider:U.provider,message:B instanceof Error?B.message:String(B)}))}this.throwIfAborted(n);let u=t.location?Qt(t.location):null,S=t.location?u?.canonical??t.location:null,o=S?Ke(S):null,r=t,h=t.includeAllSerpFeatures,c=h||t.includeLocalPack,C=h||t.includeForums,O=h||t.includeVideos,P=h||t.includeAiOverview,W=h||t.includeWhatPeopleSaying,b={surface:"web",aiOverview:{detected:!1,text:null,citations:[]},aiMode:{detected:!1,text:null,citations:[]}},{hasPaa:x}=await this.driver.navigateToSERP(r.query,o,t.gl,t.hl,{...Ut(t.recency)?{tbs:Ut(t.recency)}:{},debug:t.debug});this.throwIfAborted(n);let m=this.driver.getPage();if(await this.throwIfCaptcha(m,"Google SERP"),x||(this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"no_paa_observed",materialStatus:"complete",completionReason:"A readable Google SERP was observed without a People Also Ask block."}),t.questionsOnly){let B=x?await this.runBFS(m,r,n):[];await this.resolveMaterialLinks(m,B,[],b.aiOverview,b.aiMode);let D={seed:r.query,totalQuestions:B.length,maxDepthReached:B.reduce((at,lt)=>Math.max(at,lt.depth),0),durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(D),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:x?"paa_found":"no_paa",...x?{}:{noPaaObserved:!0},problem:null,paaLifecycle:{...this.paaLifecycle},completeness:{...this.completeness},links:{...this.linkDiagnostics},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,void 0,u)}:{}},totalQuestions:B.length,surface:"web",aiOverview:{detected:!1,text:null,citations:[]},aiMode:{detected:!1,text:null,citations:[]},whatPeopleSaying:[],tree:this.buildTree(B,r.query),flat:B,videos:[],forums:[],organicResults:[],localPack:[],entityIds:{entities:[],kgIds:[],cids:[],gcids:[]},stats:D}}if(P&&(this.aioShareUrlEarly=await this.captureAIOverviewShareUrl(m,t).catch(()=>null)),t.serpOnly){let[B,D,at]=await Promise.all([this.extractOrganicResults(m),c?this.extractLocalPack(m):Promise.resolve([]),this.extractEntityIds(m)]),lt=this.mergeLocalPackIntoEntities(at,D),X=P?await this.extractAISurfaces(m,t):b,yt=t.debug?ct(S,B,D):void 0,it=B;if((t.pages??1)>=2){let St=new URLSearchParams({q:r.query,gl:t.gl,hl:t.hl,pws:"0",start:"10"});Ut(t.recency)&&St.set("tbs",Ut(t.recency)),o&&St.set("uule",o),await this.driver.navigateTo("https://www.google.com/search?"+St.toString()),await this.throwIfCaptcha(m,"Google SERP page 2");let Vt=await this.extractOrganicResults(m);it=[...B,...Vt.map(Mt=>({...Mt,position:Mt.position+10}))],t.debug&&(yt=ct(S,it,D))}await this.resolveMaterialLinks(m,[],it,X.aiOverview,X.aiMode);let Ct={seed:r.query,totalQuestions:0,maxDepthReached:0,durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(Ct),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"serp_only",problem:null,completeness:{...this.completeness},links:{...this.linkDiagnostics},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,yt,u)}:{}},totalQuestions:0,surface:X.surface,aiOverview:X.aiOverview,aiMode:X.aiMode,whatPeopleSaying:[],tree:[],flat:[],videos:[],forums:[],organicResults:it,localPack:D,entityIds:lt,stats:Ct}}let[k,A,v,N,g,p]=await Promise.all([O?this.extractVideos(m):Promise.resolve([]),C?this.extractForums(m):Promise.resolve([]),W?this.extractWhatPeopleSaying(m):Promise.resolve([]),this.extractEntityIds(m),this.extractOrganicResults(m),c?this.extractLocalPack(m):Promise.resolve([])]),w=this.mergeLocalPackIntoEntities(N,p),T=t.debug?ct(S,g,p):void 0;this.reporter.onVideos(k),this.reporter.onForums(A);let y=P?await this.extractAISurfaces(m,t):b,M={requestedPages:(t.pages??1)>=2?2:1,capturedPages:1,page2Status:(t.pages??1)>=2?"not_attempted":"not_requested",page1OrganicCount:g.length,page2OrganicCount:0},L=g;if(this.checkpointMaterial={surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,whatPeopleSaying:v,videos:k,forums:A,organicResults:L,localPack:p,entityIds:w,pagination:M},await this.emitProgress("serp_captured"),M.requestedPages===2){let B=await this.captureSecondOrganicPage(m,r,n);L=[...g,...B.organic],M={...M,capturedPages:B.status==="captured"?2:1,page2Status:B.status,page2OrganicCount:B.organic.length,...B.failureCode?{failureCode:B.failureCode}:{}},this.checkpointMaterial={...this.checkpointMaterial,organicResults:L,pagination:M},await this.emitProgress("serp_captured"),await this.resolveMaterialLinks(m,[],L,y.aiOverview,y.aiMode),this.checkpointMaterial={...this.checkpointMaterial,organicResults:L},await this.emitProgress("serp_captured")}let f=t.debug?ct(S,L,p):T;if(!x){await this.resolveMaterialLinks(m,[],L,y.aiOverview,y.aiMode),this.checkpointMaterial={...this.checkpointMaterial,organicResults:L},await this.emitProgress("serp_captured");let B={seed:r.query,totalQuestions:0,maxDepthReached:0,durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(B),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"no_paa",noPaaObserved:!0,problem:null,pagination:M,paaLifecycle:{...this.paaLifecycle},completeness:{...this.completeness},links:{...this.linkDiagnostics},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,f,u)}:{}},totalQuestions:0,surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,whatPeopleSaying:v,tree:[],flat:[],videos:k,forums:A,organicResults:L,localPack:p,entityIds:w,stats:B}}let I=await this.runBFS(m,r,n);this.throwIfAborted(n);let R=new URLSearchParams({q:r.query,gl:t.gl,hl:t.hl,pws:"0",udm:Tt.udm});o&&R.set("uule",o);let Y=[];if(t.includeShortVideos!==!1)try{Y=await this.extractShortVideos(m,"https://www.google.com/search?"+R.toString())}catch(B){if(!(B instanceof E))throw B;d++,l.push({code:"short_videos_captcha_skipped",surface:"short_videos",message:B.message,retryable:!0})}this.reporter.onVideos(Y),await this.resolveMaterialLinks(m,I,L,y.aiOverview,y.aiMode),this.checkpointMaterial={...this.checkpointMaterial,surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,videos:[...k,...Y],organicResults:L},await this.emitProgress("expansion_finished");let F=[...k,...Y],G=this.buildTree(I,r.query),J={seed:r.query,totalQuestions:I.length,maxDepthReached:I.reduce((B,D)=>Math.max(B,D.depth),0),durationMs:Date.now()-a,errorCount:d};return this.reporter.onComplete(J),{seed:r.query,location:t.location??null,extractedAt:new Date().toISOString(),diagnostics:{completionStatus:"paa_found",problem:null,pagination:M,paaLifecycle:{...this.paaLifecycle},completeness:{...this.completeness},links:{...this.linkDiagnostics},...l.length>0?{warnings:l}:{},...t.debug?{debug:this.buildHarvestDebugSnapshot(r,S,o,f,u)}:{}},totalQuestions:I.length,surface:y.surface,aiOverview:y.aiOverview,aiMode:y.aiMode,whatPeopleSaying:v,tree:G,flat:I,videos:F,forums:A,organicResults:L,localPack:p,entityIds:w,stats:J}}catch(U){throw d++,this.paaLifecycle.discoveryStatus==="in_progress"&&(this.paaLifecycle={...this.paaLifecycle,discoveryStatus:"interrupted",completionReason:"The browser attempt ended before frontier exhaustion could be proven."}),await this.emitProgress("attempt_ending").catch(()=>{}),this.reporter.onError(U instanceof Error?U:new Error(String(U))),U}}};import{promises as K}from"fs";import nt from"path";import dt from"papaparse";var Bt=class{async writeJSON(t,n){await K.mkdir(n,{recursive:!0});let s=`${t.seed.toLowerCase().replace(/\W+/g,"-").slice(0,40)}-${Date.now()}.json`,i=nt.join(n,s);return await K.writeFile(i,JSON.stringify(t,null,2),"utf8"),await Ne({title:`${t.seed} scrape`,content:JSON.stringify(t),source:`mcp-scraper:${i}`}),i}async writeCSV(t,n){await K.mkdir(n,{recursive:!0});let s=(t[0]?.seed_query??"paa").toLowerCase().replace(/\W+/g,"-").slice(0,40),i=dt.unparse(t.map(({sources:U,...u})=>({...u,sources_json:JSON.stringify(U)})),{header:!0}),d=`${s}-${Date.now()}.csv`,l=nt.join(n,d);return await K.writeFile(l,i,"utf8"),l}async writeVideoCSV(t,n,a){await K.mkdir(a,{recursive:!0});let s=n.toLowerCase().replace(/\W+/g,"-").slice(0,40),i=dt.unparse(t,{header:!0}),d=`${s}-videos-${Date.now()}.csv`,l=nt.join(a,d);return await K.writeFile(l,i,"utf8"),l}async writeForumCSV(t,n,a){await K.mkdir(a,{recursive:!0});let s=n.toLowerCase().replace(/\W+/g,"-").slice(0,40),i=dt.unparse(t,{header:!0}),d=`${s}-forums-${Date.now()}.csv`,l=nt.join(a,d);return await K.writeFile(l,i,"utf8"),l}async writeAIOverviewCSV(t,n,a,s){await K.mkdir(s,{recursive:!0});let i=a.toLowerCase().replace(/\W+/g,"-").slice(0,40),d=t.map((S,o)=>({seed_query:a,response_text:o===0?n??"":"",citation_text:S.text,citation_href:S.href})),l=dt.unparse(d,{header:!0}),U=`${i}-ai-overview-${Date.now()}.csv`,u=nt.join(s,U);return await K.writeFile(u,l,"utf8"),u}async writeAIModeCSV(t,n,a,s){await K.mkdir(s,{recursive:!0});let i=a.toLowerCase().replace(/\W+/g,"-").slice(0,40),d=t.map((S,o)=>({seed_query:a,response_text:o===0?n??"":"",citation_text:S.text,citation_href:S.href})),l=dt.unparse(d,{header:!0}),U=`${i}-ai-mode-${Date.now()}.csv`,u=nt.join(s,U);return await K.writeFile(u,l,"utf8"),u}async writeWhatPeopleSayingCSV(t,n,a){await K.mkdir(a,{recursive:!0});let s=n.toLowerCase().replace(/\W+/g,"-").slice(0,40),i=t.map(u=>({seed_query:n,...u})),d=dt.unparse(i,{header:!0}),l=`${s}-what-people-saying-${Date.now()}.csv`,U=nt.join(a,l);return await K.writeFile(U,d,"utf8"),U}};var Ht=class{onQuestion(t){process.stdout.write(JSON.stringify({event:"question",depth:t.depth,question:t.question})+`
 `)}onDepth(t){process.stdout.write(JSON.stringify({event:"depth",depth:t})+`
 `)}onVideos(t){for(let n of t)process.stdout.write(JSON.stringify({event:"video",type:n.type,platform:n.platform,duration:n.duration,title:n.title,channel:n.channel,url:n.url})+`
 `)}onForums(t){for(let n of t)process.stdout.write(JSON.stringify({event:"forum",title:n.title,source:n.source,url:n.url})+`
 `)}onComplete(t){process.stdout.write(JSON.stringify({event:"complete",...t})+`
 `)}onError(t){process.stderr.write(JSON.stringify({event:"error",type:t.constructor.name,message:t.message})+`
-`)}};import{randomInt as ra}from"crypto";var la="pr.oxylabs.io",Sa=7777,Ua=10,$i=2;function ln(){return process.env.OXYLABS_USERNAME?.trim()||void 0}function Sn(){return process.env.OXYLABS_PASSWORD?.trim()||void 0}function ha(){return(process.env.OXYLABS_COUNTRY?.trim()||"US").toUpperCase()}function Ft(){return!!(ln()&&Sn())}function ua(){let e="";for(let t=0;t<10;t++)e+=String(ra(0,10));return e}function ca(e={}){let t=ln();if(!t)return;let n=t.startsWith("customer-")?t:`customer-${t}`,a=(e.country?.trim()||ha()).toUpperCase(),s=e.sticky?`-sessid-${ua()}-sesstime-${Ua}`:"";return`${n}-cc-${a}${s}`}function ee(e){return e.replace(/pr\.oxylabs\.io(:\d+)?/gi,"backup proxy").replace(/oxylabs/gi,"backup proxy")}function wa(e){return`mcp-backup-custom-${`${Date.now()}-${e??0}-${Math.random().toString(36).slice(2,8)}`}`}async function Rt(e,t={}){let n=ca({sticky:t.sticky}),a=Sn();if(!n||!a)throw new Error("backup proxy pool is not configured");let s=new q({apiKey:e});try{let i=await s.proxies.create({type:"custom",name:wa(t.attemptIndex),protocol:"http",config:{host:la,port:Sa,username:n,password:a}});if(!i.id)throw new Error("backup proxy creation did not return a proxy id");return i.id}catch(i){let d=i instanceof Error?i.message:String(i);throw new Error(ee(d))}}async function ji(e,t){if(!(!e||!Ft()))try{return await Rt(e,{attemptIndex:t})}catch{return}}async function Qi(e,t){if(!(!e||!t))try{await new q({apiKey:e}).proxies.delete(t)}catch{}}var ga={alabama:"AL",alaska:"AK",arizona:"AZ",arkansas:"AR",california:"CA",colorado:"CO",connecticut:"CT",delaware:"DE",florida:"FL",georgia:"GA",hawaii:"HI",idaho:"ID",illinois:"IL",indiana:"IN",iowa:"IA",kansas:"KS",kentucky:"KY",louisiana:"LA",maine:"ME",maryland:"MD",massachusetts:"MA",michigan:"MI",minnesota:"MN",mississippi:"MS",missouri:"MO",montana:"MT",nebraska:"NE",nevada:"NV","new hampshire":"NH","new jersey":"NJ","new mexico":"NM","new york":"NY","north carolina":"NC","north dakota":"ND",ohio:"OH",oklahoma:"OK",oregon:"OR",pennsylvania:"PA","rhode island":"RI","south carolina":"SC","south dakota":"SD",tennessee:"TN",texas:"TX",utah:"UT",vermont:"VT",virginia:"VA",washington:"WA","west virginia":"WV",wisconsin:"WI",wyoming:"WY"},ya={"atlanta|GA":"30303","austin|TX":"78701","baltimore|MD":"21201","boston|MA":"02108","boulder|CO":"80302","charlotte|NC":"28202","chicago|IL":"60601","colorado_springs|CO":"80903","columbus|OH":"43215","dallas|TX":"75201","denver|CO":"80202","detroit|MI":"48226","fort_collins|CO":"80524","fort_worth|TX":"76102","houston|TX":"77002","indianapolis|IN":"46204","jacksonville|FL":"32202","las_vegas|NV":"89101","los_angeles|CA":"90012","louisville|KY":"40202","loveland|CO":"80537","memphis|TN":"38103","miami|FL":"33131","minneapolis|MN":"55401","nashville|TN":"37203","new_york|NY":"10001","orlando|FL":"32801","philadelphia|PA":"19103","phoenix|AZ":"85004","portland|OR":"97205","raleigh|NC":"27601","richmond|VA":"23219","sacramento|CA":"95814","salt_lake_city|UT":"84101","san_antonio|TX":"78205","san_diego|CA":"92101","san_francisco|CA":"94103","san_jose|CA":"95113","seattle|WA":"98101"};function Ca(e){return e?e.slice(-6):null}function V(e,t,n,a,s,i=!1){return{kernelProxyId:n,...i&&n?{disposableProxyId:n}:{},resolution:{source:e,proxyType:e==="disabled"?"none":e==="configured_fallback"?"configured":e==="isp_created"?"isp":e==="backup_custom"?"custom":e==="location_created"||e==="location_reused"?"residential":"unknown",proxyMode:t,proxyIdPresent:!!n,proxyIdSuffix:Ca(n),target:a,error:s,disposable:i}}}function Ma(e){return e.trim().toLowerCase().replace(/\s+/g," ")}function ka(e){return e.trim().toLowerCase().replace(/\./g,"").replace(/\s+/g," ")}function pa(e){if(!e)return!0;let t=ka(e);return t==="united states"||t==="united states of america"||t==="usa"||t==="us"}function Un(e){let t=e.trim();return/^[A-Za-z]{2}$/.test(t)?t.toUpperCase():ga[Ma(t)]??null}function ma(e){let n=e.normalize("NFKD").replace(/[^\x00-\x7F]/g,"").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean),a=n.join("_"),s=n.join("");return Array.from(new Set([a,s].filter(Boolean)))}function Z(e,t,n){return n?`mcp-serp-residential-${e.toLowerCase()}-${t.toLowerCase()}-${n}`:`mcp-serp-residential-${e.toLowerCase()}-${t.toLowerCase()}`}function Gt(e,t){let n=`${Date.now()}-${t??0}-${Math.random().toString(36).slice(2,8)}`;return`${e}-fresh-${n}`}function va(e){return`mcp-serp-residential-us-zip-${e}`}function Na(e,t){if(!e||t.toLowerCase()!=="us")return null;let n=Je(e),a=n.split(",").map(u=>u.trim()).filter(Boolean);if(a.length>1&&pa(a[a.length-1])&&(a=a.slice(0,-1)),a.length===1){let u=Un(a[0]);return u?{canonicalLocation:n,level:"state",country:"US",state:u,city:"",cityCandidates:[],proxyName:Z("US",u),config:{country:"US",state:u}}:null}let[s="",i=""]=a;if(!s||!i)return null;let d=Un(i);if(!d)return null;let l=ma(s),U=l[0];return U?{canonicalLocation:n,level:"city",country:"US",state:d,city:U,cityCandidates:l,proxyName:Z("US",d,U),config:{country:"US",state:d,city:U}}:null}function Ta(e){return`${e.city}|${e.state}`}function hn(e,t){return t&&/^\d{5}$/.test(t)?t:ya[Ta(e)]??null}function un(e,t){return{...e,level:"zip",zip:t,proxyName:va(t),config:{country:e.country,state:e.state,zip:t}}}function ne(e,t){return{...e,proxyName:t}}function ie(e,t,n){return t.level==="zip"?e?.country?.toUpperCase()===t.country&&e?.zip===t.zip:e?.country?.toUpperCase()===t.country&&e?.state?.toUpperCase()===t.state&&(n?e?.city===n:!e?.city)}function fa(e,t){return e.find(n=>n.type==="residential"&&n.status!=="unavailable"&&!!n.id&&(n.name===t.proxyName||ie(n.config,t,t.level==="city"?t.city:void 0)))??null}function Pa(e,t){for(let n of t.cityCandidates){let a=Z(t.country,t.state,n),s=e.find(i=>i.type==="residential"&&i.status!=="unavailable"&&!!i.id&&(i.name===a||ie(i.config,t,n)));if(s)return s}return null}function Dt(e){return{...e,level:"state",proxyName:Z(e.country,e.state),config:{country:e.country,state:e.state}}}function ba(e,t){let n=Z(t.country,t.state);return e.find(a=>a.type==="residential"&&a.status!=="unavailable"&&!!a.id&&(a.name===n||ie(a.config,t)))??null}function Wa(e,t){return Dt(e)}function Q(e){return e instanceof Error?e.message:String(e)}function Ia(e,t,n){let a=[],s=hn(e,t);if(s){let d=un(e,s);a.push(ne(d,Gt(d.proxyName,n)))}for(let d of e.cityCandidates){let l={...e,level:"city",city:d,proxyName:Z(e.country,e.state,d),config:{country:e.country,state:e.state,city:d}};a.push(ne(l,Gt(l.proxyName,n)))}let i=Dt(e);return a.push(ne(i,Gt(i.proxyName,n))),a}async function ae(e,t,n){if(e.backupOnExhaustion&&n.length>0&&e.kernelApiKey&&Ft())try{let a=await Rt(e.kernelApiKey,{attemptIndex:e.attemptIndex});return V("backup_custom",e.proxyMode,a,t,null,!0)}catch(a){n.push(`backup: ${ee(Q(a))}`)}return V("configured_fallback",e.proxyMode,e.configuredKernelProxyId,t,n.join(" | "))}async function ns(e){if(!e.kernelApiKey||!Ft())return null;try{let t=await Rt(e.kernelApiKey,{attemptIndex:e.attemptIndex});return V("backup_custom",e.proxyMode,t,null,null,!0)}catch{return null}}async function Aa(e,t,n){let a=[];for(let s of Ia(n,t.proxyZip,t.attemptIndex))try{let i=await e.proxies.create({type:"residential",name:s.proxyName,config:s.level==="zip"?{country:s.country,zip:s.zip}:s.config});if(i.id)return V("location_created",t.proxyMode,i.id,s,null,!0);a.push(`${s.proxyName}: Kernel did not return a proxy id`)}catch(i){a.push(`${s.proxyName}: ${Q(i)}`)}if(process.env.KERNEL_PROXY_TYPE!=="residential"){let s=Dt(n);try{let i=await e.proxies.create({type:"isp",name:Gt(`mcp-serp-isp-${s.country.toLowerCase()}-${(s.state??"").toLowerCase()}`,t.attemptIndex),config:{country:s.country,...s.state?{state:s.state}:{}}});if(i.id)return V("isp_created",t.proxyMode,i.id,s,null,!0);a.push("isp: Kernel did not return a proxy id")}catch(i){a.push(`isp: ${Q(i)}`)}}return ae(t,n,a)}async function cn(e,t){if(!e||!t)return;await new q({apiKey:e}).proxies.delete(t)}async function wn(e){if(e.proxyMode==="none")return V("disabled",e.proxyMode,void 0,null,null);if(e.proxyMode==="configured")return V("configured_fallback",e.proxyMode,e.configuredKernelProxyId,null,null);let t=Na(e.location,e.gl);if(!t||!e.kernelApiKey)return V("configured_fallback",e.proxyMode,e.configuredKernelProxyId,t,t?null:"location could not be normalized to a US city/state proxy target");let n=new q({apiKey:e.kernelApiKey});try{let a=e.attemptIndex??0;if(e.fresh)return await Aa(n,e,t);if(a>=1){let S=Wa(t,a),o=[];try{let r=await n.proxies.create({type:"residential",name:S.proxyName,config:S.config});if(r.id)return V("location_created",e.proxyMode,r.id,S,null);o.push(`${S.state}: Kernel did not return a proxy id`)}catch(r){o.push(`${S.state}: ${Q(r)}`)}return ae(e,S,o)}let s=[];for await(let S of await n.proxies.list())s.push(S);let i=hn(t,e.proxyZip),d=[];if(i){let S=un(t,i),o=fa(s,S);if(o?.id)return V("location_reused",e.proxyMode,o.id,S,null);try{let r=await n.proxies.create({type:"residential",name:S.proxyName,config:{country:S.country,zip:i}});if(r.id)return V("location_created",e.proxyMode,r.id,S,null);d.push(`${i}: Kernel did not return a proxy id`)}catch(r){d.push(`${i}: ${Q(r)}`)}}let l=Pa(s,t);if(l?.id)return V("location_reused",e.proxyMode,l.id,t,d.join(" | ")||null);for(let S of t.cityCandidates)try{let o=await n.proxies.create({type:"residential",name:Z(t.country,t.state,S),config:{country:t.country,state:t.state,city:S}});if(o.id)return V("location_created",e.proxyMode,o.id,{...t,level:"city",city:S,proxyName:Z(t.country,t.state,S),config:{country:t.country,state:t.state,city:S}},null);d.push(`${S}: Kernel did not return a proxy id`)}catch(o){d.push(`${S}: ${Q(o)}`)}let U=Dt(t),u=ba(s,U);if(u?.id)return V("location_reused",e.proxyMode,u.id,U,d.join(" | "));try{let S=await n.proxies.create({type:"residential",name:U.proxyName,config:U.config});if(S.id)return V("location_created",e.proxyMode,S.id,U,d.join(" | "));d.push(`${U.state}: Kernel did not return a proxy id`)}catch(S){d.push(`${U.state}: ${Q(S)}`)}return ae(e,t,d)}catch(a){return V("configured_fallback",e.proxyMode,e.configuredKernelProxyId,t,Q(a))}}var La=3,Oa=5;function oe(e){return e.reason instanceof DOMException&&e.reason.name==="TimeoutError"?e.reason:new _}function xa(e){if(!e||typeof e!="object")return;let t=e.signal;if(t instanceof AbortSignal)return t}function Ba(e){if(!e||typeof e!="object")return;let t=e.onAttemptEvent;return typeof t=="function"?t:void 0}function Ha(e){if(!e||typeof e!="object")return;let t=e.onPaaProgress;return typeof t=="function"?t:void 0}async function rt(e,t){if(e)try{await e(t)}catch(n){console.warn(JSON.stringify({event:"harvest_attempt_log_failed",attempt_number:t.attemptNumber,message:n instanceof Error?n.message:String(n)}))}}function de(e){if(e instanceof E)return"captcha";if(e instanceof Kt)return"location_mismatch";if(e instanceof _)return"request_aborted";if(e instanceof ht)return"browser_session_interrupted";if(e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError"))return"timeout";let t=e instanceof Error?e.message:String(e);return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(t)?"browser_session_interrupted":Va(t)?"proxy_tunnel_failed":Ya(t)?"proxy_unavailable":/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)?"timeout":"error"}function Fa(e){return e.diagnostics.degradedResult?"degraded_result":e.diagnostics?.completionStatus??(e.totalQuestions>0?"paa_found":"no_paa")}function Ra(e){return e.flat.length>0||e.organicResults.length>0||e.localPack.length>0||e.videos.length>0||e.forums.length>0||e.whatPeopleSaying.length>0||e.aiOverview.detected||e.aiMode.detected}function gn(e,t,n){let a=Ra(e),s=(e.diagnostics.warnings?.length??0)>0,i=new Set(e.flat.map(C=>C.question.toLowerCase().replace(/[^\w\s]/g,"").replace(/\s+/g," ").trim())).size,d=n?0:e.flat.filter(C=>!C.answer?.trim()).length,l=n?0:e.flat.filter(C=>!C.source_cite?.trim()).length,U=e.diagnostics.completionStatus==="no_paa"&&(e.diagnostics.noPaaObserved===!0||a),u=e.diagnostics.completionStatus==="paa_found",S=e.diagnostics.paaLifecycle?.discoveryStatus,o=S==="interrupted"||S==="recovery_exhausted",r=u&&(i<t&&S!=="frontier_exhausted"||d>0||l>0||!!e.diagnostics.interruption||o),h=U?"complete":u&&i>0?r||s?"partial":"complete":a?s?"partial":"complete":"degraded",c=h==="degraded";return e.diagnostics={...e.diagnostics,resultQuality:h,degradedResult:c,degradationReasons:c?["empty_primary_serp"]:[],retryRecommended:c||h==="partial"&&(r||e.diagnostics.warnings?.some(C=>C.retryable)===!0),completeness:{paaWithoutAnswer:d,paaWithoutSource:l,paaAnswersRecovered:e.diagnostics.completeness?.paaAnswersRecovered??0,aioShareCaptured:e.diagnostics.completeness?.aioShareCaptured??null,paaRequested:t,paaReturned:e.flat.length,paaUnique:i},...e.diagnostics.paaLifecycle?{paaLifecycle:{...e.diagnostics.paaLifecycle,questionsFound:i,targetStatus:i>=t?"met":"not_met",materialStatus:d>0||l>0?"partial":"complete"}}:{}},e}function Ga(e){return(e.diagnostics.degradationReasons??[]).includes("empty_primary_serp")?"No usable primary SERP surface was captured; the result may be incomplete.":"The harvest result was marked degraded."}function z(e){return e instanceof Error?e.message:String(e)}function Da(e){return e==="location"?Oa:La}function Va(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function Ya(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function yn(e){return e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"}function Cn(e){return e==="browser_session_interrupted"}function Ea(e){let t=e.diagnostics.debug?.locationEvidence,n=t?.expected?.canonicalLocation??e.location??"requested location",a=t?.candidates.slice(0,3).map(s=>`${s.city}, ${s.regionCode}`).join("; ");return a?`Google returned results for ${a}, not ${n}`:`Google returned results for a different location than ${n}`}function Ka(e,t){return t==="location"&&e.diagnostics.debug?.locationEvidence?.status==="mismatch"}function Mn(e,t){if(t||!e.diagnostics.debug)return e;let n={...e.diagnostics};return delete n.debug,{...e,diagnostics:n}}async function kn(e){try{return await an(e,{totalBudgetMs:1e4})}catch(t){return console.warn(JSON.stringify({event:"harvest_goto_resolution_failed",message:z(t)})),e}}async function se(e,t){if(!(!e||!t))try{await cn(e,t),console.info(JSON.stringify({event:"kernel_proxy_deleted",proxy_id_suffix:t.slice(-6)}))}catch(n){console.warn(JSON.stringify({event:"kernel_proxy_delete_failed",proxy_id_suffix:t.slice(-6),message:z(n)}))}}async function pn(e,t,n=!1,a,s){let i=Ge(e,{forceManaged:n}),d=new Ht,l=new xt(i,d,a,s);if(t?.aborted)return{result:null,error:oe(t),cleanup:await i.close(),debug:null};let U,u=t?new Promise((c,C)=>{U=()=>C(oe(t)),t.addEventListener("abort",U,{once:!0})}):null,S=null,o=null,r,h=null;try{let c=l.extract(e,t);u&&c.catch(()=>{}),S=await(u?Promise.race([c,u]):c)}catch(c){o=c}finally{t&&U&&t.removeEventListener("abort",U),h=S?.diagnostics.debug??(e.debug?{enabled:!0,request:{query:e.query,locationInput:e.location??null,canonicalLocation:null,locationResolved:null,locationMatchSource:null,locationAmbiguousCandidates:null,uule:null,gl:e.gl,hl:e.hl,device:e.device,proxyMode:e.proxyMode,proxyZip:e.proxyZip??null,serpIdentity:e.serpIdentity??null,serpOnly:e.serpOnly,pages:e.pages??1},browser:i.getDebugSnapshot()}:null),r=await i.close()}if(o){let c=de(o),C=c==="timeout"||c==="request_aborted"||c==="browser_session_interrupted"?c:null,O=null;if(C)try{O=typeof l.getPartialResult=="function"?l.getPartialResult():null}catch{O=null}return O&&C?(O.diagnostics={...O.diagnostics,interruption:{code:C,reasonSource:"client_runtime",message:z(o)}},{result:O,error:o,cleanup:r,debug:h,salvagedFrom:o}):{result:null,error:o,cleanup:r,debug:h}}return{result:S,error:null,cleanup:r,debug:h}}async function ws(e){let t=typeof e=="object"&&e!==null?e:{},n=xa(e),a=Ba(e),s=Ha(e),i=t.forceManagedSerp===!0,d=t.proxyMode,l=d==="location"||d==="none"||d==="configured"?d:ce,U=typeof t.kernelApiKey=="string"?t.kernelApiKey.trim():ye(),u=typeof t.kernelProxyId=="string"?t.kernelProxyId.trim():void 0,S={kernelApiKey:U,proxyMode:l,configuredKernelProxyId:u,location:typeof t.location=="string"?t.location:void 0,proxyZip:typeof t.proxyZip=="string"?t.proxyZip:void 0,gl:typeof t.gl=="string"?t.gl:"us"},o=typeof t.debug=="boolean"?t.debug:!1,r=l==="location"&&!!S.location,h=t.maxAttempts,C=typeof h=="number"&&Number.isInteger(h)&&h>=1?h:Da(l)+1,O=typeof t.softDeadlineMs=="number"?t.softDeadlineMs:null,P=new Bt,W=null,b=l,x=0;for(let m=0;m<C;m++){let k=m+1,A=Date.now();try{if(n?.aborted)throw oe(n);let v=await wn({...S,proxyMode:b,attemptIndex:m,fresh:b==="location"});if(!v)break;let N={...t,kernelApiKey:U,kernelProxyId:v.kernelProxyId,kernelProxyResolution:v.resolution,proxyMode:b,keepDefaultProxy:m>0,debug:o||r};b==="none"&&(N.kernelProxyId=void 0);let g=we.parse(N);await rt(a,{type:"started",attemptNumber:k,maxAttempts:C,query:g.query,location:g.location??null,maxQuestions:g.maxQuestions,startedAt:new Date(A).toISOString()}),console.info(JSON.stringify({event:"harvest_attempt_started",attempt_number:k,max_attempts:C,query:g.query,location:g.location??null,max_questions:g.maxQuestions}));let p=Nt(),w=F=>rt(a,{type:"provider_session_started",attemptNumber:k,maxAttempts:C,provider:F.provider,providerSessionId:F.providerSessionId,observedAt:new Date().toISOString()}),T=p?.harvestBrowserMode==="headless-test"&&process.env[ke]==="true",y=p?await pe({...p,forceHeadless:T,forceHeadful:!T},()=>pn(g,n,i,s,w)):await pn(g,n,i,s,w);if(y.error){let F=y.error,G=de(F);(G==="captcha"||G==="proxy_tunnel_failed")&&b==="location"&&(x++,x>=2&&(b="none"));let J=m<C-1&&(G==="captcha"||Cn(G)||l==="location"&&yn(G));if(G==="captcha"?console.warn(JSON.stringify({event:"harvest_attempt_captcha",attempt_number:k,max_attempts:C,message:z(F),will_retry:J})):J&&console.warn(JSON.stringify({event:"harvest_attempt_proxy_retry",attempt_number:k,max_attempts:C,outcome:G,message:z(F),will_retry:J})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:G,kernelSessionId:y.cleanup.kernelSessionId,questionCount:y.result?.totalQuestions??0,durationMs:Date.now()-A,error:z(F),willRetry:J,cleanup:y.cleanup,debug:y.debug,completedAt:new Date().toISOString()}),await se(U,v.disposableProxyId),W=F,J)continue;if(y.result){let B=await kn(y.result);return gn(Mn(B,o),g.maxQuestions,g.questionsOnly)}break}let M=y.result;if(!M)throw new Error("Harvest attempt completed without a result");if(Ka(M,l)){let F=new Kt(Ea(M)),G=m<C-1;if(console.warn(JSON.stringify({event:"harvest_attempt_location_mismatch",attempt_number:k,max_attempts:C,message:F.message,will_retry:G})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:"location_mismatch",kernelSessionId:y.cleanup.kernelSessionId,questionCount:M.totalQuestions,durationMs:Date.now()-A,error:F.message,willRetry:G,cleanup:y.cleanup,debug:y.debug,completedAt:new Date().toISOString()}),await se(U,v.disposableProxyId),W=F,G)continue;break}let L=await kn(M),f=gn(Mn(L,o),g.maxQuestions,g.questionsOnly),I=Fa(f),R=I==="degraded_result"&&m<C-1,Y=I==="degraded_result"?Ga(f):null;if(R&&console.warn(JSON.stringify({event:"harvest_attempt_degraded_result",attempt_number:k,max_attempts:C,degradation_reasons:f.diagnostics.degradationReasons,will_retry:!0})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:I,kernelSessionId:y.cleanup.kernelSessionId,questionCount:f.totalQuestions,durationMs:Date.now()-A,error:Y,willRetry:R,cleanup:y.cleanup,debug:y.debug,completedAt:new Date().toISOString()}),await se(U,v.disposableProxyId),R)continue;return(g.format==="json"||g.format==="both")&&await P.writeJSON(f,g.outputDir),(g.format==="csv"||g.format==="both")&&await Promise.all([P.writeCSV(f.flat,g.outputDir),f.videos.length>0?P.writeVideoCSV(f.videos,f.seed,g.outputDir):Promise.resolve(""),f.forums.length>0?P.writeForumCSV(f.forums,f.seed,g.outputDir):Promise.resolve(""),f.aiOverview.detected?P.writeAIOverviewCSV(f.aiOverview.citations,f.aiOverview.text,f.seed,g.outputDir):Promise.resolve(""),f.aiMode.detected?P.writeAIModeCSV(f.aiMode.citations,f.aiMode.text,f.seed,g.outputDir):Promise.resolve(""),f.whatPeopleSaying.length>0?P.writeWhatPeopleSayingCSV(f.whatPeopleSaying,f.seed,g.outputDir):Promise.resolve("")]),f}catch(v){let N=de(v);(N==="captcha"||N==="proxy_tunnel_failed")&&b==="location"&&(x++,x>=2&&(b="none"));let g=m<C-1&&(N==="captcha"||Cn(N)||l==="location"&&yn(N));if(N==="captcha"?console.warn(JSON.stringify({event:"harvest_attempt_captcha",attempt_number:k,max_attempts:C,message:z(v),will_retry:g})):g&&console.warn(JSON.stringify({event:"harvest_attempt_proxy_retry",attempt_number:k,max_attempts:C,outcome:N,message:z(v),will_retry:g})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:N,kernelSessionId:null,questionCount:0,durationMs:Date.now()-A,error:z(v),willRetry:g,cleanup:{kernelSessionId:null,kernelDeleteStarted:!1,kernelDeleteSucceeded:null,kernelDeleteError:null,browserCloseSucceeded:null,browserCloseError:null},debug:null,completedAt:new Date().toISOString()}),W=v,g)continue;if(N==="captcha")break;throw v}}throw W&&!(W instanceof E)?W:(console.warn(JSON.stringify({event:"harvest_captcha_exhausted",max_attempts:C,session_kind:U?"kernel":"local"})),new E(Me(`CAPTCHA on all ${C} fresh sessions. Try again in a few minutes.`)))}export{_a as a,Pt as b,bt as c,ja as d,Le as e,Bn as f,It as g,xe as h,Be as i,Fe as j,Re as k,et as l,Ot as m,$i as n,Ft as o,ji as p,Qi as q,ns as r,cn as s,wn as t,ws as u};
+`)}};import{randomInt as ra}from"crypto";var la="pr.oxylabs.io",Sa=7777,Ua=10,$i=2;function ln(){return process.env.OXYLABS_USERNAME?.trim()||void 0}function Sn(){return process.env.OXYLABS_PASSWORD?.trim()||void 0}function ha(){return(process.env.OXYLABS_COUNTRY?.trim()||"US").toUpperCase()}function Ft(){return!!(ln()&&Sn())}function ua(){let e="";for(let t=0;t<10;t++)e+=String(ra(0,10));return e}function ca(e={}){let t=ln();if(!t)return;let n=t.startsWith("customer-")?t:`customer-${t}`,a=(e.country?.trim()||ha()).toUpperCase(),s=e.sticky?`-sessid-${ua()}-sesstime-${Ua}`:"";return`${n}-cc-${a}${s}`}function ee(e){return e.replace(/pr\.oxylabs\.io(:\d+)?/gi,"backup proxy").replace(/oxylabs/gi,"backup proxy")}function wa(e){return`mcp-backup-custom-${`${Date.now()}-${e??0}-${Math.random().toString(36).slice(2,8)}`}`}async function Rt(e,t={}){let n=ca({sticky:t.sticky}),a=Sn();if(!n||!a)throw new Error("backup proxy pool is not configured");let s=new q({apiKey:e});try{let i=await s.proxies.create({type:"custom",name:wa(t.attemptIndex),protocol:"http",config:{host:la,port:Sa,username:n,password:a}});if(!i.id)throw new Error("backup proxy creation did not return a proxy id");return i.id}catch(i){let d=i instanceof Error?i.message:String(i);throw new Error(ee(d))}}async function ji(e,t){if(!(!e||!Ft()))try{return await Rt(e,{attemptIndex:t})}catch{return}}async function Qi(e,t){if(!(!e||!t))try{await new q({apiKey:e}).proxies.delete(t)}catch{}}var ga={alabama:"AL",alaska:"AK",arizona:"AZ",arkansas:"AR",california:"CA",colorado:"CO",connecticut:"CT",delaware:"DE",florida:"FL",georgia:"GA",hawaii:"HI",idaho:"ID",illinois:"IL",indiana:"IN",iowa:"IA",kansas:"KS",kentucky:"KY",louisiana:"LA",maine:"ME",maryland:"MD",massachusetts:"MA",michigan:"MI",minnesota:"MN",mississippi:"MS",missouri:"MO",montana:"MT",nebraska:"NE",nevada:"NV","new hampshire":"NH","new jersey":"NJ","new mexico":"NM","new york":"NY","north carolina":"NC","north dakota":"ND",ohio:"OH",oklahoma:"OK",oregon:"OR",pennsylvania:"PA","rhode island":"RI","south carolina":"SC","south dakota":"SD",tennessee:"TN",texas:"TX",utah:"UT",vermont:"VT",virginia:"VA",washington:"WA","west virginia":"WV",wisconsin:"WI",wyoming:"WY"},ya={"atlanta|GA":"30303","austin|TX":"78701","baltimore|MD":"21201","boston|MA":"02108","boulder|CO":"80302","charlotte|NC":"28202","chicago|IL":"60601","colorado_springs|CO":"80903","columbus|OH":"43215","dallas|TX":"75201","denver|CO":"80202","detroit|MI":"48226","fort_collins|CO":"80524","fort_worth|TX":"76102","houston|TX":"77002","indianapolis|IN":"46204","jacksonville|FL":"32202","las_vegas|NV":"89101","los_angeles|CA":"90012","louisville|KY":"40202","loveland|CO":"80537","memphis|TN":"38103","miami|FL":"33131","minneapolis|MN":"55401","nashville|TN":"37203","new_york|NY":"10001","orlando|FL":"32801","philadelphia|PA":"19103","phoenix|AZ":"85004","portland|OR":"97205","raleigh|NC":"27601","richmond|VA":"23219","sacramento|CA":"95814","salt_lake_city|UT":"84101","san_antonio|TX":"78205","san_diego|CA":"92101","san_francisco|CA":"94103","san_jose|CA":"95113","seattle|WA":"98101"};function Ca(e){return e?e.slice(-6):null}function V(e,t,n,a,s,i=!1){return{\u006b\u0065\u0072\u006e\u0065\u006cProxyId:n,...i&&n?{disposableProxyId:n}:{},resolution:{source:e,proxyType:e==="disabled"?"none":e==="configured_fallback"?"configured":e==="isp_created"?"isp":e==="backup_custom"?"custom":e==="location_created"||e==="location_reused"?"residential":"unknown",proxyMode:t,proxyIdPresent:!!n,proxyIdSuffix:Ca(n),target:a,error:s,disposable:i}}}function Ma(e){return e.trim().toLowerCase().replace(/\s+/g," ")}function ka(e){return e.trim().toLowerCase().replace(/\./g,"").replace(/\s+/g," ")}function pa(e){if(!e)return!0;let t=ka(e);return t==="united states"||t==="united states of america"||t==="usa"||t==="us"}function Un(e){let t=e.trim();return/^[A-Za-z]{2}$/.test(t)?t.toUpperCase():ga[Ma(t)]??null}function ma(e){let n=e.normalize("NFKD").replace(/[^\x00-\x7F]/g,"").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean),a=n.join("_"),s=n.join("");return Array.from(new Set([a,s].filter(Boolean)))}function Z(e,t,n){return n?`mcp-serp-residential-${e.toLowerCase()}-${t.toLowerCase()}-${n}`:`mcp-serp-residential-${e.toLowerCase()}-${t.toLowerCase()}`}function Gt(e,t){let n=`${Date.now()}-${t??0}-${Math.random().toString(36).slice(2,8)}`;return`${e}-fresh-${n}`}function va(e){return`mcp-serp-residential-us-zip-${e}`}function Na(e,t){if(!e||t.toLowerCase()!=="us")return null;let n=Je(e),a=n.split(",").map(u=>u.trim()).filter(Boolean);if(a.length>1&&pa(a[a.length-1])&&(a=a.slice(0,-1)),a.length===1){let u=Un(a[0]);return u?{canonicalLocation:n,level:"state",country:"US",state:u,city:"",cityCandidates:[],proxyName:Z("US",u),config:{country:"US",state:u}}:null}let[s="",i=""]=a;if(!s||!i)return null;let d=Un(i);if(!d)return null;let l=ma(s),U=l[0];return U?{canonicalLocation:n,level:"city",country:"US",state:d,city:U,cityCandidates:l,proxyName:Z("US",d,U),config:{country:"US",state:d,city:U}}:null}function Ta(e){return`${e.city}|${e.state}`}function hn(e,t){return t&&/^\d{5}$/.test(t)?t:ya[Ta(e)]??null}function un(e,t){return{...e,level:"zip",zip:t,proxyName:va(t),config:{country:e.country,state:e.state,zip:t}}}function ne(e,t){return{...e,proxyName:t}}function ie(e,t,n){return t.level==="zip"?e?.country?.toUpperCase()===t.country&&e?.zip===t.zip:e?.country?.toUpperCase()===t.country&&e?.state?.toUpperCase()===t.state&&(n?e?.city===n:!e?.city)}function fa(e,t){return e.find(n=>n.type==="residential"&&n.status!=="unavailable"&&!!n.id&&(n.name===t.proxyName||ie(n.config,t,t.level==="city"?t.city:void 0)))??null}function Pa(e,t){for(let n of t.cityCandidates){let a=Z(t.country,t.state,n),s=e.find(i=>i.type==="residential"&&i.status!=="unavailable"&&!!i.id&&(i.name===a||ie(i.config,t,n)));if(s)return s}return null}function Dt(e){return{...e,level:"state",proxyName:Z(e.country,e.state),config:{country:e.country,state:e.state}}}function ba(e,t){let n=Z(t.country,t.state);return e.find(a=>a.type==="residential"&&a.status!=="unavailable"&&!!a.id&&(a.name===n||ie(a.config,t)))??null}function Wa(e,t){return Dt(e)}function Q(e){return e instanceof Error?e.message:String(e)}function Ia(e,t,n){let a=[],s=hn(e,t);if(s){let d=un(e,s);a.push(ne(d,Gt(d.proxyName,n)))}for(let d of e.cityCandidates){let l={...e,level:"city",city:d,proxyName:Z(e.country,e.state,d),config:{country:e.country,state:e.state,city:d}};a.push(ne(l,Gt(l.proxyName,n)))}let i=Dt(e);return a.push(ne(i,Gt(i.proxyName,n))),a}async function ae(e,t,n){if(e.backupOnExhaustion&&n.length>0&&e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey&&Ft())try{let a=await Rt(e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey,{attemptIndex:e.attemptIndex});return V("backup_custom",e.proxyMode,a,t,null,!0)}catch(a){n.push(`backup: ${ee(Q(a))}`)}return V("configured_fallback",e.proxyMode,e.configured\u004b\u0065\u0072\u006e\u0065\u006cProxyId,t,n.join(" | "))}async function ns(e){if(!e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey||!Ft())return null;try{let t=await Rt(e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey,{attemptIndex:e.attemptIndex});return V("backup_custom",e.proxyMode,t,null,null,!0)}catch{return null}}async function Aa(e,t,n){let a=[];for(let s of Ia(n,t.proxyZip,t.attemptIndex))try{let i=await e.proxies.create({type:"residential",name:s.proxyName,config:s.level==="zip"?{country:s.country,zip:s.zip}:s.config});if(i.id)return V("location_created",t.proxyMode,i.id,s,null,!0);a.push(`${s.proxyName}: \u004b\u0065\u0072\u006e\u0065\u006c did not return a proxy id`)}catch(i){a.push(`${s.proxyName}: ${Q(i)}`)}if(process.env.\u004b\u0045\u0052\u004e\u0045\u004c_PROXY_TYPE!=="residential"){let s=Dt(n);try{let i=await e.proxies.create({type:"isp",name:Gt(`mcp-serp-isp-${s.country.toLowerCase()}-${(s.state??"").toLowerCase()}`,t.attemptIndex),config:{country:s.country,...s.state?{state:s.state}:{}}});if(i.id)return V("isp_created",t.proxyMode,i.id,s,null,!0);a.push("isp: \u004b\u0065\u0072\u006e\u0065\u006c did not return a proxy id")}catch(i){a.push(`isp: ${Q(i)}`)}}return ae(t,n,a)}async function cn(e,t){if(!e||!t)return;await new q({apiKey:e}).proxies.delete(t)}async function wn(e){if(e.proxyMode==="none")return V("disabled",e.proxyMode,void 0,null,null);if(e.proxyMode==="configured")return V("configured_fallback",e.proxyMode,e.configured\u004b\u0065\u0072\u006e\u0065\u006cProxyId,null,null);let t=Na(e.location,e.gl);if(!t||!e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey)return V("configured_fallback",e.proxyMode,e.configured\u004b\u0065\u0072\u006e\u0065\u006cProxyId,t,t?null:"location could not be normalized to a US city/state proxy target");let n=new q({apiKey:e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey});try{let a=e.attemptIndex??0;if(e.fresh)return await Aa(n,e,t);if(a>=1){let S=Wa(t,a),o=[];try{let r=await n.proxies.create({type:"residential",name:S.proxyName,config:S.config});if(r.id)return V("location_created",e.proxyMode,r.id,S,null);o.push(`${S.state}: \u004b\u0065\u0072\u006e\u0065\u006c did not return a proxy id`)}catch(r){o.push(`${S.state}: ${Q(r)}`)}return ae(e,S,o)}let s=[];for await(let S of await n.proxies.list())s.push(S);let i=hn(t,e.proxyZip),d=[];if(i){let S=un(t,i),o=fa(s,S);if(o?.id)return V("location_reused",e.proxyMode,o.id,S,null);try{let r=await n.proxies.create({type:"residential",name:S.proxyName,config:{country:S.country,zip:i}});if(r.id)return V("location_created",e.proxyMode,r.id,S,null);d.push(`${i}: \u004b\u0065\u0072\u006e\u0065\u006c did not return a proxy id`)}catch(r){d.push(`${i}: ${Q(r)}`)}}let l=Pa(s,t);if(l?.id)return V("location_reused",e.proxyMode,l.id,t,d.join(" | ")||null);for(let S of t.cityCandidates)try{let o=await n.proxies.create({type:"residential",name:Z(t.country,t.state,S),config:{country:t.country,state:t.state,city:S}});if(o.id)return V("location_created",e.proxyMode,o.id,{...t,level:"city",city:S,proxyName:Z(t.country,t.state,S),config:{country:t.country,state:t.state,city:S}},null);d.push(`${S}: \u004b\u0065\u0072\u006e\u0065\u006c did not return a proxy id`)}catch(o){d.push(`${S}: ${Q(o)}`)}let U=Dt(t),u=ba(s,U);if(u?.id)return V("location_reused",e.proxyMode,u.id,U,d.join(" | "));try{let S=await n.proxies.create({type:"residential",name:U.proxyName,config:U.config});if(S.id)return V("location_created",e.proxyMode,S.id,U,d.join(" | "));d.push(`${U.state}: \u004b\u0065\u0072\u006e\u0065\u006c did not return a proxy id`)}catch(S){d.push(`${U.state}: ${Q(S)}`)}return ae(e,t,d)}catch(a){return V("configured_fallback",e.proxyMode,e.configured\u004b\u0065\u0072\u006e\u0065\u006cProxyId,t,Q(a))}}var La=3,Oa=5;function oe(e){return e.reason instanceof DOMException&&e.reason.name==="TimeoutError"?e.reason:new _}function xa(e){if(!e||typeof e!="object")return;let t=e.signal;if(t instanceof AbortSignal)return t}function Ba(e){if(!e||typeof e!="object")return;let t=e.onAttemptEvent;return typeof t=="function"?t:void 0}function Ha(e){if(!e||typeof e!="object")return;let t=e.onPaaProgress;return typeof t=="function"?t:void 0}async function rt(e,t){if(e)try{await e(t)}catch(n){console.warn(JSON.stringify({event:"harvest_attempt_log_failed",attempt_number:t.attemptNumber,message:n instanceof Error?n.message:String(n)}))}}function de(e){if(e instanceof E)return"captcha";if(e instanceof Kt)return"location_mismatch";if(e instanceof _)return"request_aborted";if(e instanceof ht)return"browser_session_interrupted";if(e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError"))return"timeout";let t=e instanceof Error?e.message:String(e);return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(t)?"browser_session_interrupted":Va(t)?"proxy_tunnel_failed":Ya(t)?"proxy_unavailable":/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)?"timeout":"error"}function Fa(e){return e.diagnostics.degradedResult?"degraded_result":e.diagnostics?.completionStatus??(e.totalQuestions>0?"paa_found":"no_paa")}function Ra(e){return e.flat.length>0||e.organicResults.length>0||e.localPack.length>0||e.videos.length>0||e.forums.length>0||e.whatPeopleSaying.length>0||e.aiOverview.detected||e.aiMode.detected}function gn(e,t,n){let a=Ra(e),s=(e.diagnostics.warnings?.length??0)>0,i=new Set(e.flat.map(C=>C.question.toLowerCase().replace(/[^\w\s]/g,"").replace(/\s+/g," ").trim())).size,d=n?0:e.flat.filter(C=>!C.answer?.trim()).length,l=n?0:e.flat.filter(C=>!C.source_cite?.trim()).length,U=e.diagnostics.completionStatus==="no_paa"&&(e.diagnostics.noPaaObserved===!0||a),u=e.diagnostics.completionStatus==="paa_found",S=e.diagnostics.paaLifecycle?.discoveryStatus,o=S==="interrupted"||S==="recovery_exhausted",r=u&&(i<t&&S!=="frontier_exhausted"||d>0||l>0||!!e.diagnostics.interruption||o),h=U?"complete":u&&i>0?r||s?"partial":"complete":a?s?"partial":"complete":"degraded",c=h==="degraded";return e.diagnostics={...e.diagnostics,resultQuality:h,degradedResult:c,degradationReasons:c?["empty_primary_serp"]:[],retryRecommended:c||h==="partial"&&(r||e.diagnostics.warnings?.some(C=>C.retryable)===!0),completeness:{paaWithoutAnswer:d,paaWithoutSource:l,paaAnswersRecovered:e.diagnostics.completeness?.paaAnswersRecovered??0,aioShareCaptured:e.diagnostics.completeness?.aioShareCaptured??null,paaRequested:t,paaReturned:e.flat.length,paaUnique:i},...e.diagnostics.paaLifecycle?{paaLifecycle:{...e.diagnostics.paaLifecycle,questionsFound:i,targetStatus:i>=t?"met":"not_met",materialStatus:d>0||l>0?"partial":"complete"}}:{}},e}function Ga(e){return(e.diagnostics.degradationReasons??[]).includes("empty_primary_serp")?"No usable primary SERP surface was captured; the result may be incomplete.":"The harvest result was marked degraded."}function z(e){return e instanceof Error?e.message:String(e)}function Da(e){return e==="location"?Oa:La}function Va(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function Ya(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function yn(e){return e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"}function Cn(e){return e==="browser_session_interrupted"}function Ea(e){let t=e.diagnostics.debug?.locationEvidence,n=t?.expected?.canonicalLocation??e.location??"requested location",a=t?.candidates.slice(0,3).map(s=>`${s.city}, ${s.regionCode}`).join("; ");return a?`Google returned results for ${a}, not ${n}`:`Google returned results for a different location than ${n}`}function Ka(e,t){return t==="location"&&e.diagnostics.debug?.locationEvidence?.status==="mismatch"}function Mn(e,t){if(t||!e.diagnostics.debug)return e;let n={...e.diagnostics};return delete n.debug,{...e,diagnostics:n}}async function kn(e){try{return await an(e,{totalBudgetMs:1e4})}catch(t){return console.warn(JSON.stringify({event:"harvest_goto_resolution_failed",message:z(t)})),e}}async function se(e,t){if(!(!e||!t))try{await cn(e,t),console.info(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_proxy_deleted",proxy_id_suffix:t.slice(-6)}))}catch(n){console.warn(JSON.stringify({event:"\u006b\u0065\u0072\u006e\u0065\u006c_proxy_delete_failed",proxy_id_suffix:t.slice(-6),message:z(n)}))}}async function pn(e,t,n=!1,a,s){let i=Ge(e,{forceManaged:n}),d=new Ht,l=new xt(i,d,a,s);if(t?.aborted)return{result:null,error:oe(t),cleanup:await i.close(),debug:null};let U,u=t?new Promise((c,C)=>{U=()=>C(oe(t)),t.addEventListener("abort",U,{once:!0})}):null,S=null,o=null,r,h=null;try{let c=l.extract(e,t);u&&c.catch(()=>{}),S=await(u?Promise.race([c,u]):c)}catch(c){o=c}finally{t&&U&&t.removeEventListener("abort",U),h=S?.diagnostics.debug??(e.debug?{enabled:!0,request:{query:e.query,locationInput:e.location??null,canonicalLocation:null,locationResolved:null,locationMatchSource:null,locationAmbiguousCandidates:null,uule:null,gl:e.gl,hl:e.hl,device:e.device,proxyMode:e.proxyMode,proxyZip:e.proxyZip??null,serpIdentity:e.serpIdentity??null,serpOnly:e.serpOnly,pages:e.pages??1},browser:i.getDebugSnapshot()}:null),r=await i.close()}if(o){let c=de(o),C=c==="timeout"||c==="request_aborted"||c==="browser_session_interrupted"?c:null,O=null;if(C)try{O=typeof l.getPartialResult=="function"?l.getPartialResult():null}catch{O=null}return O&&C?(O.diagnostics={...O.diagnostics,interruption:{code:C,reasonSource:"client_runtime",message:z(o)}},{result:O,error:o,cleanup:r,debug:h,salvagedFrom:o}):{result:null,error:o,cleanup:r,debug:h}}return{result:S,error:null,cleanup:r,debug:h}}async function ws(e){let t=typeof e=="object"&&e!==null?e:{},n=xa(e),a=Ba(e),s=Ha(e),i=t.forceManagedSerp===!0,d=t.proxyMode,l=d==="location"||d==="none"||d==="configured"?d:ce,U=typeof t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey=="string"?t.\u006b\u0065\u0072\u006e\u0065\u006cApiKey.trim():ye(),u=typeof t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId=="string"?t.\u006b\u0065\u0072\u006e\u0065\u006cProxyId.trim():void 0,S={\u006b\u0065\u0072\u006e\u0065\u006cApiKey:U,proxyMode:l,configured\u004b\u0065\u0072\u006e\u0065\u006cProxyId:u,location:typeof t.location=="string"?t.location:void 0,proxyZip:typeof t.proxyZip=="string"?t.proxyZip:void 0,gl:typeof t.gl=="string"?t.gl:"us"},o=typeof t.debug=="boolean"?t.debug:!1,r=l==="location"&&!!S.location,h=t.maxAttempts,C=typeof h=="number"&&Number.isInteger(h)&&h>=1?h:Da(l)+1,O=typeof t.softDeadlineMs=="number"?t.softDeadlineMs:null,P=new Bt,W=null,b=l,x=0;for(let m=0;m<C;m++){let k=m+1,A=Date.now();try{if(n?.aborted)throw oe(n);let v=await wn({...S,proxyMode:b,attemptIndex:m,fresh:b==="location"});if(!v)break;let N={...t,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:U,\u006b\u0065\u0072\u006e\u0065\u006cProxyId:v.\u006b\u0065\u0072\u006e\u0065\u006cProxyId,\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution:v.resolution,proxyMode:b,keepDefaultProxy:m>0,debug:o||r};b==="none"&&(N.\u006b\u0065\u0072\u006e\u0065\u006cProxyId=void 0);let g=we.parse(N);await rt(a,{type:"started",attemptNumber:k,maxAttempts:C,query:g.query,location:g.location??null,maxQuestions:g.maxQuestions,startedAt:new Date(A).toISOString()}),console.info(JSON.stringify({event:"harvest_attempt_started",attempt_number:k,max_attempts:C,query:g.query,location:g.location??null,max_questions:g.maxQuestions}));let p=Nt(),w=F=>rt(a,{type:"provider_session_started",attemptNumber:k,maxAttempts:C,provider:F.provider,providerSessionId:F.providerSessionId,observedAt:new Date().toISOString()}),T=p?.harvestBrowserMode==="headless-test"&&process.env[ke]==="true",y=p?await pe({...p,forceHeadless:T,forceHeadful:!T},()=>pn(g,n,i,s,w)):await pn(g,n,i,s,w);if(y.error){let F=y.error,G=de(F);(G==="captcha"||G==="proxy_tunnel_failed")&&b==="location"&&(x++,x>=2&&(b="none"));let J=m<C-1&&(G==="captcha"||Cn(G)||l==="location"&&yn(G));if(G==="captcha"?console.warn(JSON.stringify({event:"harvest_attempt_captcha",attempt_number:k,max_attempts:C,message:z(F),will_retry:J})):J&&console.warn(JSON.stringify({event:"harvest_attempt_proxy_retry",attempt_number:k,max_attempts:C,outcome:G,message:z(F),will_retry:J})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:G,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:y.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:y.result?.totalQuestions??0,durationMs:Date.now()-A,error:z(F),willRetry:J,cleanup:y.cleanup,debug:y.debug,completedAt:new Date().toISOString()}),await se(U,v.disposableProxyId),W=F,J)continue;if(y.result){let B=await kn(y.result);return gn(Mn(B,o),g.maxQuestions,g.questionsOnly)}break}let M=y.result;if(!M)throw new Error("Harvest attempt completed without a result");if(Ka(M,l)){let F=new Kt(Ea(M)),G=m<C-1;if(console.warn(JSON.stringify({event:"harvest_attempt_location_mismatch",attempt_number:k,max_attempts:C,message:F.message,will_retry:G})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:"location_mismatch",\u006b\u0065\u0072\u006e\u0065\u006cSessionId:y.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:M.totalQuestions,durationMs:Date.now()-A,error:F.message,willRetry:G,cleanup:y.cleanup,debug:y.debug,completedAt:new Date().toISOString()}),await se(U,v.disposableProxyId),W=F,G)continue;break}let L=await kn(M),f=gn(Mn(L,o),g.maxQuestions,g.questionsOnly),I=Fa(f),R=I==="degraded_result"&&m<C-1,Y=I==="degraded_result"?Ga(f):null;if(R&&console.warn(JSON.stringify({event:"harvest_attempt_degraded_result",attempt_number:k,max_attempts:C,degradation_reasons:f.diagnostics.degradationReasons,will_retry:!0})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:I,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:y.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:f.totalQuestions,durationMs:Date.now()-A,error:Y,willRetry:R,cleanup:y.cleanup,debug:y.debug,completedAt:new Date().toISOString()}),await se(U,v.disposableProxyId),R)continue;return(g.format==="json"||g.format==="both")&&await P.writeJSON(f,g.outputDir),(g.format==="csv"||g.format==="both")&&await Promise.all([P.writeCSV(f.flat,g.outputDir),f.videos.length>0?P.writeVideoCSV(f.videos,f.seed,g.outputDir):Promise.resolve(""),f.forums.length>0?P.writeForumCSV(f.forums,f.seed,g.outputDir):Promise.resolve(""),f.aiOverview.detected?P.writeAIOverviewCSV(f.aiOverview.citations,f.aiOverview.text,f.seed,g.outputDir):Promise.resolve(""),f.aiMode.detected?P.writeAIModeCSV(f.aiMode.citations,f.aiMode.text,f.seed,g.outputDir):Promise.resolve(""),f.whatPeopleSaying.length>0?P.writeWhatPeopleSayingCSV(f.whatPeopleSaying,f.seed,g.outputDir):Promise.resolve("")]),f}catch(v){let N=de(v);(N==="captcha"||N==="proxy_tunnel_failed")&&b==="location"&&(x++,x>=2&&(b="none"));let g=m<C-1&&(N==="captcha"||Cn(N)||l==="location"&&yn(N));if(N==="captcha"?console.warn(JSON.stringify({event:"harvest_attempt_captcha",attempt_number:k,max_attempts:C,message:z(v),will_retry:g})):g&&console.warn(JSON.stringify({event:"harvest_attempt_proxy_retry",attempt_number:k,max_attempts:C,outcome:N,message:z(v),will_retry:g})),await rt(a,{type:"finished",attemptNumber:k,maxAttempts:C,outcome:N,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,questionCount:0,durationMs:Date.now()-A,error:z(v),willRetry:g,cleanup:{\u006b\u0065\u0072\u006e\u0065\u006cSessionId:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:!1,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:null,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:null,browserCloseSucceeded:null,browserCloseError:null},debug:null,completedAt:new Date().toISOString()}),W=v,g)continue;if(N==="captcha")break;throw v}}throw W&&!(W instanceof E)?W:(console.warn(JSON.stringify({event:"harvest_captcha_exhausted",max_attempts:C,session_kind:U?"\u006b\u0065\u0072\u006e\u0065\u006c":"local"})),new E(Me(`CAPTCHA on all ${C} fresh sessions. Try again in a few minutes.`)))}export{_a as a,Pt as b,bt as c,ja as d,Le as e,Bn as f,It as g,xe as h,Be as i,Fe as j,Re as k,et as l,Ot as m,$i as n,Ft as o,ji as p,Qi as q,ns as r,cn as s,wn as t,ws as u};

package/dist/{chunk-57AHYL2O.js → chunk-K7KMLT33.js}

@@ -1,4 +1,4 @@
-import{a as N}from"./chunk-YQZGZBB4.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,s as x}from"./chunk-63T4W2LX.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
+import{a as N}from"./chunk-YQZGZBB4.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,s as x}from"./chunk-Z66Z2HV4.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
 `).matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?([^"')&]+)(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?\s*\)/gi))n(l[1],`static-css-url${i}`,"image");for(let l of d.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi))n(l[1],`static-bare-url${i}`)},c=typeof e=="string"?P(e):e;return a(c),c.$("script").each((d,i)=>{let o=c.$(i).text(),h=o.replace(/\\u003[cC]/g,"<").replace(/\\u003[eE]/g,">").replace(/\\u0026/gi,"&").replace(/\\["']/g,l=>l.slice(1)).replace(/\\\//g,"/");h!==o&&/<(?:img|source|video|audio)\b/i.test(h)&&a(P(h),"-json-unescaped")}),[...r.values()]}function fe(e,t,r=100){let n=new Map;for(let a of e)A(n,a);for(let a of t?.assets??[])A(n,{...a,type:a.type});let g=new Map;for(let a of n.values()){let c=pe(a.url),d=g.get(c)??[];d.push(a),g.set(c,d)}return[...g.values()].map(a=>{let c=[...a].sort((i,o)=>S(o)-S(i)||i.url.localeCompare(o.url)),d={...c[0]};return d.discoveryMethods=[...new Set(a.flatMap(i=>i.discoveryMethods))],d.altTexts=[...new Set(a.flatMap(i=>i.altTexts))],d.contexts=[...new Set(a.flatMap(i=>i.contexts))],{...d,variants:c.slice(1).map(i=>i.url)}}).sort((a,c)=>S(c)-S(a)||a.url.localeCompare(c.url)).slice(0,Math.max(1,r))}async function ye(e,t,r,n={}){let g=Math.max(1,Math.min(n.maxBytes??52428800,104857600)),a=e,c=null;for(let p=0;p<=5;p++){let y=await L(a,{field:"media URL"});if(y.error||!y.parsed)throw new Error(y.error??"Media URL was rejected");if(c=await fetch(y.parsed.href,{signal:AbortSignal.timeout(15e3),redirect:"manual"}),c.status>=300&&c.status<400){let w=c.headers.get("location");if(!w)throw new Error(`HTTP ${c.status} redirect did not include Location`);a=new URL(w,y.parsed.href).href,c=null;continue}break}if(!c)throw new Error("Media download exceeded five redirects");if(!c.ok)throw new Error(`HTTP ${c.status}`);if(!c.body)throw new Error("Empty response body");let d=c.headers.get("content-type")?.split(";")[0].trim()??null;if(n.expectedType&&(!d||q(d)!==n.expectedType))throw new Error(`Expected ${n.expectedType} content but received ${d??"no content-type"}`);let i=Number(c.headers.get("content-length")??0);if(Number.isFinite(i)&&i>g)throw new Error(`Media asset exceeds ${g} byte limit`);let o=I(t,r);if(d&&!k(r)){let y={"image/jpeg":".jpg","image/png":".png","image/webp":".webp","image/gif":".gif","image/svg+xml":".svg","image/avif":".avif","video/mp4":".mp4","video/webm":".webm","audio/mpeg":".mp3","audio/ogg":".ogg","audio/wav":".wav"}[d];y&&(o=o+y)}let h=Z(o);await new Promise((p,y)=>{h.once("open",()=>p()),h.once("error",y)});let l=0,u=new ie({transform(p,y,w){let _=p.length;if(l+_>g){w(new Error(`Media asset exceeds ${g} byte limit`));return}if(n.consumeBytes&&!n.consumeBytes(_)){w(new Error("Media export total byte limit exceeded"));return}l+=_,w(null,p)}});try{await ne(oe.fromWeb(c.body),u,h)}catch(p){throw h.destroy(),te(o,{force:!0}),p}let{statSync:s}=await import("fs"),m=s(o).size;return{savedPath:o,sizeBytes:m,mimeType:d}}async function Le(e,t,r={}){let n=r.types??["image","video","audio"],g=new Set(n),a=he(e,t),c=Math.max(1,Math.min(r.maxAssets??100,250)),d=fe(a,r.rendered,2e3),i=d.slice(0,c),o=new Set([...a.map(f=>f.url),...(r.rendered?.assets??[]).map(f=>f.url)]).size,h=i.flatMap(f=>{let b=f.type??F(f.url);return b&&g.has(b)?[{...f,type:b}]:[]}),l=o-h.length,u=d.length>c,s=r.rendered?.exhausted===!0&&!u,m=u?"asset_limit":r.rendered?.stopReason??"render_unavailable",p=u?[`Media discovery found ${d.length} responsive families; only the requested ${c} records were retained.`]:r.rendered?r.rendered.exhausted?[]:[`Rendered media discovery stopped with ${r.rendered.stopReason}; the inventory may be incomplete.`]:["Rendered media discovery was unavailable; this manifest contains static-source evidence only."],y={pageUrl:t,staticFound:a.length,renderedFound:r.rendered?.assets.length??0,filteredCount:l,totalFound:o,completeness:s?"complete":"partial",exhausted:s,stopReason:m,scrollRounds:r.rendered?.scrollRounds??0,warnings:p,artifact:null};if(r.outputDir===null)return{...y,outputDir:null,assets:h.map((f,b)=>({...f,mimeType:null,filename:U(f.url,b),savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"not_attempted",downloadError:null,inlinePreview:null}))};let w=(()=>{try{return new URL(t).hostname.replace(/^www\./,"")}catch{return"unknown"}})(),_=new Date().toISOString().replace(/[:.]/g,"-").slice(0,19),C=r.outputDir??I(re(),"Downloads","mcp-scraper","media",`${_}-${w}`);ee(C,{recursive:!0});let T=[];return await Promise.allSettled(h.map(async(f,b)=>{let R=U(f.url,b);try{let{savedPath:v,sizeBytes:Q,mimeType:D}=await ye(f.url,C,R),G=D?q(D)??f.type:f.type;T.push({...f,type:G,mimeType:D,filename:B(v),savedPath:v,sizeBytes:Q,sha256:null,downloadStatus:"downloaded",downloadError:null,inlinePreview:null})}catch(v){T.push({...f,mimeType:null,filename:R,savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"failed",downloadError:v instanceof Error?v.message:"media_download_failed",inlinePreview:null})}})),T.sort((f,b)=>f.savedPath&&!b.savedPath?-1:!f.savedPath&&b.savedPath?1:f.url.localeCompare(b.url)),{...y,outputDir:C,assets:T}}var be=new Set(["browser_navigation_blocked","browser_navigation_failed","browser_timeout","browser_session_failed","browser_result_missing"]);function we(e){return e instanceof Error?e.code:void 0}function _e(e){return e instanceof Error?e.message:String(e)}function H(e){let t=e?.match(/^browser_http_(\d+)$/);return t?Number(t[1]):null}function ve(e){return e===404?"page_not_found":e===403?"page_forbidden":e===429?"page_rate_limited":e>=500&&e<=599?"page_server_error":"page_http_error"}function xe(e){let t=_e(e),r=we(e);if(N(e,t))return{errorCode:"vendor_unavailable",httpStatus:null,retryable:!0};let n=H(r)??H(t.match(/^(browser_http_\d+)/)?.[1]);return n!=null?{errorCode:ve(n),httpStatus:n,retryable:n===429||n>=500}:r==="browser_challenge_unresolved"||/^browser_challenge_unresolved/.test(t)?{errorCode:"bot_check_unresolved",httpStatus:null,retryable:!0}:r==="browser_response_too_large"||/exceeds \d+ byte limit/i.test(t)?{errorCode:"page_too_large",httpStatus:null,retryable:!1}:/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(t)?{errorCode:"browser_session_interrupted",httpStatus:null,retryable:!0}:be.has(r??"")||/timeout|navigation_failed|ERR_/i.test(t)?{errorCode:"page_unreachable",httpStatus:null,retryable:!0}:{errorCode:"extraction_failed",httpStatus:null,retryable:!1}}function j(e,t){switch(e){case"page_not_found":return`The page could not be found (HTTP ${t??404}). It may have been moved or deleted.`;case"page_forbidden":return`The site refused the request (HTTP ${t??403}).`;case"page_rate_limited":return"The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";case"page_server_error":return`The page's own server returned an error (HTTP ${t}). This is on the target site's side, not something a retry here can fix.`;case"page_http_error":return`The page returned an unexpected HTTP ${t} response.`;case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. A retry uses a fresh browser session.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"extraction_failed":return"The page could not be extracted. Please retry, or contact support if this persists for the same URL."}}function Ie(e,t={}){let r=j(e.errorCode,e.httpStatus);return{...$({errorCode:e.errorCode,retryable:e.retryable,chargeStatus:t.chargeStatus,details:{...t.details??{},...e.httpStatus!=null?{http_status:e.httpStatus}:{}}}),message:r,error:r}}function ke(e){return e.httpStatus===404?404:e.httpStatus===403?403:e.httpStatus===429?429:e.httpStatus!=null&&e.httpStatus>=500&&e.httpStatus<=599?500:e.httpStatus!=null?502:e.errorCode==="vendor_unavailable"?503:502}var Ee="wayback_capture_missing";function Be(e,t){if(e==null)return{failureCode:null,failureReason:null};if(e===Ee)return{failureCode:e,failureReason:t??null};let r=new Error(t??e);r.code=e;let n=xe(r);return{failureCode:n.errorCode,failureReason:j(n.errorCode,n.httpStatus)}}import{neon as Te}from"@neondatabase/serverless";var Se="jina",M=null,z=!1;function W(){return(process.env.JINA_EMBED_MODEL??"jina-embeddings-v5-omni-small").trim()}function Y(){return Number((process.env.JINA_EMBED_DIM??"1024").trim())}function J(){return!!(process.env.JINA_API_KEY?.trim()&&process.env.MEMORY_DATABASE_URL?.trim())}function E(){if(M)return M;let e=process.env.MEMORY_DATABASE_URL?.trim();if(!e)throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");return M=Te(e),M}async function K(){if(z)return;let e=Y();await E().query("CREATE EXTENSION IF NOT EXISTS vector"),await E().query(`
     CREATE TABLE IF NOT EXISTS commons_index_vectors (
       document_id   TEXT PRIMARY KEY,

package/dist/{chunk-YUPLOOTB.js → chunk-MKEMCMRO.js}

@@ -1,4 +1,4 @@
-import{Ab as R,Bb as E,Db as v,Lb as T,Nb as h,Ob as w,Pb as _,Qb as y,Rb as C,Va as g,Xa as S,w as u,x as f}from"./chunk-MEM4D2QN.js";import{a as P}from"./chunk-CCYWSJNG.js";import{a as m}from"./chunk-5H3MCTTP.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
+import{Ab as R,Bb as E,Db as v,Lb as T,Nb as h,Ob as w,Pb as _,Qb as y,Rb as C,Va as g,Xa as S,w as u,x as f}from"./chunk-3MUEBDWY.js";import{a as P}from"./chunk-ZWZ6OWY6.js";import{a as m}from"./chunk-TW46NKVH.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
 `).filter(o=>o.startsWith("data:")).map(o=>o.slice(5).trim()).filter(Boolean);if(!s.length)return JSON.parse(r);for(let o of s){let e=JSON.parse(o);if(e.result||e.error)return e}throw new Error("hosted MCP returned no JSON-RPC result")}function a(r){return{content:[{type:"text",text:JSON.stringify({ok:!1,error:r})}],isError:!0}}var c=class{baseUrl;apiKey;constructor(s,o){this.baseUrl=s.replace(/\/$/,""),this.apiKey=o}async callMemoryTool(s,o){let e=x.get(s);if(!e)return a(`unknown memory tool: ${s}`);try{let n=await fetch(`${this.baseUrl}/mcp`,{method:"POST",headers:{"content-type":"application/json",accept:"application/json, text/event-stream","x-api-key":this.apiKey},body:JSON.stringify({jsonrpc:"2.0",id:`memory:${e}`,method:"tools/call",params:{name:e,arguments:o}})}),l=await n.text();if(!n.ok)return a(`hosted memory ${e} failed (HTTP ${n.status})`);let t=O(l);return t.error?a(t.error.message??`hosted memory ${e} failed`):t.result??a(`hosted memory ${e} returned no result`)}catch(n){return a(n instanceof Error?n.message:`hosted memory ${e} call failed`)}}};function M(r,s){if(s.length===0)throw new Error("Restricted MCP tool allowlist must contain at least one tool name");let o=new Set;for(let t of s){if(typeof t!="string"||t.trim()!==t||t.length===0)throw new Error("Restricted MCP tool allowlist contains a malformed tool name");if(o.has(t))throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${t}`);o.add(t)}let e=new Set,n=r,l=n.registerTool.bind(r);return n.registerTool=(t,i,p)=>{if(o.has(t))return e.add(t),l(t,i,p)},{assertComplete(){let t=[...o].filter(i=>!e.has(i));if(t.length>0)throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${t.join(", ")}`)},registeredToolNames(){return[...e]}}}var L=new Set(["paa","serp","browser-agent","scheduled-results","memory"]);function $(){let s=[process.env.MCP_SCRAPER_KEY_PATH?.trim(),A(I(),".mcp-scraper-key")].filter(Boolean);for(let o of s)try{let e=N(o,"utf8").trim();if(e)return e}catch{}}function Y(r,s={}){let o=s.toolsets??L,e=process.env.MCP_SCRAPER_BASE_URL?.trim()||process.env.MCP_BASE_URL?.trim()||"https://mcpscraper.dev",n=g(),l=S({deploymentProfile:n,transportProfile:"stdio",baseUrl:e,explicitlyEnabled:process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK==="1"}),t=process.env.BROWSER_AGENT_CONSOLE_URL?.trim()||e,i=new k({name:"mcp-scraper",version:m},{instructions:u,cacheHints:{"server/discover":{ttlMs:3e5,cacheScope:"private"},"tools/list":{ttlMs:3e5,cacheScope:"private"},"prompts/list":{ttlMs:3e5,cacheScope:"private"},"resources/list":{ttlMs:3e5,cacheScope:"private"},"resources/templates/list":{ttlMs:3e5,cacheScope:"private"},"resources/read":{ttlMs:6e4,cacheScope:"private"}}});f(i);let p=s.allowedToolNames?M(i,s.allowedToolNames):void 0,d=s.httpExecutor??new T(e,r,{localNetworkAccess:l});return o.has("paa")&&v(i,d,{ownerId:R(r),deploymentProfile:n,transportProfile:"stdio",baseUrl:e,localNetworkAccess:l,taskHandleSecret:r}),o.has("serp")&&E(i,d,{exposeDevelopmentDiagnostics:n==="development"||n==="test"}),o.has("browser-agent")&&h(i,{baseUrl:e,apiKey:r,consoleBaseUrl:t}),o.has("scheduled-results")&&C(i,new y(e,r)),o.has("memory")&&_(i,new c(e,r)),p?.assertComplete(),i}function se(r={}){let s=process.argv.includes("--stdio")||process.env.MCP_SCRAPER_FORCE_STDIO==="1",o=!!(process.stdin.isTTY&&process.stdout.isTTY),e=process.argv.includes("--help")||process.argv.includes("-h");if(!s&&(o||e)){let l=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY;process.stdout.write(P({version:m,color:!l,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()})),process.exit(0)}let n=(process.env.MCP_SCRAPER_API_KEY??process.env.MCP_SCRAPER_KEY??process.env.MCP_API_KEY??$())?.trim();n||(process.stderr.write(`MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required
 `),process.exit(1)),b(()=>Y(n,r),{legacy:"serve",onerror(l){process.stderr.write(`${l.message}
 `)}})}export{se as a};