mcp-scraper 0.93.4 → 0.94.1

package/dist/chunk-ZG52SKGD.js

@@ -1,6 +0,0 @@
-import{a as Tt}from"./chunk-XG6GCEUE.js";import{l as St,m as Rt,w as At}from"./chunk-M5VZVMPC.js";import{a as G,c as Oe,d as L,e as V,f as le,j as Ie,k as Ct}from"./chunk-DT2FYN6N.js";import{b as Pt,c as Et,d as $,f as z,g as H,h as U,i as kt,k as B,n as Ot,q as It}from"./chunk-ORB4RHCK.js";import{a as Ce,b as Te,g as xe}from"./chunk-UN6FVDJQ.js";import{a as it,b as lt}from"./chunk-SH5KB4P7.js";import{a as bt,b as wt,d as vt}from"./chunk-2SP57VCG.js";import{b as yt,c as _t}from"./chunk-6DTXIZY2.js";import{c as at}from"./chunk-Y6MKMSOC.js";import{Nb as ie,Rb as Ee,Tb as ke,Va as Ae,b as Se,g as M,h as ut,hb as ht,ia as mt,j as ct,ja as gt,jb as Pe,la as ft,m as dt,o as pt,oa as Re}from"./chunk-WJ4XFLS4.js";import{request as wr}from"https";import{HttpsProxyAgent as vr}from"https-proxy-agent";var ue={source:"https://\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061.com/users/zone/premium_domains",retrievedAt:"2026-09-23T20:14:30.483Z",includedStatuses:["premium","remove_candidate"],domains:["advanceautoparts.com","affitto.it","agoda.cn","albertsons.com","allpeople.com","autozone.com","bestbuy.com","bestwestern.com","billiger.de","bottlerover.com","carousell.com","carousell.com.hk","carousell.com.my","carousell.ph","carousell.sg","carsales.com.au","cdiscount.com","chewy.com","costco.com","cvs.com","despegar.com.mx","dickssportinggoods.com","dynos.es","emaxme.com","familytreenow.com","feuvert.fr","flooranddecor.com","foodlion.com","footlocker.co.uk","footlocker.com","giantfoodstores.com","gopuff.com","gplay.bg","hermes.com","hyatt.com","idealo.de","immobilienscout24.de","ingatlan.com","instacart.com","intersport.fr","joann.com","kroger.com","lazada.co.id","lazada.co.th","lazada.com.my","lazada.com.ph","lazada.sg","lazada.vn","lowes.ca","lowes.com","mcmaster.com","mediamarkt.de","mediamarkt.es","medline.com","mscdirect.com","napaonline.com","nofrills.ca","peoplefinders.com","platt.com","publicdatausa.com","realcanadiansuperstore.ca","realestate.com.au","restaurantguru.com","searchpeoplefree.com","shopee.cl","shopee.co.id","shopee.co.th","shopee.com.br","shopee.com.co","shopee.com.mx","shopee.com.my","shopee.ph","shopee.sg","shopee.tw","shopee.vn","similarweb.com","skyscanner.co.kr","skyscanner.net","stopandshop.com","target.com","temu.com","ticketmaster.com","totalwine.com","tractorsupply.com","walmart.com.mx","wayfair.com","weismarkets.com","wizzair.com","worten.pt"]};var Zr=ue.source,xt=ue.retrievedAt,yr=new Set(ue.domains);function Z(e){let t=new URL(e).hostname.toLowerCase().replace(/\.$/,"");for(let n of yr)if(t===n||t.endsWith(`.${n}`))return"premium";return"standard"}function ee(e){for(let t of["providerProduct","billingTier","billableEvent","unitType","source","policyVersion","effectiveAt"])if(!e[t]?.trim())throw new Error(`Provider rate ${t} is required`);if(!Number.isSafeInteger(e.numeratorUsdNanos)||e.numeratorUsdNanos<0)throw new Error("Provider rate numerator must be a non-negative safe integer");if(!Number.isSafeInteger(e.denominatorUnits)||e.denominatorUnits<1)throw new Error("Provider rate denominator must be a positive safe integer");if(Number.isNaN(Date.parse(e.effectiveAt)))throw new Error("Provider rate effectiveAt must be a date");return e}function ce(e,t){if(ee(e),!Number.isSafeInteger(t)||t<0)throw new Error("Billable units must be a non-negative safe integer");let n=BigInt(e.numeratorUsdNanos)*BigInt(t),r=BigInt(e.denominatorUnits),o=(n*2n+r)/(r*2n);if(o>BigInt(Number.MAX_SAFE_INTEGER))throw new Error("Provider amount exceeds safe integer range");return Number(o)}function _r(){return ee({providerProduct:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_api",billingTier:"payg",billableEvent:"successful_delivery",unitType:"request",numeratorUsdNanos:15e8,denominatorUnits:1e3,source:"owner_confirmed_2026-09-24;docs/decisions/2026-08-11-preserve-questions-only-paa-while-evaluating-serp-api.md",policyVersion:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061-serp-payg-2026-09-24",effectiveAt:"2026-08-11T00:00:00Z"})}function Mt(e=process.env){let t=e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_CONTRACT_USD_PER_1000?.trim(),n=e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_RATE_SOURCE?.trim(),r=e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_RATE_POLICY_VERSION?.trim(),o=e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_RATE_EFFECTIVE_AT?.trim();if(!t&&!n&&!r&&!o)return _r();if(!t||!n||!r||!o)throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP contract rate requires amount, source, policy version, and effective date");let a=Number(t);if(!Number.isFinite(a)||a<0)throw new Error("Invalid \u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP contract amount");return ee({providerProduct:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_api",billingTier:"payg",billableEvent:"successful_delivery",unitType:"request",numeratorUsdNanos:Math.round(a*1e9),denominatorUnits:1e3,source:n,policyVersion:r,effectiveAt:o})}function br(e){return ee({providerProduct:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api",billingTier:e,billableEvent:"transferred_byte",unitType:"byte",numeratorUsdNanos:(e==="premium"?11:8)*1e9,denominatorUnits:1e9,source:`owner_confirmed_payg_2026-09-23;\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_domain_snapshot_${xt}`,policyVersion:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061-browser-domain-tier-2026-09-23",effectiveAt:"2026-09-23T00:00:00Z"})}function ro(e){return br(Z(e))}function Nt(){return ee({providerProduct:"apify_google_serp_scraperlink",billingTier:"store_pay_per_event",billableEvent:"serp_page",unitType:"page",numeratorUsdNanos:5e8,denominatorUnits:1e3,source:"https://apify.com/scraperlink/google-search-results-serp-scraper",policyVersion:"scraperlink-listing-2026-09-24",effectiveAt:"2026-09-24T00:00:00Z"})}var Sr="https://api.\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061.com/request",Rr=8e3,Ar="brd.superproxy.io",Pr="44445",Er=10*1024*1024;async function kr(e,t){let n=new URL(`http://${t.host}:${t.port}`);n.username=t.username,n.password=t.password;let r=new vr(n);try{return await new Promise((o,a)=>{let s=wr(e,{agent:r,headers:{"x-unblock-data-format":"parsed"},rejectUnauthorized:!1,signal:t.signal},i=>{let l=[],u=0;i.on("data",c=>{let p=Buffer.isBuffer(c)?c:Buffer.from(c);if(u+=p.length,u>Er){s.destroy(new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP response exceeded the maximum size"));return}l.push(p)}),i.on("end",()=>o({ok:(i.statusCode??500)>=200&&(i.statusCode??500)<300,status:i.statusCode??500,rawText:Buffer.concat(l).toString("utf8"),providerRequestId:typeof i.headers["x-response-id"]=="string"?i.headers["x-response-id"]:typeof i.headers["x-request-id"]=="string"?i.headers["x-request-id"]:null}))});s.on("error",()=>a(new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 native SERP request failed"))),s.end()})}finally{r.destroy()}}function zt(e=process.env){let t=!!(e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_PROXY_USERNAME?.trim()&&e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_PROXY_PASSWORD?.trim()),n=!!((e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_API_KEY||e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_API_KEY)?.trim()&&e.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_ZONE?.trim());return t||n}function m(e){return e!=null&&typeof e=="object"&&!Array.isArray(e)?e:null}function N(e){return Array.isArray(e)?e:[]}function Ne(e){return typeof e=="string"&&e.trim()?e.trim():null}function A(e){return typeof e=="number"&&Number.isFinite(e)?e:typeof e=="string"&&e.trim()&&Number.isFinite(Number(e))?Number(e):null}function g(e,t){for(let n of t){let r=Ne(e[n]);if(r)return r}return null}function J(e){try{return new URL(e).hostname.replace(/^www\./,"")}catch{return""}}function Kt(e){return{rawUrl:e,resolvedUrl:e,linkType:"plain",resolutionStatus:"not_needed"}}function Or(e){let t=e;if(typeof t=="string")try{t=JSON.parse(t)}catch{throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP API returned invalid JSON")}let n=m(t);if(!n)throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP API returned an invalid payload");if(typeof n.body=="string")try{let o=JSON.parse(n.body);if(m(o))return m(o)}catch{}let r=m(n.body);return r?m(r.result)??m(r.data)??r:m(n.result)??m(n.data)??n}function j(e,t){for(let r of t){let o=N(e[r]);if(o.length)return o}let n=m(e.results);if(n)for(let r of t){let o=N(n[r]);if(o.length)return o}return[]}function K(e,t){let n=[e,m(e.results)].filter(r=>!!r);for(let r of n)for(let o of t)if(Object.prototype.hasOwnProperty.call(r,o))return{observed:!0,rawShapeValid:Array.isArray(r[o]),rawCount:N(r[o]).length};return{observed:!1,rawShapeValid:!1,rawCount:0}}function Me(e,t){let n=[e,m(e.results)].filter(r=>!!r);for(let r of n)for(let o of t){if(!Object.prototype.hasOwnProperty.call(r,o))continue;let a=m(r[o]);return{observed:!0,rawShapeValid:a!==null,rawCount:a&&Object.keys(a).length>0?1:0}}return{observed:!1,rawShapeValid:!1,rawCount:0}}function Dt(e){return j(e,["organic","organic_results","organicResults"]).flatMap((t,n)=>{let r=m(t);if(!r)return[];let o=g(r,["link","url","href"]),a=g(r,["title","name"]);if(!o||!a)return[];let s=m(r.rating)??m(r.inline_rating);return[{position:Math.max(1,Math.trunc(A(r.global_rank)??A(r.rank)??A(r.position)??n+1)),title:a,url:o,...Kt(o),domain:J(o),cite:g(r,["display_link","displayLink","cite"]),snippet:g(r,["description","snippet","text"]),isRedditStyle:J(o)==="reddit.com",inlineRating:s?{value:String(s.value??s.rating??""),count:String(s.count??s.reviews??"")}:null}]}).sort((t,n)=>t.position-n.position)}function qt(e){return j(e,["snack_pack","local","local_results","local_pack","localPack"]).flatMap((t,n)=>{let r=m(t);if(!r)return[];let o=g(r,["title","name"]);if(!o)return[];let a=N(r.extensions??r.metadata).map(Ne).filter(s=>!!s);return[{position:Math.max(1,Math.trunc(A(r.global_rank)??A(r.rank)??A(r.position)??n+1)),name:o,cid:g(r,["cid"]),rating:g(r,["rating"])??(A(r.rating)!=null?String(A(r.rating)):null),reviewCount:g(r,["reviews_cnt","reviews","review_count","reviewCount"])??(A(r.reviews_cnt)!=null?String(A(r.reviews_cnt)):A(r.reviews)!=null?String(A(r.reviews)):null),metadata:a,websiteUrl:g(r,["website","website_url","link"]),directionsUrl:g(r,["directions","directions_url"])}]})}function Jt(e){let t=m(e);return N(t?.citations??t?.sources??t?.references).flatMap(n=>{let r=m(n);if(!r)return[];let o=g(r,["link","url","href"]);return o?[{text:g(r,["title","text","name"])??J(o),href:o,...Kt(o)}]:[]})}function me(e,t=0){if(t>4)return[];if(Array.isArray(e))return e.flatMap(a=>me(a,t+1));let n=m(e);if(!n)return[];let r=g(n,["snippet","text","answer"]),o=me(n.texts??n.list??n.items,t+1);return[...r?[r]:[],...o]}function Ht(e,t){if(!t)return{detected:!1,text:null,citations:[]};let n=m(e.ai_overview)??m(e.aiOverview)??m(e.overview);if(!n)return{detected:!1,text:null,citations:[]};let r=g(n,["text","answer","description"])??(me(n.texts).join(`
-
-`).slice(0,5e4)||null),o=Jt(n);return{detected:!!(r||o.length),text:r,citations:o}}function Ut(e,t){if(!t)return{detected:!1,text:null,citations:[]};let n=m(e.ai_mode)??m(e.aiMode);if(!n)return{detected:!1,text:null,citations:[]};let r=g(n,["text","answer","description"])??(me(n.texts).join(`
-
-`).slice(0,5e4)||null),o=Jt(n);return{detected:!!(r||o.length),text:r,citations:o}}function Bt(e,t){return t?[...j(e,["videos","video_results"]),...j(e,["short_videos","shorts"])].flatMap(n=>{let r=m(n);if(!r)return[];let o=g(r,["link","url"]),a=g(r,["title","name"]);return!o||!a?[]:[{type:"video",title:a,channel:g(r,["channel","source"])??"",platform:J(o),duration:g(r,["duration"])??"",url:o}]}):[]}function Ft(e,t){if(!t)return[];let n=m(e.forums);return[...j(e,["discussions","forums","forum_results"]),...N(n?.items)].flatMap(r=>{let o=m(r);if(!o)return[];let a=g(o,["link","url"]),s=g(o,["title","name"]);return!a||!s?[]:[{title:s,source:g(o,["source"])??J(a),url:a}]})}function Gt(e,t){return t?j(e,["what_people_are_saying","whatPeopleSaying"]).flatMap(n=>{let r=m(n);if(!r)return[];let o=g(r,["link","url"]),a=g(r,["title","name"]);if(!o||!a)return[];let s=g(r,["platform","source"])??J(o),i=s.toLowerCase();return[{type:i.includes("reddit")?"reddit":i.includes("facebook")?"facebook":i.includes("instagram")?"instagram":i.includes("tiktok")?"tiktok":i.includes("youtube")?"youtube":i.includes("news")?"news":"unknown",title:a,url:o,source:s,platform:s,popularComment:g(r,["comment","popular_comment"]),engagement:g(r,["engagement"])??"",date:g(r,["date"])??"",duration:g(r,["duration"]),authorNote:g(r,["author_note"])}]}):[]}var Ir={featuredSnippets:["featured_snippets","featured_snippet","featuredSnippet"],knowledgePanel:["knowledge","knowledge_graph","knowledgeGraph"],topStories:["top_stories","topStories"],news:["news","news_results"],images:["images","image_results"],shopping:["shopping","shopping_results"],relatedSearches:["related","related_searches","relatedSearches"],recipes:["recipes","recipe_results"],jobs:["jobs","job_results"],topAds:["top_ads"],bottomAds:["bottom_ads"],topProducts:["top_pla"],bottomProducts:["bottom_pla"],jackpotProducts:["jackpot_pla"],shortVideos:["short_videos","shorts"],localMap:["snack_pack_map"],flights:["flights"],hotels:["hotels_selection"],popularProducts:["popular_products"],weather:["weather"],currencyExchange:["currency_exchange"],latestPosts:["latest_posts"],perspectives:["perspectives"],navigation:["navigation"],chips:["chips"],spelling:["spelling"]};function de(e,t){for(let n of[e,m(e.results)])if(n){for(let r of t)if(Object.prototype.hasOwnProperty.call(n,r))return{observed:!0,value:n[r]}}return{observed:!1,value:null}}function pe(e,t,n){return e?t==null||Array.isArray(t)&&t.length===0?"observed_absent":Array.isArray(t)?n>0?"observed_present":"incomplete":m(t)?n>0?"observed_present":Object.keys(m(t)).length===0?"observed_absent":"incomplete":"incomplete":"unknown"}function Cr(e,t){let n={},r=[["localPack",["snack_pack","local","local_results","local_pack","localPack"],t.localPack],["forums",["discussions","forums","forum_results"],t.forums],["videos",["videos","video_results","short_videos","shorts"],t.videos],["aiOverview",["ai_overview","aiOverview","overview"],t.aiOverview],["whatPeopleSaying",["what_people_are_saying","whatPeopleSaying"],t.whatPeopleSaying],["entityIds",["knowledge","knowledge_graph","knowledgeGraph"],t.entityIds]];for(let[l,u,c]of r){let p=de(e,u);n[l]=pe(p.observed,p.value,c)}let o=de(e,["ai_mode","aiMode"]);n.aiMode=pe(o.observed,o.value,t.aiMode);let a=de(e,["people_also_ask","peopleAlsoAsk","related_questions","paa"]),s=N(a.value).slice(0,30).flatMap(l=>{let u=m(l);if(!u)return[];let c=g(u,["question","title"]);return c?[{question:c.slice(0,1e3),answer:g(u,["answer","snippet","text"])?.slice(0,1e4)??null,url:g(u,["link","url"])??null}]:[]});n.paaPreview=pe(a.observed,a.value,s.length);let i=[];for(let[l,u]of Object.entries(Ir)){let c=de(e,u);if(!c.observed)continue;let p=m(c.value),w=(Array.isArray(c.value)?c.value:p&&(Array.isArray(p.items)||Array.isArray(p.posts))?[...N(p.items),...N(p.posts)]:[c.value]).slice(0,20).flatMap(S=>{let y=m(S);if(!y)return[];let C=g(y,["title","name","heading"]),k=g(y,["link","url","website"]),D=g(y,["description","snippet","text","summary","story"]);return!C&&!k&&!D?[]:[{title:C?.slice(0,1e3)??null,url:k,description:D?.slice(0,1e4)??null,position:A(y.global_rank)??A(y.rank)??A(y.position)??null}]});n[l]=pe(c.observed,c.value,w.length);let R=Object.fromEntries(Object.entries(p??{}).slice(0,30).flatMap(([S,y])=>/^[a-zA-Z0-9_]{1,80}$/.test(S)&&(y===null||["string","number","boolean"].includes(typeof y))?[[S,typeof y=="string"?y.slice(0,2e3):y]]:[]));i.push({name:l,attributes:R,items:w})}return{status:n,paaPreview:s,additional:i}}function Lt(e,t){let n=new Set,r=new Set(t.map(s=>s.cid).filter(s=>!!s)),o=new Set,a=(s,i=0)=>{if(i>6||s==null)return;if(Array.isArray(s)){for(let u of s)a(u,i+1);return}let l=m(s);if(l)for(let[u,c]of Object.entries(l)){let p=u.toLowerCase().replace(/[^a-z]/g,""),_=Ne(c);_&&(p==="kgid"||p==="kgmid"||p==="mid")?n.add(_):_&&p==="cid"?r.add(_):_&&p==="gcid"?o.add(_):typeof c=="object"&&a(c,i+1)}};return a(e),{entities:[],kgIds:[...n],cids:[...r],gcids:[...o]}}function Vt(e,t){let n=e instanceof Error?e.message:String(e);return t.aborted||/timeout|timed out/i.test(n)?"timeout":/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(n)?"captcha":/invalid (?:json|payload)/i.test(n)?"invalid_payload":"provider_error"}function $t(e){return e==="invalid_payload"?"invalid_payload":e==="captcha"?"serp_readiness":e==="timeout"||e==="provider_error"?"provider_transport":"pagination"}async function De(e,t=process.env,n=fetch,r=kr,o){if(e.pages!==1)throw new Error("\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP supports one page only; use Apify for two pages");let a=(t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_API_KEY||t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_API_KEY)?.trim(),s=t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_ZONE?.trim(),i=t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_PROXY_USERNAME?.trim(),l=t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_PROXY_PASSWORD?.trim(),u=!!(i&&l);if(!u&&(!a||!s))throw new Error("Direct SERP API is not configured");let c=Date.now(),p=_t(),_=null,w=e.location?Rt(e.location):null;if(e.location&&!w?.canonical)throw new G("query_integrity","location_unresolved",`SERP location could not be resolved without changing the query (candidates: ${w?.ambiguousCandidates??0})`);let R=w?.canonical?St(w.canonical):null,S=AbortSignal.timeout(e.includeAllSerpFeatures?33e3:Rr),y=e.signal?AbortSignal.any([e.signal,S]):S,C=1,k=[],D=[],ze=0,Ke=async d=>{let O=Date.now(),b=u?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_native_proxy":"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_rest",I=null,q=null,Q=!1,W=!1,F=null,Y=null;if(p?.operationRunId&&(d===1||_))try{I=await $({runId:p.operationRunId,attemptNumber:o?o.attemptNumber+d-1:d,method:b,provider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",...d===2?{parentAttemptId:_,edgeKind:"next_phase"}:o?{parentAttemptId:o.parentAttemptId,edgeKind:"provider_fallback",retryKind:"provider_fallback"}:{},normalizedFlags:{page:d,device:e.device,hasLocation:!!e.location}}),d===1&&(_=I),q=(await H({attemptId:I,provider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",providerProduct:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_api",accountScope:p.operationAccountScope??"production-default",method:b,providerStatus:"started"})).id}catch(h){throw console.warn(JSON.stringify({event:"direct_serp_telemetry_start_failed",page:d,message:h instanceof Error?h.message:String(h)})),h}let P=new URLSearchParams({q:e.query,gl:e.gl,hl:e.hl,pws:"0",num:"10",...d===2?{start:"10"}:{}}),ae=at(e.recency);ae&&P.set("tbs",ae),e.device==="mobile"&&P.set("brd_mobile","1"),(e.includeAiOverview||e.includeAllSerpFeatures)&&P.set("brd_ai_overview","2"),R&&P.set("uule",R),u&&P.set("brd_json","1"),ze+=1;let x;try{if(u){let E=t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_PROXY_HOST?.trim()||Ar,X=t.\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_PROXY_PORT?.trim()||Pr;x=await r(`https://www.google.com/search?${P}`,{username:i,password:l,host:E,port:X,signal:y})}else{let E=await n(Sr,{method:"POST",headers:{Authorization:`Bearer ${a}`,"Content-Type":"application/json"},body:JSON.stringify({zone:s,url:`https://www.google.com/search?${P}`,format:"json",data_format:"parsed"}),signal:y});x={ok:E.ok,status:E.status,rawText:await E.text(),providerRequestId:E.headers.get("x-response-id")??E.headers.get("x-request-id")}}if(Y=`http_${x.status}`,q&&x.providerRequestId&&await U(q,x.providerRequestId),!x.ok)throw new Error(`\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP API failed with HTTP ${x.status}`);Q=!0;let h;try{h=Or(JSON.parse(x.rawText))}catch(E){let X=E instanceof Error&&E.message!=="Unexpected end of JSON input"?E.message:"\u0042\u0072\u0069\u0067\u0068\u0074 \u0044\u0061\u0074\u0061 SERP API returned invalid JSON";throw new G("invalid_payload","invalid_provider_payload",X)}let f=Dt(h);return k.push({page:d,attempted:!0,captured:f.length>0,resultCount:f.length,requestOffset:d===2?10:0,providerRequestIdObserved:!!x.providerRequestId,...f.length===0?{failureCode:"empty_page"}:{}}),W=!0,h}catch(h){F=h;let f=Vt(h,y);if(k.some(X=>X.page===d)||k.push({page:d,attempted:!0,captured:!1,resultCount:0,requestOffset:d===2?10:0,providerRequestIdObserved:!1,failureCode:f}),d===2)throw h;let E=h instanceof G?h.stage:$t(f);throw h instanceof G?h:new G(E,f,h instanceof Error?h.message:String(h))}finally{let h=null;if(q&&Q)try{let f=Mt(t);await B({receiptId:q,costState:"measured",units:1,unitType:"request",amountUsdNanos:ce(f,1),rateSource:f.source,providerProduct:f.providerProduct,billingTier:f.billingTier,billableEvent:f.billableEvent,measurementSource:"provider_http_delivery",rateNumerator:f.numeratorUsdNanos,rateDenominator:f.denominatorUnits,ratePolicyVersion:f.policyVersion,rateEffectiveAt:f.effectiveAt,providerStatus:Y??"delivered",providerDurationMs:Date.now()-O})}catch(f){console.warn(JSON.stringify({event:"direct_serp_receipt_finalize_failed",page:d,message:f instanceof Error?f.message:String(f)})),h=f}if(I&&await z({attemptId:I,status:W?"succeeded":y.aborted?"timed_out":"failed",errorCode:F?Vt(F,y):null,durationMs:Date.now()-O}).catch(f=>console.warn(JSON.stringify({event:"direct_serp_attempt_finish_failed",page:d,message:f instanceof Error?f.message:String(f)}))),h)throw h}};try{D.push({page:1,payload:await Ke(1)})}catch(d){throw d}let Je=null;if(C===2)try{D.push({page:2,payload:await Ke(2)})}catch(d){Je=d}let ir=Date.now()-c,lr=Date.now(),v=D[0].payload,ur=D.flatMap(({page:d,payload:O})=>Dt(O).map(b=>({...b,position:d===2&&b.position<=10?b.position+10:b.position,sourcePage:d}))).filter((d,O,b)=>b.findIndex(I=>Oe(I.url)===Oe(d.url))===O),je=e.includeLocalPack||e.includeAllSerpFeatures,ye=je?qt(v):[],_e=K(v,["snack_pack","local","local_results","local_pack","localPack"]),cr=V({requested:je,moduleObserved:_e.observed,rawCount:_e.rawCount,mappedCount:ye.length,rawShapeValid:_e.rawShapeValid}),Qe=e.includeAiOverview||e.includeAllSerpFeatures,We=Ht(v,Qe),Ye=e.includeForums||e.includeAllSerpFeatures,Xe=Ft(v,Ye),Ze=e.includeVideos||e.includeAllSerpFeatures,et=Bt(v,Ze),tt=e.includeWhatPeopleSaying||e.includeAllSerpFeatures,rt=Gt(v,tt),be=m(v.forums),we=be&&Array.isArray(be.items)?{observed:!0,rawCount:be.items.length,rawShapeValid:!0}:K(v,["discussions","forums","forum_results"]),ve=K(v,["videos","video_results","short_videos","shorts"]),ot={localPack:cr,forums:V({requested:Ye,moduleObserved:we.observed,rawCount:we.rawCount,mappedCount:Xe.length,rawShapeValid:we.rawShapeValid}),videos:V({requested:Ze,moduleObserved:ve.observed,rawCount:ve.rawCount,mappedCount:et.length,rawShapeValid:ve.rawShapeValid}),aiOverview:V({requested:Qe,moduleObserved:Me(v,["ai_overview","aiOverview","overview"]).observed,rawCount:Me(v,["ai_overview","aiOverview","overview"]).rawCount,mappedCount:We.detected?1:0,rawShapeValid:Me(v,["ai_overview","aiOverview","overview"]).rawShapeValid}),whatPeopleSaying:V({requested:tt,moduleObserved:K(v,["what_people_are_saying","whatPeopleSaying"]).observed,rawCount:K(v,["what_people_are_saying","whatPeopleSaying"]).rawCount,mappedCount:rt.length,rawShapeValid:K(v,["what_people_are_saying","whatPeopleSaying"]).rawShapeValid})},dr=L(1,k),pr=k.find(d=>d.page===1)?.resultCount??0,ne=k.find(d=>d.page===2)?.resultCount??0,mr=le({query:e.query,finalQuery:e.query,canonicalLocation:w?.canonical??null,locationResolutionSource:w?.source??"not_requested",locationAmbiguousCandidates:w?.ambiguousCandidates??0,uule:R,outboundProviderMethod:u?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_native_proxy":"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_serp_rest"}),se=Je?$t(k.find(d=>d.page===2)?.failureCode??"provider_error"):Object.values(ot).some(d=>d==="incomplete"||d==="unknown")?"optional_feature_parsing":null,gr={pagination:{...dr,providerRequestCount:ze},features:ot,queryIntegrity:mr,failureStage:se},nt=e.includeAllSerpFeatures?(()=>{let d=D.map(({page:O,payload:b})=>{let I=qt(b),q=Ft(b,!0),Q=Bt(b,!0),W=Gt(b,!0),F=Ht(b,!0),Y=Ut(b,!0),P=Lt(b,I),ae=Cr(b,{localPack:I.length,forums:q.length,videos:Q.length,aiOverview:F.detected?1:0,aiMode:Y.detected?1:0,whatPeopleSaying:W.length,entityIds:P.entities.length+P.kgIds.length+P.cids.length+P.gcids.length});return{page:O,...ae,localPack:I,forums:q,videos:Q,whatPeopleSaying:W,aiOverview:F,aiMode:Y,entityIds:P}});return{status:d[0].status,paaPreview:d.flatMap(O=>O.paaPreview),additional:d.flatMap(O=>O.additional),pages:d}})():void 0,fr=new Date().toISOString(),st=Date.now()-c;return{seed:e.query,location:e.location??null,extractedAt:fr,diagnostics:{phaseTimings:{providerConnectMs:ir,serpParseMs:Date.now()-lr,totalServerMs:st},pagination:{requestedPages:C,capturedPages:C===2&&ne>0?2:1,page2Status:C===2?ne>0?"captured":"failed":"not_requested",page1OrganicCount:pr,page2OrganicCount:ne,...C===2&&ne===0?{failureCode:k.find(d=>d.page===2)?.failureCode==="captcha"?"captcha":k.find(d=>d.page===2)?.failureCode==="timeout"?"timeout":"empty_page"}:{}},completionStatus:"serp_only",problem:null,resultQuality:se?"partial":"complete",degradedResult:!!se,degradationReasons:[],retryRecommended:!!se,serpCompleteness:gr},totalQuestions:0,surface:"web",aiOverview:We,aiMode:Ut(v,e.includeAllSerpFeatures),...nt?{fullSerpFeatures:nt}:{},whatPeopleSaying:rt,tree:[],flat:[],videos:et,forums:Xe,organicResults:ur,localPack:ye,entityIds:Lt(v,ye),stats:{seed:e.query,totalQuestions:0,maxDepthReached:0,durationMs:st,errorCount:0}}}var Tr="scraperlink~google-search-results-serp-scraper",te="https://api.apify.com/v2";var T=class extends Error{constructor(n,r=null,o=!1,a=null){super(n);this.runId=r;this.retrySafe=o;this.terminalStatus=a;this.name="ApifyGoogleRunError"}runId;retrySafe;terminalStatus};function re(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)?e:null}function oe(e){return typeof e=="string"&&e.trim()?e.trim():null}function xr(e){try{return["http:","https:"].includes(new URL(e).protocol)}catch{return!1}}function Mr(e){if(typeof e!="string")return e;try{return JSON.parse(e)}catch{throw new Error("Apify searchResults is not valid JSON")}}function jt(e,t){if(!Array.isArray(e)||e.length===0)throw new Error("Apify dataset contains no search results");let n=[];for(let[a,s]of e.entries()){let i=re(s);if(!i)throw new Error("Apify dataset contains an invalid row");let l=Mr(i.searchResults??i),u=re(l),c=Array.isArray(l)?l:Array.isArray(u?.results)?u.results:Array.isArray(u?.organicResults)?u.organicResults:Array.isArray(u?.organic)?u.organic:u&&(u.position!==void 0||u.title!==void 0)?[u]:null;if(!c)throw new Error("Apify dataset search result shape is unsupported");let p=Number(i.page_number??u?.page_number??a+1);if(!Number.isInteger(p)||p<1||p>t)throw new Error("Apify dataset has an unexpected page number");n.push(...c.map(_=>({row:_,page:p})))}let r=n.map(({row:a,page:s},i)=>{let l=re(a),u=l?.position,c=oe(l?.title),p=oe(l?.url??l?.link);if(!l||!c||!p||!xr(p)||u!==void 0&&(!Number.isInteger(u)||u<1))throw new Error(`Apify search result ${i+1} is missing a valid position, URL, or title`);let _=oe(l.description??l.snippet),w=typeof u=="number"?u:i%10+1;return{position:s>1&&w<=10?w+(s-1)*10:w,url:p,title:c,description:_}});if(r.length>t*10)throw new Error("Apify returned more results than requested");let o={page1:r.filter(a=>a.position<=10).length,page2:r.filter(a=>a.position>10).length};if(o.page1===0)throw new Error("Apify did not return the first result page");return{results:r,pageResultCounts:o}}async function ge(e,t,n,r){let o=await n(e,{headers:{Authorization:`Bearer ${t}`},signal:r});if(!o.ok)throw new Error(`Apify API returned HTTP ${o.status}`);return o.json()}function qe(e){let t=re(re(e)?.data),n=oe(t?.id),r=oe(t?.status);if(!n||!r)throw new Error("Apify returned an invalid run receipt");return{id:n,status:r}}async function Nr(e,t=fetch){let n=performance.now(),r=AbortSignal.timeout(9e4),o=e.signal?AbortSignal.any([e.signal,r]):r,a=e.runId,s=!1,i=null;try{if(!/^[A-Za-z0-9]+$/.test(a))throw new Error("Invalid Apify run ID");let l=qe(await ge(`${te}/actor-runs/${a}?waitForFinish=30`,e.token,t,o));for(;["READY","RUNNING","TIMING-OUT","ABORTING"].includes(l.status);)l=qe(await ge(`${te}/actor-runs/${a}?waitForFinish=30`,e.token,t,o));if(s=!0,i=l.status,l.status!=="SUCCEEDED")throw new Error(`Apify run ended with status ${l.status}`);let u=e.pages??1,c=await ge(`${te}/actor-runs/${a}/dataset/items?format=json`,e.token,t,o);return{query:e.query,...jt(c,u),requestedPages:u,runId:a,elapsedMs:Math.round(performance.now()-n)}}catch(l){throw new T(l instanceof Error?l.message:String(l),a,s,i)}}async function Qt(e,t,n=fetch){let r=performance.now(),o=e.query.trim(),a=e.pages??1;if(!o||o.includes(`
-`))throw new Error("Apify search requires one nonempty query");if(a!==1&&a!==2)throw new Error("Apify search supports one or two pages");let s=t.token.trim();if(!s)throw new Error("Apify API token is required");let i=t.maxChargeUsd??.01;if(!Number.isFinite(i)||i<=0)throw new Error("Invalid Apify charge cap");let l=AbortSignal.timeout(9e4),u=e.signal?AbortSignal.any([e.signal,l]):l,c=`${te}/actors/${Tr}/runs?maxTotalChargeUsd=${i}&restartOnError=false&waitForFinish=30`,p=null,_=!1,w=null;try{let R=await n(c,{method:"POST",headers:{Authorization:`Bearer ${s}`,"Content-Type":"application/json"},body:JSON.stringify({keyword:o,limit:a===2?"20":"10",page:1,proxy_location:"us",include_merged:!1}),signal:u});if(!R.ok)throw new Error(`Apify run start returned HTTP ${R.status}`);let S=qe(await R.json());if(p=S.id,_=!["READY","RUNNING","TIMING-OUT","ABORTING"].includes(S.status),_&&(w=S.status),S.status!=="SUCCEEDED"&&S.status!=="READY"&&S.status!=="RUNNING")throw new Error(`Apify run ended with status ${S.status}`);if(S.status==="SUCCEEDED"){let C=await ge(`${te}/actor-runs/${p}/dataset/items?format=json`,s,n,u);return{query:o,...jt(C,a),requestedPages:a,runId:p,elapsedMs:Math.round(performance.now()-r)}}return{...await Nr({runId:p,query:o,token:s,pages:a,signal:u},n),elapsedMs:Math.round(performance.now()-r)}}catch(R){throw R instanceof T?R:new T(R instanceof Error?R.message:String(R),p,_,w)}}var Wt="apify_google_serp",Yt="apify",Dr="apify_google_serp_scraperlink";async function Xt(e,t,n,r){let o=Nt();await B({receiptId:e,costState:"estimated",units:t,unitType:o.unitType,amountUsdNanos:ce(o,t),rateSource:o.source,providerProduct:o.providerProduct,billingTier:o.billingTier,billableEvent:o.billableEvent,measurementSource:"actor_run_succeeded",rateNumerator:o.numeratorUsdNanos,rateDenominator:o.denominatorUnits,ratePolicyVersion:o.policyVersion,rateEffectiveAt:o.effectiveAt,providerStatus:r,providerDurationMs:n})}async function Zt(e){let t=performance.now(),n=await $({runId:e.operationRunId,attemptNumber:e.attemptNumber,method:Wt,provider:Yt,parentAttemptId:e.parentAttemptId,edgeKind:e.edgeKind,retryKind:e.retryKind,normalizedFlags:{page:1,pages:e.options.pages??1,proxyLocation:"us"}}),r=await H({attemptId:n,provider:Yt,providerProduct:Dr,billingTier:"store_pay_per_event",accountScope:e.accountScope,method:Wt,providerStatus:"before_actor_start"}),o=null,a=null;try{return o=await Qt(e.options,{token:e.token},e.fetchImpl),await U(r.id,o.runId),await Xt(r.id,o.requestedPages,o.elapsedMs,"succeeded_estimated"),{...o,attemptId:n}}catch(s){throw a=s,o?Object.assign(new T("Apify cost tracking failed after run completion",o.runId),{attemptId:n}):(s instanceof T&&s.runId&&(await U(r.id,s.runId),s.terminalStatus==="SUCCEEDED"?await Xt(r.id,e.options.pages??1,Math.round(performance.now()-t),"succeeded_dataset_unavailable"):await kt({receiptId:r.id,units:e.options.pages??1,unitType:"page",providerStatus:"run_outcome_or_cost_pending"})),Object.assign(s instanceof Error?s:new Error(String(s)),{attemptId:n}))}finally{await z({attemptId:n,status:a?e.options.signal?.aborted?"timed_out":"failed":"succeeded",errorCode:a?"apify_search_failed":null,durationMs:Math.round(performance.now()-t)})}}function er(e,t=e.requestedPages){let n=e.results.map(s=>{let i=new URL(s.url).hostname.replace(/^www\./,"");return{position:s.position,title:s.title,url:s.url,rawUrl:s.url,resolvedUrl:s.url,linkType:"plain",resolutionStatus:"not_needed",domain:i,cite:null,snippet:s.description,isRedditStyle:i==="reddit.com",inlineRating:null}}),r=t===2&&e.pageResultCounts.page2>0?2:1,o=t===2&&r===1,a=L(t,[{page:1,attempted:!0,captured:!0,resultCount:e.pageResultCounts.page1,requestOffset:0,providerRequestIdObserved:!0},...t===2?[{page:2,attempted:!0,captured:!o,resultCount:e.pageResultCounts.page2,requestOffset:10,providerRequestIdObserved:!0,...o?{failureCode:"empty_page"}:{}}]:[]]);return{seed:e.query,location:null,extractedAt:new Date().toISOString(),diagnostics:{phaseTimings:{providerConnectMs:e.elapsedMs,totalServerMs:e.elapsedMs},pagination:{requestedPages:t,capturedPages:r,page2Status:t===2?o?"failed":"captured":"not_requested",page1OrganicCount:e.pageResultCounts.page1,page2OrganicCount:e.pageResultCounts.page2},completionStatus:"serp_only",problem:null,resultQuality:o?"partial":"complete",degradedResult:o,degradationReasons:o?["page_two_missing"]:[],retryRecommended:!1,serpCompleteness:{pagination:a,features:{localPack:"not_requested",forums:"not_requested",videos:"not_requested",aiOverview:"not_requested",whatPeopleSaying:"not_requested"},queryIntegrity:le({query:e.query,finalQuery:e.query,locationResolutionSource:"not_requested",outboundProviderMethod:"default_search"}),failureStage:null}},totalQuestions:0,surface:"web",aiOverview:{detected:!1,text:null,citations:[],shareUrl:null},aiMode:{detected:!1,text:null,citations:[]},whatPeopleSaying:[],tree:[],flat:[],videos:[],forums:[],organicResults:n,localPack:[],entityIds:{entities:[],kgIds:[],cids:[],gcids:[]},stats:{seed:e.query,totalQuestions:0,maxDepthReached:0,durationMs:e.elapsedMs,errorCount:0}}}var qr=12e3;async function He(e){if(e.mode==="full")return Ur(e);if(!e.token)throw new Error("Apify search is not configured");let t=performance.now(),n;for(let o=1;o<=2;o++){let a=AbortSignal.any([e.signal,AbortSignal.timeout(qr)]),s=e.pages===2&&o===2?1:e.pages;try{let i=await Zt({options:{query:e.query,pages:s,signal:a},token:e.token,operationRunId:e.operationRunId,accountScope:e.accountScope,attemptNumber:o,...n?{parentAttemptId:n,edgeKind:"same_method_retry",retryKind:"same_method"}:{}});console.info(JSON.stringify({event:"serp_provider_succeeded",provider:"apify",operationRunId:e.operationRunId,attemptNumber:o,durationMs:i.elapsedMs,resultCount:i.results.length,requestedPages:e.pages,providerRequestedPages:i.requestedPages,deliveredPages:i.pageResultCounts.page2>0?2:1,runId:i.runId}));let l=er(i,e.pages);return l.stats.durationMs=Math.round(performance.now()-t),l.diagnostics.phaseTimings&&(l.diagnostics.phaseTimings.totalServerMs=l.stats.durationMs),{result:l,mode:"light",deliveryProvider:"apify",deliveredPages:i.pageResultCounts.page2>0?2:1}}catch(i){n=i&&typeof i=="object"&&"attemptId"in i?String(i.attemptId):void 0;let l=i instanceof T?i:null;if(console.warn(JSON.stringify({event:"serp_provider_failed",provider:"apify",operationRunId:e.operationRunId,attemptNumber:o,runId:l?.runId??null,requestedPages:e.pages,attemptPages:s,retrySafe:l?.retrySafe===!0,error:i instanceof Error?i.message:String(i)})),e.signal.aborted||!n||l?.retrySafe!==!0)throw i}}if(!n)throw new Error("Search attempt receipt is missing");let r={query:e.query,gl:"us",hl:"en",device:"desktop",pages:1,includeAllSerpFeatures:!1,includeLocalPack:!1,includeForums:!1,includeVideos:!1,includeAiOverview:!1,includeWhatPeopleSaying:!1,signal:e.signal};console.info(JSON.stringify({event:"serp_provider_fallback",provider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",operationRunId:e.operationRunId,attemptNumber:3,requestedPages:e.pages,providerRequestedPages:1}));try{let o=await De(r,process.env,fetch,void 0,{attemptNumber:3,parentAttemptId:n});if(o.organicResults.length===0)throw new Error("Search returned no organic results");return e.pages===2&&Hr(o),o.stats.durationMs=Math.round(performance.now()-t),o.diagnostics.phaseTimings&&(o.diagnostics.phaseTimings.totalServerMs=o.stats.durationMs),console.info(JSON.stringify({event:"serp_provider_succeeded",provider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",operationRunId:e.operationRunId,attemptNumber:3,durationMs:o.stats.durationMs,resultCount:o.organicResults.length,requestedPages:e.pages,providerRequestedPages:1,deliveredPages:1,resultQuality:o.diagnostics.resultQuality})),{result:o,mode:"light",deliveryProvider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",deliveredPages:1}}catch(o){throw console.warn(JSON.stringify({event:"serp_provider_failed",provider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",operationRunId:e.operationRunId,attemptNumber:3,error:o instanceof Error?o.message:String(o)})),o}}function Hr(e){let t=e.organicResults.length,n=e.diagnostics.pagination;n&&(n.requestedPages=2,n.capturedPages=1,n.page2Status="failed",n.page2OrganicCount=0),e.diagnostics.resultQuality="partial",e.diagnostics.degradedResult=!0,e.diagnostics.degradationReasons=["page_two_missing"];let r=e.diagnostics;if(r.serpCompleteness){let o=r.serpCompleteness.pagination.evidence.find(a=>a.page===1);r.serpCompleteness.pagination={...L(2,[{page:1,attempted:!0,captured:!0,resultCount:t,requestOffset:0,providerRequestIdObserved:o?.providerRequestIdObserved??!1},{page:2,attempted:!0,captured:!1,resultCount:0,requestOffset:10,providerRequestIdObserved:!1,failureCode:"provider_error"}]),providerRequestCount:r.serpCompleteness.pagination.providerRequestCount}}}async function Ur(e){if(e.pages===2){let o=await He({...e,mode:"light",pages:2}),a=Object.fromEntries(["localPack","forums","videos","aiOverview","aiMode","whatPeopleSaying","entityIds","paaPreview","topStories"].map(i=>[i,"unsupported"]));o.result.fullSerpFeatures={status:a,paaPreview:[],additional:[],pages:[]};let s=o.result.diagnostics;return s.serpCompleteness&&(s.serpCompleteness.features={localPack:"unsupported",forums:"unsupported",videos:"unsupported",aiOverview:"unsupported",whatPeopleSaying:"unsupported"}),{...o,mode:"full"}}let t=performance.now(),n={query:e.query,gl:"us",hl:"en",device:"desktop",pages:1,includeAllSerpFeatures:!0,includeLocalPack:!0,includeForums:!0,includeVideos:!0,includeAiOverview:!0,includeWhatPeopleSaying:!0,signal:e.signal},r=await De(n);if(r.organicResults.length===0)throw new Error("Full Search returned no organic results");return r.stats.durationMs=Math.round(performance.now()-t),r.diagnostics.phaseTimings&&(r.diagnostics.phaseTimings.totalServerMs=r.stats.durationMs),{result:r,mode:"full",deliveryProvider:"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061",deliveredPages:r.diagnostics.pagination?.capturedPages===2?2:1}}async function Ue(e,t,n=3){for(let r=1;r<=n;r++){try{let o=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(t),signal:AbortSignal.timeout(1e4)});if(o.ok)return;console.warn(`[webhook] attempt ${r} \u2192 ${o.status} from ${e}`)}catch(o){console.warn(`[webhook] attempt ${r} failed:`,o instanceof Error?o.message:o)}r<n&&await new Promise(o=>setTimeout(o,1e3*r*2))}console.error(`[webhook] gave up after ${n} attempts for ${e}`)}function Br(e){return e instanceof Error?e.message:String(e)}function Fr(e,t){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)}function Gr(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function Lr(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function Vr(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function $r(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function zr(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function Be(e){let t=Br(e);return e instanceof dt?{error_code:"request_aborted",error_type:"request_aborted",message:M("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof ct||Gr(t)?{error_code:"captcha_exhausted",error_type:"captcha",message:M("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Tt(e,t)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:M("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof pt?{error_code:"location_mismatch",error_type:"location_mismatch",message:M("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Lr(t)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:M("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Vr(t)?{error_code:"proxy_unavailable",error_type:"connection",message:M("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:zr(t)?{error_code:"browser_session_interrupted",error_type:"extraction",message:M("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Fr(e,t)?{error_code:"harvest_timeout",error_type:"timeout",message:M("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:$r(t)?{error_code:"service_unavailable",error_type:"service_unavailable",message:Se,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:Se,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function Fe(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function fe(e,t={}){let n=ut({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:t.retryAfterSeconds,chargeStatus:t.chargeStatus,details:t.details});return{error:n.message,...n}}function Ge(e,t={}){let{error:n,...r}=fe(e,t);return r}function Co(e,t={}){let n=Be(e);return{body:fe(n,t),status:n.httpStatus}}import{createHash as Kr}from"crypto";var Jr=1e9;function jr(e){return Kr("sha256").update(JSON.stringify(e)).digest("hex")}function he(e){return`harvest:${e}`}async function qo(e){let t=he(e.jobId);try{return(await Pt({id:t,userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,idempotencyKey:`job:${e.jobId}`,requestFingerprint:jr(e.normalizedFlags),jobId:e.jobId,requestId:e.requestId??null,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id}catch(n){return console.warn(JSON.stringify({event:"operation_run_start_failed",job_id:e.jobId,tool:e.tool,message:n instanceof Error?n.message:String(n)})),null}}async function Ve(e,t,n){if(e)try{await Et({runId:e,status:t,failureClass:n})}catch(r){console.warn(JSON.stringify({event:"operation_run_finish_failed",operation_run_id:e,message:r instanceof Error?r.message:String(r)}))}}async function Ho(e,t,n,r){if(!(!e||!t))try{await Ot({runId:e,billingTable:"billing_debits",billingKey:t,relation:n,amountMc:r})}catch(o){console.warn(JSON.stringify({event:"operation_billing_link_failed",operation_run_id:e,relation:n,message:o instanceof Error?o.message:String(o)}))}}function Le(e){return e==="timeout"?"timed_out":e==="request_aborted"?"cancelled":e==="browser_session_interrupted"?"interrupted":e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"||e==="location_mismatch"||e==="error"?"failed":"succeeded"}function Qr(e){return e==="\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061":"\u006b\u0065\u0072\u006e\u0065\u006c"}function tr(e,t,n=0){let r=new Map,o=null;return async a=>{if(e)try{if(a.type==="started"){let i=n+a.attemptNumber;if(i>1&&!o){let p=(await It(e))?.attempts.at(-1);o=p?.id==null?null:String(p.id)}let l=await $({runId:e,attemptNumber:i,method:a.method,provider:a.provider,parentAttemptId:i===1?null:o,edgeKind:i===1?null:n>0&&a.attemptNumber===1?"durable_resume":a.retryKind==="provider_fallback"?"provider_fallback":"same_method_retry",retryKind:i===1?null:n>0&&a.attemptNumber===1?"durable_resume":a.retryKind,retryReason:i===1?null:n>0&&a.attemptNumber===1?"durable_worker_resume":a.retryReason,retryOwner:i===1?null:n>0&&a.attemptNumber===1?"durable_reconciler":"harvest",normalizedFlags:{maxAttempts:a.maxAttempts,maxQuestions:a.maxQuestions,hasLocation:!!a.location,plannedProvider:a.provider,planReason:a.planReason??null,planTimeoutMs:a.planTimeoutMs??null},startedAt:a.startedAt}),u=await H({attemptId:l,provider:a.provider,providerProduct:a.provider==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",billingTier:a.provider==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?Z("https://www.google.com/search"):null,accountScope:t,method:a.method,providerStatus:"started"});r.set(a.attemptNumber,{attemptId:l,receiptId:u.id,plannedProvider:a.provider,observedProvider:null,providerMismatch:!1,method:a.method,startedAtMs:Date.parse(a.startedAt),billingMode:null}),o=l;return}let s=r.get(a.attemptNumber);if(!s)throw new Error(`Missing operation attempt state for attempt ${a.attemptNumber}`);if(a.type==="provider_session_started"){let i=Qr(a.provider);if(s.observedProvider=i,s.billingMode=a.billingMode??null,i!==s.plannedProvider){s.providerMismatch=!0,await B({receiptId:s.receiptId,costState:"unavailable",providerStatus:"provider_mismatch",errorCode:"provider_mismatch",errorMessage:`Planned ${s.plannedProvider}; observed ${i}`,finalizedAt:a.observedAt});let l=i==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",u=await H({attemptId:s.attemptId,provider:i,providerProduct:i==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",billingTier:i==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?Z("https://www.google.com/search"):null,accountScope:t,method:l,providerStatus:"observed_provider_mismatch"});s.receiptId=u.id,s.method=l}await U(s.receiptId,a.providerSessionId);return}if(await z({attemptId:s.attemptId,status:s.providerMismatch?"failed":Le(a.outcome),errorCode:s.providerMismatch?"provider_mismatch":Le(a.outcome)==="succeeded"?null:a.outcome,errorMessage:a.error,durationMs:a.durationMs,endedAt:a.completedAt}),(s.observedProvider??s.plannedProvider)==="\u006b\u0065\u0072\u006e\u0065\u006c"){let i=s.billingMode==="headful",l=s.billingMode==="headless"||s.billingMode==="headful"?vt(a.durationMs,i):null;await B({receiptId:s.receiptId,costState:"estimated",units:a.durationMs/1e3,unitType:"wall_open_second",amountUsdNanos:l==null?null:Math.round(l*Jr),providerProduct:"\u006b\u0065\u0072\u006e\u0065\u006c_browser",billingTier:s.billingMode,billableEvent:"provider_active_gb_second",measurementSource:"wall_open_time_proxy",rateSource:l!=null?`\u006b\u0065\u0072\u006e\u0065\u006c_${s.billingMode}_usd_per_second:${i?wt:bt}`:null,providerStatus:s.providerMismatch?"provider_mismatch":a.outcome,providerDurationMs:a.durationMs,errorCode:s.providerMismatch?"provider_mismatch":Le(a.outcome)==="succeeded"?null:a.outcome,errorMessage:a.error,finalizedAt:a.completedAt})}}catch(s){console.warn(JSON.stringify({event:"operation_attempt_telemetry_failed",operation_run_id:e,attempt_number:a.attemptNumber,message:s instanceof Error?s.message:String(s)}))}}}function rr(e,t,n=null,r="production-default",o=0){let a=tr(n,r,o);return async s=>{if(await a(s),s.type==="started"){await mt({jobId:e,userId:t,attemptNumber:s.attemptNumber,maxAttempts:s.maxAttempts,query:s.query,location:s.location,maxQuestions:s.maxQuestions,startedAt:s.startedAt});return}if(s.type==="provider_session_started"){await gt({jobId:e,attemptNumber:s.attemptNumber,browserProvider:s.provider,providerSessionId:s.providerSessionId,observedAt:s.observedAt});return}await ft({jobId:e,attemptNumber:s.attemptNumber,outcome:s.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:s.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:s.questionCount,durationMs:s.durationMs,error:s.error,willRetry:s.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:s.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:s.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:s.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:s.cleanup.browserCloseSucceeded,browserCloseError:s.cleanup.browserCloseError,browserProvider:s.cleanup.provider??null,providerSessionId:s.cleanup.providerSessionId??null,providerLookupStatus:s.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:s.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:s.cleanup.providerDisconnectMessage??null,providerReasonSource:s.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:s.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:s.cleanup.providerDisconnectMessage??null,debug:s.debug,completedAt:s.completedAt})}}var or=2,$e=0;function nr(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function sr(e){return Ce.paa_base+Math.max(1,e)*Ce.paa}async function ar(e){$e++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,n={value:null},r=null,o=t.serpOnly&&!t.serpIdentity,a=he(e.id),s=process.env.APIFY_API_TOKEN?.trim();if(o&&(t.pages===2?!s:!zt()||t.mode!=="full"&&!s))throw new Error("Search providers are not configured");let i=await yt({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:n,operationRunId:a,operationAccountScope:"production-default"},()=>o?He({query:t.query??e.query,pages:t.pages===2?2:1,token:s,mode:t.mode,operationRunId:a,accountScope:"production-default",signal:AbortSignal.timeout(it(t.maxQuestions??30,!0,t.mode,t.pages).serverMs)}).then(u=>(r=u,u.result)):At({...t,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:lt(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:rr(e.id,e.user_id,a)}));if(o&&!r)throw new Error("Search provider billing receipt is missing");await ht(e.id,i),await Ve(a,"succeeded");let l=await Re(e.id,e.user_id);if(typeof t.billingHoldMc=="number"&&t.billingDebitKey){let u=t.serpOnly?o?xe(r.deliveredPages,r.deliveryProvider):Te(n.value):sr(nr(i));await ke(Number(e.user_id),t.billingDebitKey,u,t.serpOnly?"serp_refund":"paa_refund",t.serpOnly?"SERP search settlement":"PAA harvest settlement",t.serpOnly?"serp_search":"paa_harvest")}else if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let u=sr(nr(i)),c=t.billingHoldMc-u;c>0?await ie(e.user_id,c,"paa_refund","overestimate refund"):c<0&&await Ee(e.user_id,-c,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let u=o?xe(r.deliveredPages,r.deliveryProvider):Te(n.value),c=t.billingHoldMc-u;c>0?await ie(e.user_id,c,"serp_refund","headless-mode pricing settle"):c<0&&await Ee(e.user_id,-c,"serp",t.query??e.query)}e.callback_url&&await Ue(e.callback_url,{job_id:e.id,status:"done",result:Ct(i),attempts:Ie(l)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let n=Be(t);await Ve(he(e.id),n.error_code==="harvest_timeout"?"timed_out":n.terminalStatus,n.error_code);let r=typeof e.options=="string"?JSON.parse(e.options):e.options,o=typeof r.billingHoldMc=="number"&&r.billingHoldMc>0;await Pe(e.id,Fe(n),Ge(n,{chargeStatus:o?"refund_pending":"not_charged"}));let a=await Re(e.id,e.user_id);try{o&&(r.billingDebitKey?await ke(Number(e.user_id),r.billingDebitKey,0,r.serpOnly?"serp_refund":"paa_refund","failed call",r.serpOnly?"serp_search":"paa_harvest"):await ie(e.user_id,r.billingHoldMc,"refund","failed call"),await Pe(e.id,Fe(n),Ge(n,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await Ue(e.callback_url,{job_id:e.id,status:"failed",...fe(n),attempts:Ie(a)})}finally{$e--}}async function Wr(){let e=await Ae();if(!e)return{claimed:!1};let t=Date.now();return await ar(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function tn(e){let t=[];for(let n=0;n<e.maxJobs&&!(Date.now()>=e.deadlineMs);n++){let r=await Wr();if(t.push(r),!r.claimed)break}return t}function rn(){setInterval(async()=>{if($e>=or)return;let e=await Ae();e&&ar(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${or} concurrent`)}export{Z as a,ce as b,br as c,ro as d,zt as e,He as f,Be as g,Fe as h,fe as i,Ge as j,Co as k,he as l,qo as m,Ve as n,Ho as o,rr as p,Ue as q,Wr as r,tn as s,rn as t};

package/dist/worker-P2BICG36.js

@@ -1 +0,0 @@
-import{r as a,s as b,t as c}from"./chunk-ZG52SKGD.js";import"./chunk-XG6GCEUE.js";import"./chunk-M5VZVMPC.js";import"./chunk-M5QHXNFZ.js";import"./chunk-DT2FYN6N.js";import"./chunk-ORB4RHCK.js";import"./chunk-UN6FVDJQ.js";import"./chunk-SH5KB4P7.js";import"./chunk-2SP57VCG.js";import"./chunk-6DTXIZY2.js";import"./chunk-Y6MKMSOC.js";import"./chunk-WJ4XFLS4.js";export{b as drainQueue,c as startWorker,a as tickOnce};