mcp-scraper 0.93.2 → 0.93.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -26
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-2SP57VCG.js +15 -15
- package/dist/{chunk-CG2FHIBA.js → chunk-4O2FOBZR.js} +1 -1
- package/dist/{chunk-5VB3I7UX.js → chunk-DFJT2YX6.js} +1 -1
- package/dist/chunk-DT2FYN6N.js +1 -1
- package/dist/{chunk-KPDNM7E7.js → chunk-IK5BG7MO.js} +1 -1
- package/dist/{chunk-VVG7LFFF.js → chunk-M5QHXNFZ.js} +3 -3
- package/dist/{chunk-MJ2OICVY.js → chunk-M5VZVMPC.js} +3 -3
- package/dist/chunk-ORB4RHCK.js +4 -4
- package/dist/{chunk-QWSWHUOK.js → chunk-OYJ4HES6.js} +4 -4
- package/dist/chunk-POASCYQ7.js +1 -0
- package/dist/chunk-SH5KB4P7.js +1 -1
- package/dist/chunk-WJ4XFLS4.js +15 -15
- package/dist/{chunk-EWR7BPPD.js → chunk-XG6GCEUE.js} +1 -1
- package/dist/chunk-Y6MKMSOC.js +1 -1
- package/dist/chunk-ZG52SKGD.js +6 -0
- package/dist/{extract-bundle-MQOAKQDV.js → extract-bundle-C3N6E6V7.js} +1 -1
- package/dist/index.cjs +21 -21
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/index.js +1 -1
- package/dist/{server-GRB6VNT6.js → server-HAV33FP7.js} +42 -42
- package/dist/{worker-US6CTFYG.js → worker-P2BICG36.js} +1 -1
- package/package.json +17 -136
- package/dist/chunk-73LAMUS5.js +0 -6
- package/dist/chunk-XJ6PJWT5.js +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
import{r as a,s as b,t as c}from"./chunk-
|
|
1
|
+
import{r as a,s as b,t as c}from"./chunk-ZG52SKGD.js";import"./chunk-XG6GCEUE.js";import"./chunk-M5VZVMPC.js";import"./chunk-M5QHXNFZ.js";import"./chunk-DT2FYN6N.js";import"./chunk-ORB4RHCK.js";import"./chunk-UN6FVDJQ.js";import"./chunk-SH5KB4P7.js";import"./chunk-2SP57VCG.js";import"./chunk-6DTXIZY2.js";import"./chunk-Y6MKMSOC.js";import"./chunk-WJ4XFLS4.js";export{b as drainQueue,c as startWorker,a as tickOnce};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mcp-scraper",
|
|
3
|
-
"version": "0.93.
|
|
3
|
+
"version": "0.93.4",
|
|
4
4
|
"description": "MCP server for MCP Scraper web intelligence tools",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|
|
@@ -23,109 +23,11 @@
|
|
|
23
23
|
"mcp-scraper-install": "dist/bin/mcp-scraper-install.js",
|
|
24
24
|
"mcp-scraper-cli": "dist/bin/mcp-scraper-cli.js"
|
|
25
25
|
},
|
|
26
|
-
"files": [
|
|
27
|
-
"dist/*.js",
|
|
28
|
-
"dist/*.cjs",
|
|
29
|
-
"dist/*.d.ts",
|
|
30
|
-
"dist/*.d.cts",
|
|
31
|
-
"dist/bin/*.js",
|
|
32
|
-
"dist/editorial-reading-room/assets",
|
|
33
|
-
"public/agent-install.md",
|
|
34
|
-
"public/.well-known/xray-install.json",
|
|
35
|
-
"README.md",
|
|
36
|
-
"CHANGELOG.md"
|
|
37
|
-
],
|
|
38
|
-
"repository": {
|
|
39
|
-
"type": "git",
|
|
40
|
-
"url": "git+https://github.com/VilovietaSEO/mcp-scraper.git"
|
|
41
|
-
},
|
|
42
26
|
"homepage": "https://mcpscraper.dev",
|
|
43
|
-
"
|
|
44
|
-
"
|
|
45
|
-
},
|
|
46
|
-
"scripts": {
|
|
47
|
-
"build": "tsup",
|
|
48
|
-
"build:md": "node scripts/build-md-mirror.mjs",
|
|
49
|
-
"build:mcpb": "npm run build && node scripts/build-mcpb.mjs",
|
|
50
|
-
"build:mcpb:candidate": "npm run build && node scripts/build-mcpb.mjs --candidate",
|
|
51
|
-
"build:rates": "node --import tsx scripts/gen-rates.ts",
|
|
52
|
-
"build:geotargets": "node --import tsx scripts/gen-geotargets.ts",
|
|
53
|
-
"build:blog": "node scripts/run-ts-sandbox-safe.mjs scripts/build-blog.ts",
|
|
54
|
-
"mcp:manifest": "node scripts/run-ts-sandbox-safe.mjs scripts/generate-mcp-tool-manifest.ts",
|
|
55
|
-
"mcp:cost-coverage": "node scripts/generate-operation-coverage.mjs",
|
|
56
|
-
"mcp:cost-coverage:check": "node scripts/generate-operation-coverage.mjs --check",
|
|
57
|
-
"mcp:cost-closure:check": "node scripts/verify-operation-cost-closure.mjs",
|
|
58
|
-
"paa:benchmark:dry-run": "node --import tsx scripts/paa-provider-benchmark.ts --dry-run",
|
|
59
|
-
"contracts:memory": "node --import tsx scripts/generate-memory-tool-contract.ts",
|
|
60
|
-
"contracts:memory-governed": "node --import tsx scripts/generate-governed-memory-mcp-schemas.ts",
|
|
61
|
-
"verify:crm-authority": "node --import tsx scripts/verify-crm-authority-boundary.ts",
|
|
62
|
-
"mcp:memory-image-schemas": "node scripts/generate-memory-image-mcp-schemas.mjs",
|
|
63
|
-
"mcp:measure-catalog": "node scripts/run-ts-sandbox-safe.mjs scripts/measure-mcp-catalog.ts",
|
|
64
|
-
"mcp:lint-tools": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts",
|
|
65
|
-
"mcp:lint-tools:write": "node scripts/run-ts-sandbox-safe.mjs scripts/lint-mcp-tool-craft.ts --write",
|
|
66
|
-
"xray:install-contract": "node --import tsx scripts/generate-xray-install-contract.ts",
|
|
67
|
-
"xray:install-contract:check": "node --import tsx scripts/generate-xray-install-contract.ts --check",
|
|
68
|
-
"api": "node scripts/run-ts-sandbox-safe.mjs bin/api-server.ts",
|
|
69
|
-
"dev": "node scripts/run-ts-sandbox-safe.mjs src/cli.ts",
|
|
70
|
-
"test": "vitest run tests/unit tests/contract",
|
|
71
|
-
"test:live": "vitest run --config vitest.live.config.ts",
|
|
72
|
-
"test:site-similarity": "node --import tsx scripts/run-site-similarity-live.ts",
|
|
73
|
-
"reanalyze:site-similarity": "node --import tsx scripts/reanalyze-site-similarity-zip.ts",
|
|
74
|
-
"test:headless-harvest": "node scripts/run-ts-sandbox-safe.mjs tests/live/harness/headless-harvest-canary.ts",
|
|
75
|
-
"test:integration": "node scripts/run-integration-tests.mjs",
|
|
76
|
-
"test:smoke": "vitest run tests/smoke",
|
|
77
|
-
"test:stripe": "node scripts/stripe-setup-smoke.mjs",
|
|
78
|
-
"typecheck": "tsc --noEmit",
|
|
79
|
-
"test:kernel": "vitest run tests/integration/kernel.smoke.test.ts",
|
|
80
|
-
"sync:env": "bash scripts/push-env.sh",
|
|
81
|
-
"sync:env:apply": "bash scripts/push-env.sh --apply",
|
|
82
|
-
"locations:publish-geonames": "node --import tsx scripts/publish-geonames-location-data.ts",
|
|
83
|
-
"locations:sync": "node --import tsx scripts/sync-hosted-location-data.ts",
|
|
84
|
-
"integrations:migrate-scheduler": "node --import tsx scripts/migrate-scheduler-integrations.ts",
|
|
85
|
-
"test:webhook": "node scripts/test-webhook.mjs",
|
|
86
|
-
"test:e2e": "playwright test",
|
|
87
|
-
"email:preview": "node scripts/run-ts-sandbox-safe.mjs scripts/email/render-previews.ts",
|
|
88
|
-
"email:preflight": "node --import tsx scripts/email/preflight.ts",
|
|
89
|
-
"email:avatar": "node --import tsx scripts/email/generate-sender-avatar.ts",
|
|
90
|
-
"email:inbound:setup": "node --import tsx scripts/email/setup-resend-inbound.ts",
|
|
91
|
-
"email:audience": "node scripts/run-ts-sandbox-safe.mjs scripts/email/sync-resend-audience.ts",
|
|
92
|
-
"email:drafts": "node scripts/run-ts-sandbox-safe.mjs scripts/email/create-resend-drafts.ts",
|
|
93
|
-
"email:workshop-reminders": "node --import tsx scripts/email/refresh-workshop-reminders.ts",
|
|
94
|
-
"email:workshop-reminders:apply": "node --import tsx scripts/email/refresh-workshop-reminders.ts --apply",
|
|
95
|
-
"billing:preview-credit-lot-repair": "node --import tsx scripts/billing/preview-credit-lot-repair.ts",
|
|
96
|
-
"billing:apply-credit-lot-repair": "node --import tsx scripts/billing/apply-credit-lot-repair.ts",
|
|
97
|
-
"brightdata:usage": "node scripts/brightdata-usage-report.mjs",
|
|
98
|
-
"brightdata:premium-domains:refresh": "node scripts/refresh-brightdata-premium-domains.mjs",
|
|
99
|
-
"brightdata:test:bandwidth": "node --import tsx scripts/brightdata-paa-bandwidth-test.ts",
|
|
100
|
-
"db:migrate": "node --import tsx scripts/run-core-migration.ts",
|
|
101
|
-
"db:verify": "node --import tsx scripts/verify-core-schema.ts",
|
|
102
|
-
"release:server:predeploy": "npm run db:migrate && npm run db:verify && npm run version:check:core",
|
|
103
|
-
"oauth-store:migrate": "node --import tsx scripts/migrate-oauth-store.ts --apply-schema",
|
|
104
|
-
"oauth-store:verify": "node --import tsx scripts/migrate-oauth-store.ts --verify",
|
|
105
|
-
"oauth-store:test": "vitest run tests/unit/oauth-store.test.ts",
|
|
106
|
-
"oauth-store:test-remote": "OAUTH_STORE_REMOTE_REQUIRED=1 vitest run --config vitest.live.config.ts tests/live/oauth-store-remote.live.test.ts",
|
|
107
|
-
"oauth:prepare-production-canary": "node --import tsx scripts/prepare-mcp-oauth-production-canary.ts",
|
|
108
|
-
"oauth:verify-live": "node scripts/verify-mcp-oauth-live.mjs",
|
|
109
|
-
"analytics:migrate": "node --import tsx scripts/run-analytics-migration.ts",
|
|
110
|
-
"analytics:contract": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts",
|
|
111
|
-
"analytics:contract:check": "node --import tsx scripts/analytics/export-analytics-rest-contract.ts --check",
|
|
112
|
-
"analytics:rollups": "node --import tsx scripts/run-analytics-rollups.ts",
|
|
113
|
-
"test:mcp-output-contract": "vitest run --config vitest.mcp-output-contract.config.ts tests/unit/golden-payloads.test.ts tests/unit/mcp-formatter-structured.test.ts tests/unit/mcp-2026-stdio.test.ts",
|
|
114
|
-
"version:check": "node scripts/verify-release-version-sync.mjs",
|
|
115
|
-
"version:check:core": "node scripts/verify-release-version-sync.mjs --core",
|
|
116
|
-
"prebuild": "npm run version:check:core",
|
|
117
|
-
"mcp:verify": "npm run xray:install-contract:check && npm run build && npm run mcp:manifest && npm run mcp:cost-coverage:check && npm run mcp:cost-closure:check && npm run version:check && npm run build:rates && npm run mcp:measure-catalog && npm run mcp:lint-tools && npm run test:mcp-output-contract && npm run typecheck && npm test && npm run package:verify-entrypoints",
|
|
118
|
-
"prepack": "npm run mcp:verify",
|
|
119
|
-
"package:verify-entrypoints": "node scripts/verify-package-entrypoints.mjs",
|
|
120
|
-
"product:install": "node scripts/run-product-command.mjs install",
|
|
121
|
-
"product:build": "node scripts/run-product-command.mjs build",
|
|
122
|
-
"product:typecheck": "node scripts/run-product-command.mjs typecheck",
|
|
123
|
-
"product:test": "node scripts/run-product-command.mjs test",
|
|
124
|
-
"product:verify-layout": "node scripts/verify-product-repository-layout.mjs",
|
|
125
|
-
"professional:verify": "node scripts/professional-verify.mjs",
|
|
126
|
-
"package:public": "npm run mcp:verify && node scripts/build-public-package.mjs",
|
|
127
|
-
"prepublishOnly": "node -e \"throw new Error('Publish the verified archive produced by npm run package:public')\""
|
|
27
|
+
"engines": {
|
|
28
|
+
"node": ">=20.18.1"
|
|
128
29
|
},
|
|
30
|
+
"license": "ISC",
|
|
129
31
|
"dependencies": {
|
|
130
32
|
"@anthropic-ai/sdk": "^0.96.0",
|
|
131
33
|
"@fal-ai/client": "^1.10.1",
|
|
@@ -165,38 +67,17 @@
|
|
|
165
67
|
"youtube-transcript": "^1.3.1",
|
|
166
68
|
"zod": "4.4.3"
|
|
167
69
|
},
|
|
168
|
-
"
|
|
169
|
-
"
|
|
170
|
-
"
|
|
171
|
-
"
|
|
172
|
-
"
|
|
173
|
-
"
|
|
174
|
-
"
|
|
175
|
-
"
|
|
176
|
-
"
|
|
177
|
-
"
|
|
178
|
-
"
|
|
179
|
-
"
|
|
180
|
-
|
|
181
|
-
"dotenv": "^17.4.2",
|
|
182
|
-
"esbuild": "^0.27.7",
|
|
183
|
-
"fflate": "^0.7.4",
|
|
184
|
-
"jsdom": "^29.1.1",
|
|
185
|
-
"tsup": "^8.0.0",
|
|
186
|
-
"tsx": "^4.0.0",
|
|
187
|
-
"typescript": "^5.4.0",
|
|
188
|
-
"vitest": "^1.6.0",
|
|
189
|
-
"youtubei.js": "^17.0.1",
|
|
190
|
-
"@onkernel/sdk": "^0.93.0"
|
|
191
|
-
},
|
|
192
|
-
"engines": {
|
|
193
|
-
"node": ">=20.18.1"
|
|
194
|
-
},
|
|
195
|
-
"directories": {
|
|
196
|
-
"doc": "docs",
|
|
197
|
-
"test": "tests"
|
|
198
|
-
},
|
|
199
|
-
"keywords": [],
|
|
200
|
-
"author": "",
|
|
201
|
-
"license": "ISC"
|
|
70
|
+
"files": [
|
|
71
|
+
"dist/*.js",
|
|
72
|
+
"dist/*.cjs",
|
|
73
|
+
"dist/*.d.ts",
|
|
74
|
+
"dist/*.d.cts",
|
|
75
|
+
"dist/bin/*.js",
|
|
76
|
+
"dist/editorial-reading-room/assets",
|
|
77
|
+
"public/agent-install.md",
|
|
78
|
+
"public/.well-known/xray-install.json",
|
|
79
|
+
"README.md",
|
|
80
|
+
"CHANGELOG.md",
|
|
81
|
+
"THIRD_PARTY_NOTICES.html"
|
|
82
|
+
]
|
|
202
83
|
}
|
package/dist/chunk-73LAMUS5.js
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import{a as Tt}from"./chunk-EWR7BPPD.js";import{l as Rt,m as At,w as Pt}from"./chunk-MJ2OICVY.js";import{a as G,c as Ie,d as L,e as V,f as ie,j as Ce,k as Ct}from"./chunk-DT2FYN6N.js";import{b as Et,c as kt,d as $,f as z,g as H,h as U,i as le,k as B,n as Ot,q as It}from"./chunk-ORB4RHCK.js";import{a as Te,b as xe,g as Me}from"./chunk-UN6FVDJQ.js";import{a as lt,b as ut}from"./chunk-SH5KB4P7.js";import{a as wt,b as vt,d as St}from"./chunk-2SP57VCG.js";import{b as _t,c as bt}from"./chunk-6DTXIZY2.js";import{c as it}from"./chunk-Y6MKMSOC.js";import{Nb as ae,Rb as ke,Tb as Oe,Va as Pe,b as Re,g as N,h as ct,hb as yt,ia as gt,j as dt,ja as ft,jb as Ee,la as ht,m as pt,o as mt,oa as Ae}from"./chunk-WJ4XFLS4.js";import{request as br}from"https";import{HttpsProxyAgent as wr}from"https-proxy-agent";var ue={source:"https://brightdata.com/users/zone/premium_domains",retrievedAt:"2026-09-23T20:14:30.483Z",includedStatuses:["premium","remove_candidate"],domains:["advanceautoparts.com","affitto.it","agoda.cn","albertsons.com","allpeople.com","autozone.com","bestbuy.com","bestwestern.com","billiger.de","bottlerover.com","carousell.com","carousell.com.hk","carousell.com.my","carousell.ph","carousell.sg","carsales.com.au","cdiscount.com","chewy.com","costco.com","cvs.com","despegar.com.mx","dickssportinggoods.com","dynos.es","emaxme.com","familytreenow.com","feuvert.fr","flooranddecor.com","foodlion.com","footlocker.co.uk","footlocker.com","giantfoodstores.com","gopuff.com","gplay.bg","hermes.com","hyatt.com","idealo.de","immobilienscout24.de","ingatlan.com","instacart.com","intersport.fr","joann.com","kroger.com","lazada.co.id","lazada.co.th","lazada.com.my","lazada.com.ph","lazada.sg","lazada.vn","lowes.ca","lowes.com","mcmaster.com","mediamarkt.de","mediamarkt.es","medline.com","mscdirect.com","napaonline.com","nofrills.ca","peoplefinders.com","platt.com","publicdatausa.com","realcanadiansuperstore.ca","realestate.com.au","restaurantguru.com","searchpeoplefree.com","shopee.cl","shopee.co.id","shopee.co.th","shopee.com.br","shopee.com.co","shopee.com.mx","shopee.com.my","shopee.ph","shopee.sg","shopee.tw","shopee.vn","similarweb.com","skyscanner.co.kr","skyscanner.net","stopandshop.com","target.com","temu.com","ticketmaster.com","totalwine.com","tractorsupply.com","walmart.com.mx","wayfair.com","weismarkets.com","wizzair.com","worten.pt"]};var Xr=ue.source,xt=ue.retrievedAt,yr=new Set(ue.domains);function Z(e){let t=new URL(e).hostname.toLowerCase().replace(/\.$/,"");for(let n of yr)if(t===n||t.endsWith(`.${n}`))return"premium";return"standard"}function ce(e){for(let t of["providerProduct","billingTier","billableEvent","unitType","source","policyVersion","effectiveAt"])if(!e[t]?.trim())throw new Error(`Provider rate ${t} is required`);if(!Number.isSafeInteger(e.numeratorUsdNanos)||e.numeratorUsdNanos<0)throw new Error("Provider rate numerator must be a non-negative safe integer");if(!Number.isSafeInteger(e.denominatorUnits)||e.denominatorUnits<1)throw new Error("Provider rate denominator must be a positive safe integer");if(Number.isNaN(Date.parse(e.effectiveAt)))throw new Error("Provider rate effectiveAt must be a date");return e}function de(e,t){if(ce(e),!Number.isSafeInteger(t)||t<0)throw new Error("Billable units must be a non-negative safe integer");let n=BigInt(e.numeratorUsdNanos)*BigInt(t),r=BigInt(e.denominatorUnits),o=(n*2n+r)/(r*2n);if(o>BigInt(Number.MAX_SAFE_INTEGER))throw new Error("Provider amount exceeds safe integer range");return Number(o)}function Mt(e=process.env){let t=e.BRIGHTDATA_SERP_CONTRACT_USD_PER_1000?.trim(),n=e.BRIGHTDATA_SERP_RATE_SOURCE?.trim(),r=e.BRIGHTDATA_SERP_RATE_POLICY_VERSION?.trim(),o=e.BRIGHTDATA_SERP_RATE_EFFECTIVE_AT?.trim();if(!t&&!n&&!r&&!o)return null;if(!t||!n||!r||!o)throw new Error("Bright Data SERP contract rate requires amount, source, policy version, and effective date");let a=Number(t);if(!Number.isFinite(a)||a<0)throw new Error("Invalid Bright Data SERP contract amount");return ce({providerProduct:"brightdata_serp_api",billingTier:"payg",billableEvent:"successful_delivery",unitType:"request",numeratorUsdNanos:Math.round(a*1e9),denominatorUnits:1e3,source:n,policyVersion:r,effectiveAt:o})}function _r(e){return ce({providerProduct:"brightdata_browser_api",billingTier:e,billableEvent:"transferred_byte",unitType:"byte",numeratorUsdNanos:(e==="premium"?11:8)*1e9,denominatorUnits:1e9,source:`owner_confirmed_payg_2026-09-23;brightdata_domain_snapshot_${xt}`,policyVersion:"brightdata-browser-domain-tier-2026-09-23",effectiveAt:"2026-09-23T00:00:00Z"})}function to(e){return _r(Z(e))}function Nt(){return ce({providerProduct:"apify_google_serp_scraperlink",billingTier:"store_pay_per_event",billableEvent:"serp_page",unitType:"page",numeratorUsdNanos:5e8,denominatorUnits:1e3,source:"https://apify.com/scraperlink/google-search-results-serp-scraper",policyVersion:"scraperlink-listing-2026-09-24",effectiveAt:"2026-09-24T00:00:00Z"})}var vr="https://api.brightdata.com/request",Sr=8e3,Rr="brd.superproxy.io",Ar="44445",Pr=10*1024*1024;async function Er(e,t){let n=new URL(`http://${t.host}:${t.port}`);n.username=t.username,n.password=t.password;let r=new wr(n);try{return await new Promise((o,a)=>{let s=br(e,{agent:r,headers:{"x-unblock-data-format":"parsed"},rejectUnauthorized:!1,signal:t.signal},i=>{let l=[],u=0;i.on("data",c=>{let p=Buffer.isBuffer(c)?c:Buffer.from(c);if(u+=p.length,u>Pr){s.destroy(new Error("Bright Data SERP response exceeded the maximum size"));return}l.push(p)}),i.on("end",()=>o({ok:(i.statusCode??500)>=200&&(i.statusCode??500)<300,status:i.statusCode??500,rawText:Buffer.concat(l).toString("utf8"),providerRequestId:typeof i.headers["x-response-id"]=="string"?i.headers["x-response-id"]:typeof i.headers["x-request-id"]=="string"?i.headers["x-request-id"]:null}))});s.on("error",()=>a(new Error("Bright Data native SERP request failed"))),s.end()})}finally{r.destroy()}}function zt(e=process.env){let t=!!(e.BRIGHTDATA_SERP_PROXY_USERNAME?.trim()&&e.BRIGHTDATA_SERP_PROXY_PASSWORD?.trim()),n=!!((e.BRIGHTDATA_SERP_API_KEY||e.BRIGHTDATA_API_KEY)?.trim()&&e.BRIGHTDATA_SERP_ZONE?.trim());return t||n}function m(e){return e!=null&&typeof e=="object"&&!Array.isArray(e)?e:null}function D(e){return Array.isArray(e)?e:[]}function De(e){return typeof e=="string"&&e.trim()?e.trim():null}function A(e){return typeof e=="number"&&Number.isFinite(e)?e:typeof e=="string"&&e.trim()&&Number.isFinite(Number(e))?Number(e):null}function g(e,t){for(let n of t){let r=De(e[n]);if(r)return r}return null}function J(e){try{return new URL(e).hostname.replace(/^www\./,"")}catch{return""}}function Kt(e){return{rawUrl:e,resolvedUrl:e,linkType:"plain",resolutionStatus:"not_needed"}}function kr(e){let t=e;if(typeof t=="string")try{t=JSON.parse(t)}catch{throw new Error("Bright Data SERP API returned invalid JSON")}let n=m(t);if(!n)throw new Error("Bright Data SERP API returned an invalid payload");if(typeof n.body=="string")try{let o=JSON.parse(n.body);if(m(o))return m(o)}catch{}let r=m(n.body);return r?m(r.result)??m(r.data)??r:m(n.result)??m(n.data)??n}function j(e,t){for(let r of t){let o=D(e[r]);if(o.length)return o}let n=m(e.results);if(n)for(let r of t){let o=D(n[r]);if(o.length)return o}return[]}function K(e,t){let n=[e,m(e.results)].filter(r=>!!r);for(let r of n)for(let o of t)if(Object.prototype.hasOwnProperty.call(r,o))return{observed:!0,rawShapeValid:Array.isArray(r[o]),rawCount:D(r[o]).length};return{observed:!1,rawShapeValid:!1,rawCount:0}}function Ne(e,t){let n=[e,m(e.results)].filter(r=>!!r);for(let r of n)for(let o of t){if(!Object.prototype.hasOwnProperty.call(r,o))continue;let a=m(r[o]);return{observed:!0,rawShapeValid:a!==null,rawCount:a&&Object.keys(a).length>0?1:0}}return{observed:!1,rawShapeValid:!1,rawCount:0}}function Dt(e){return j(e,["organic","organic_results","organicResults"]).flatMap((t,n)=>{let r=m(t);if(!r)return[];let o=g(r,["link","url","href"]),a=g(r,["title","name"]);if(!o||!a)return[];let s=m(r.rating)??m(r.inline_rating);return[{position:Math.max(1,Math.trunc(A(r.global_rank)??A(r.rank)??A(r.position)??n+1)),title:a,url:o,...Kt(o),domain:J(o),cite:g(r,["display_link","displayLink","cite"]),snippet:g(r,["description","snippet","text"]),isRedditStyle:J(o)==="reddit.com",inlineRating:s?{value:String(s.value??s.rating??""),count:String(s.count??s.reviews??"")}:null}]}).sort((t,n)=>t.position-n.position)}function qt(e){return j(e,["snack_pack","local","local_results","local_pack","localPack"]).flatMap((t,n)=>{let r=m(t);if(!r)return[];let o=g(r,["title","name"]);if(!o)return[];let a=D(r.extensions??r.metadata).map(De).filter(s=>!!s);return[{position:Math.max(1,Math.trunc(A(r.global_rank)??A(r.rank)??A(r.position)??n+1)),name:o,cid:g(r,["cid"]),rating:g(r,["rating"])??(A(r.rating)!=null?String(A(r.rating)):null),reviewCount:g(r,["reviews_cnt","reviews","review_count","reviewCount"])??(A(r.reviews_cnt)!=null?String(A(r.reviews_cnt)):A(r.reviews)!=null?String(A(r.reviews)):null),metadata:a,websiteUrl:g(r,["website","website_url","link"]),directionsUrl:g(r,["directions","directions_url"])}]})}function Jt(e){let t=m(e);return D(t?.citations??t?.sources??t?.references).flatMap(n=>{let r=m(n);if(!r)return[];let o=g(r,["link","url","href"]);return o?[{text:g(r,["title","text","name"])??J(o),href:o,...Kt(o)}]:[]})}function ge(e,t=0){if(t>4)return[];if(Array.isArray(e))return e.flatMap(a=>ge(a,t+1));let n=m(e);if(!n)return[];let r=g(n,["snippet","text","answer"]),o=ge(n.texts??n.list??n.items,t+1);return[...r?[r]:[],...o]}function Ht(e,t){if(!t)return{detected:!1,text:null,citations:[]};let n=m(e.ai_overview)??m(e.aiOverview)??m(e.overview);if(!n)return{detected:!1,text:null,citations:[]};let r=g(n,["text","answer","description"])??(ge(n.texts).join(`
|
|
2
|
-
|
|
3
|
-
`).slice(0,5e4)||null),o=Jt(n);return{detected:!!(r||o.length),text:r,citations:o}}function Ut(e,t){if(!t)return{detected:!1,text:null,citations:[]};let n=m(e.ai_mode)??m(e.aiMode);if(!n)return{detected:!1,text:null,citations:[]};let r=g(n,["text","answer","description"])??(ge(n.texts).join(`
|
|
4
|
-
|
|
5
|
-
`).slice(0,5e4)||null),o=Jt(n);return{detected:!!(r||o.length),text:r,citations:o}}function Bt(e,t){return t?[...j(e,["videos","video_results"]),...j(e,["short_videos","shorts"])].flatMap(n=>{let r=m(n);if(!r)return[];let o=g(r,["link","url"]),a=g(r,["title","name"]);return!o||!a?[]:[{type:"video",title:a,channel:g(r,["channel","source"])??"",platform:J(o),duration:g(r,["duration"])??"",url:o}]}):[]}function Ft(e,t){if(!t)return[];let n=m(e.forums);return[...j(e,["discussions","forums","forum_results"]),...D(n?.items)].flatMap(r=>{let o=m(r);if(!o)return[];let a=g(o,["link","url"]),s=g(o,["title","name"]);return!a||!s?[]:[{title:s,source:g(o,["source"])??J(a),url:a}]})}function Gt(e,t){return t?j(e,["what_people_are_saying","whatPeopleSaying"]).flatMap(n=>{let r=m(n);if(!r)return[];let o=g(r,["link","url"]),a=g(r,["title","name"]);if(!o||!a)return[];let s=g(r,["platform","source"])??J(o),i=s.toLowerCase();return[{type:i.includes("reddit")?"reddit":i.includes("facebook")?"facebook":i.includes("instagram")?"instagram":i.includes("tiktok")?"tiktok":i.includes("youtube")?"youtube":i.includes("news")?"news":"unknown",title:a,url:o,source:s,platform:s,popularComment:g(r,["comment","popular_comment"]),engagement:g(r,["engagement"])??"",date:g(r,["date"])??"",duration:g(r,["duration"]),authorNote:g(r,["author_note"])}]}):[]}var Or={featuredSnippets:["featured_snippets","featured_snippet","featuredSnippet"],knowledgePanel:["knowledge","knowledge_graph","knowledgeGraph"],topStories:["top_stories","topStories"],news:["news","news_results"],images:["images","image_results"],shopping:["shopping","shopping_results"],relatedSearches:["related","related_searches","relatedSearches"],recipes:["recipes","recipe_results"],jobs:["jobs","job_results"],topAds:["top_ads"],bottomAds:["bottom_ads"],topProducts:["top_pla"],bottomProducts:["bottom_pla"],jackpotProducts:["jackpot_pla"],shortVideos:["short_videos","shorts"],localMap:["snack_pack_map"],flights:["flights"],hotels:["hotels_selection"],popularProducts:["popular_products"],weather:["weather"],currencyExchange:["currency_exchange"],latestPosts:["latest_posts"],perspectives:["perspectives"],navigation:["navigation"],chips:["chips"],spelling:["spelling"]};function pe(e,t){for(let n of[e,m(e.results)])if(n){for(let r of t)if(Object.prototype.hasOwnProperty.call(n,r))return{observed:!0,value:n[r]}}return{observed:!1,value:null}}function me(e,t,n){return e?t==null||Array.isArray(t)&&t.length===0?"observed_absent":Array.isArray(t)?n>0?"observed_present":"incomplete":m(t)?n>0?"observed_present":Object.keys(m(t)).length===0?"observed_absent":"incomplete":"incomplete":"unknown"}function Ir(e,t){let n={},r=[["localPack",["snack_pack","local","local_results","local_pack","localPack"],t.localPack],["forums",["discussions","forums","forum_results"],t.forums],["videos",["videos","video_results","short_videos","shorts"],t.videos],["aiOverview",["ai_overview","aiOverview","overview"],t.aiOverview],["whatPeopleSaying",["what_people_are_saying","whatPeopleSaying"],t.whatPeopleSaying],["entityIds",["knowledge","knowledge_graph","knowledgeGraph"],t.entityIds]];for(let[l,u,c]of r){let p=pe(e,u);n[l]=me(p.observed,p.value,c)}let o=pe(e,["ai_mode","aiMode"]);n.aiMode=me(o.observed,o.value,t.aiMode);let a=pe(e,["people_also_ask","peopleAlsoAsk","related_questions","paa"]),s=D(a.value).slice(0,30).flatMap(l=>{let u=m(l);if(!u)return[];let c=g(u,["question","title"]);return c?[{question:c.slice(0,1e3),answer:g(u,["answer","snippet","text"])?.slice(0,1e4)??null,url:g(u,["link","url"])??null}]:[]});n.paaPreview=me(a.observed,a.value,s.length);let i=[];for(let[l,u]of Object.entries(Or)){let c=pe(e,u);if(!c.observed)continue;let p=m(c.value),w=(Array.isArray(c.value)?c.value:p&&(Array.isArray(p.items)||Array.isArray(p.posts))?[...D(p.items),...D(p.posts)]:[c.value]).slice(0,20).flatMap(S=>{let y=m(S);if(!y)return[];let C=g(y,["title","name","heading"]),k=g(y,["link","url","website"]),q=g(y,["description","snippet","text","summary","story"]);return!C&&!k&&!q?[]:[{title:C?.slice(0,1e3)??null,url:k,description:q?.slice(0,1e4)??null,position:A(y.global_rank)??A(y.rank)??A(y.position)??null}]});n[l]=me(c.observed,c.value,w.length);let R=Object.fromEntries(Object.entries(p??{}).slice(0,30).flatMap(([S,y])=>/^[a-zA-Z0-9_]{1,80}$/.test(S)&&(y===null||["string","number","boolean"].includes(typeof y))?[[S,typeof y=="string"?y.slice(0,2e3):y]]:[]));i.push({name:l,attributes:R,items:w})}return{status:n,paaPreview:s,additional:i}}function Lt(e,t){let n=new Set,r=new Set(t.map(s=>s.cid).filter(s=>!!s)),o=new Set,a=(s,i=0)=>{if(i>6||s==null)return;if(Array.isArray(s)){for(let u of s)a(u,i+1);return}let l=m(s);if(l)for(let[u,c]of Object.entries(l)){let p=u.toLowerCase().replace(/[^a-z]/g,""),_=De(c);_&&(p==="kgid"||p==="kgmid"||p==="mid")?n.add(_):_&&p==="cid"?r.add(_):_&&p==="gcid"?o.add(_):typeof c=="object"&&a(c,i+1)}};return a(e),{entities:[],kgIds:[...n],cids:[...r],gcids:[...o]}}function Vt(e,t){let n=e instanceof Error?e.message:String(e);return t.aborted||/timeout|timed out/i.test(n)?"timeout":/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(n)?"captcha":/invalid (?:json|payload)/i.test(n)?"invalid_payload":"provider_error"}function $t(e){return e==="invalid_payload"?"invalid_payload":e==="captcha"?"serp_readiness":e==="timeout"||e==="provider_error"?"provider_transport":"pagination"}async function qe(e,t=process.env,n=fetch,r=Er,o){if(e.pages!==1)throw new Error("Bright Data SERP supports one page only; use Apify for two pages");let a=(t.BRIGHTDATA_SERP_API_KEY||t.BRIGHTDATA_API_KEY)?.trim(),s=t.BRIGHTDATA_SERP_ZONE?.trim(),i=t.BRIGHTDATA_SERP_PROXY_USERNAME?.trim(),l=t.BRIGHTDATA_SERP_PROXY_PASSWORD?.trim(),u=!!(i&&l);if(!u&&(!a||!s))throw new Error("Direct SERP API is not configured");let c=Date.now(),p=bt(),_=null,w=e.location?At(e.location):null;if(e.location&&!w?.canonical)throw new G("query_integrity","location_unresolved",`SERP location could not be resolved without changing the query (candidates: ${w?.ambiguousCandidates??0})`);let R=w?.canonical?Rt(w.canonical):null,S=AbortSignal.timeout(e.includeAllSerpFeatures?33e3:Sr),y=e.signal?AbortSignal.any([e.signal,S]):S,C=1,k=[],q=[],Ke=0,Je=async d=>{let O=Date.now(),b=u?"brightdata_serp_native_proxy":"brightdata_serp_rest",I=null,x=null,Q=!1,W=!1,F=null,Y=null;if(p?.operationRunId&&(d===1||_))try{I=await $({runId:p.operationRunId,attemptNumber:o?o.attemptNumber+d-1:d,method:b,provider:"brightdata",...d===2?{parentAttemptId:_,edgeKind:"next_phase"}:o?{parentAttemptId:o.parentAttemptId,edgeKind:"provider_fallback",retryKind:"provider_fallback"}:{},normalizedFlags:{page:d,device:e.device,hasLocation:!!e.location}}),d===1&&(_=I),x=(await H({attemptId:I,provider:"brightdata",providerProduct:"brightdata_serp_api",accountScope:p.operationAccountScope??"production-default",method:b,providerStatus:"started"})).id}catch(h){throw console.warn(JSON.stringify({event:"direct_serp_telemetry_start_failed",page:d,message:h instanceof Error?h.message:String(h)})),h}let P=new URLSearchParams({q:e.query,gl:e.gl,hl:e.hl,pws:"0",num:"10",...d===2?{start:"10"}:{}}),se=it(e.recency);se&&P.set("tbs",se),e.device==="mobile"&&P.set("brd_mobile","1"),(e.includeAiOverview||e.includeAllSerpFeatures)&&P.set("brd_ai_overview","2"),R&&P.set("uule",R),u&&P.set("brd_json","1"),Ke+=1;let M;try{if(u){let E=t.BRIGHTDATA_SERP_PROXY_HOST?.trim()||Rr,X=t.BRIGHTDATA_SERP_PROXY_PORT?.trim()||Ar;M=await r(`https://www.google.com/search?${P}`,{username:i,password:l,host:E,port:X,signal:y})}else{let E=await n(vr,{method:"POST",headers:{Authorization:`Bearer ${a}`,"Content-Type":"application/json"},body:JSON.stringify({zone:s,url:`https://www.google.com/search?${P}`,format:"json",data_format:"parsed"}),signal:y});M={ok:E.ok,status:E.status,rawText:await E.text(),providerRequestId:E.headers.get("x-response-id")??E.headers.get("x-request-id")}}if(Y=`http_${M.status}`,x&&M.providerRequestId&&await U(x,M.providerRequestId),!M.ok)throw new Error(`Bright Data SERP API failed with HTTP ${M.status}`);Q=!0;let h;try{h=kr(JSON.parse(M.rawText))}catch(E){let X=E instanceof Error&&E.message!=="Unexpected end of JSON input"?E.message:"Bright Data SERP API returned invalid JSON";throw new G("invalid_payload","invalid_provider_payload",X)}let f=Dt(h);return k.push({page:d,attempted:!0,captured:f.length>0,resultCount:f.length,requestOffset:d===2?10:0,providerRequestIdObserved:!!M.providerRequestId,...f.length===0?{failureCode:"empty_page"}:{}}),W=!0,h}catch(h){F=h;let f=Vt(h,y);if(k.some(X=>X.page===d)||k.push({page:d,attempted:!0,captured:!1,resultCount:0,requestOffset:d===2?10:0,providerRequestIdObserved:!1,failureCode:f}),d===2)throw h;let E=h instanceof G?h.stage:$t(f);throw h instanceof G?h:new G(E,f,h instanceof Error?h.message:String(h))}finally{let h=null;if(x&&Q)try{let f=Mt(t);f?await B({receiptId:x,costState:"measured",units:1,unitType:"request",amountUsdNanos:de(f,1),rateSource:f.source,providerProduct:f.providerProduct,billingTier:f.billingTier,billableEvent:f.billableEvent,measurementSource:"provider_http_delivery",rateNumerator:f.numeratorUsdNanos,rateDenominator:f.denominatorUnits,ratePolicyVersion:f.policyVersion,rateEffectiveAt:f.effectiveAt,providerStatus:Y??"delivered",providerDurationMs:Date.now()-O}):await le({receiptId:x,units:1,unitType:"request",providerStatus:"delivered_rate_unconfigured"})}catch(f){console.warn(JSON.stringify({event:"direct_serp_receipt_finalize_failed",page:d,message:f instanceof Error?f.message:String(f)})),h=f}if(I&&await z({attemptId:I,status:W?"succeeded":y.aborted?"timed_out":"failed",errorCode:F?Vt(F,y):null,durationMs:Date.now()-O}).catch(f=>console.warn(JSON.stringify({event:"direct_serp_attempt_finish_failed",page:d,message:f instanceof Error?f.message:String(f)}))),h)throw h}};try{q.push({page:1,payload:await Je(1)})}catch(d){throw d}let je=null;if(C===2)try{q.push({page:2,payload:await Je(2)})}catch(d){je=d}let ir=Date.now()-c,lr=Date.now(),v=q[0].payload,ur=q.flatMap(({page:d,payload:O})=>Dt(O).map(b=>({...b,position:d===2&&b.position<=10?b.position+10:b.position,sourcePage:d}))).filter((d,O,b)=>b.findIndex(I=>Ie(I.url)===Ie(d.url))===O),Qe=e.includeLocalPack||e.includeAllSerpFeatures,_e=Qe?qt(v):[],be=K(v,["snack_pack","local","local_results","local_pack","localPack"]),cr=V({requested:Qe,moduleObserved:be.observed,rawCount:be.rawCount,mappedCount:_e.length,rawShapeValid:be.rawShapeValid}),We=e.includeAiOverview||e.includeAllSerpFeatures,Ye=Ht(v,We),Xe=e.includeForums||e.includeAllSerpFeatures,Ze=Ft(v,Xe),et=e.includeVideos||e.includeAllSerpFeatures,tt=Bt(v,et),rt=e.includeWhatPeopleSaying||e.includeAllSerpFeatures,ot=Gt(v,rt),we=m(v.forums),ve=we&&Array.isArray(we.items)?{observed:!0,rawCount:we.items.length,rawShapeValid:!0}:K(v,["discussions","forums","forum_results"]),Se=K(v,["videos","video_results","short_videos","shorts"]),nt={localPack:cr,forums:V({requested:Xe,moduleObserved:ve.observed,rawCount:ve.rawCount,mappedCount:Ze.length,rawShapeValid:ve.rawShapeValid}),videos:V({requested:et,moduleObserved:Se.observed,rawCount:Se.rawCount,mappedCount:tt.length,rawShapeValid:Se.rawShapeValid}),aiOverview:V({requested:We,moduleObserved:Ne(v,["ai_overview","aiOverview","overview"]).observed,rawCount:Ne(v,["ai_overview","aiOverview","overview"]).rawCount,mappedCount:Ye.detected?1:0,rawShapeValid:Ne(v,["ai_overview","aiOverview","overview"]).rawShapeValid}),whatPeopleSaying:V({requested:rt,moduleObserved:K(v,["what_people_are_saying","whatPeopleSaying"]).observed,rawCount:K(v,["what_people_are_saying","whatPeopleSaying"]).rawCount,mappedCount:ot.length,rawShapeValid:K(v,["what_people_are_saying","whatPeopleSaying"]).rawShapeValid})},dr=L(1,k),pr=k.find(d=>d.page===1)?.resultCount??0,oe=k.find(d=>d.page===2)?.resultCount??0,mr=ie({query:e.query,finalQuery:e.query,canonicalLocation:w?.canonical??null,locationResolutionSource:w?.source??"not_requested",locationAmbiguousCandidates:w?.ambiguousCandidates??0,uule:R,outboundProviderMethod:u?"brightdata_serp_native_proxy":"brightdata_serp_rest"}),ne=je?$t(k.find(d=>d.page===2)?.failureCode??"provider_error"):Object.values(nt).some(d=>d==="incomplete"||d==="unknown")?"optional_feature_parsing":null,gr={pagination:{...dr,providerRequestCount:Ke},features:nt,queryIntegrity:mr,failureStage:ne},st=e.includeAllSerpFeatures?(()=>{let d=q.map(({page:O,payload:b})=>{let I=qt(b),x=Ft(b,!0),Q=Bt(b,!0),W=Gt(b,!0),F=Ht(b,!0),Y=Ut(b,!0),P=Lt(b,I),se=Ir(b,{localPack:I.length,forums:x.length,videos:Q.length,aiOverview:F.detected?1:0,aiMode:Y.detected?1:0,whatPeopleSaying:W.length,entityIds:P.entities.length+P.kgIds.length+P.cids.length+P.gcids.length});return{page:O,...se,localPack:I,forums:x,videos:Q,whatPeopleSaying:W,aiOverview:F,aiMode:Y,entityIds:P}});return{status:d[0].status,paaPreview:d.flatMap(O=>O.paaPreview),additional:d.flatMap(O=>O.additional),pages:d}})():void 0,fr=new Date().toISOString(),at=Date.now()-c;return{seed:e.query,location:e.location??null,extractedAt:fr,diagnostics:{phaseTimings:{providerConnectMs:ir,serpParseMs:Date.now()-lr,totalServerMs:at},pagination:{requestedPages:C,capturedPages:C===2&&oe>0?2:1,page2Status:C===2?oe>0?"captured":"failed":"not_requested",page1OrganicCount:pr,page2OrganicCount:oe,...C===2&&oe===0?{failureCode:k.find(d=>d.page===2)?.failureCode==="captcha"?"captcha":k.find(d=>d.page===2)?.failureCode==="timeout"?"timeout":"empty_page"}:{}},completionStatus:"serp_only",problem:null,resultQuality:ne?"partial":"complete",degradedResult:!!ne,degradationReasons:[],retryRecommended:!!ne,serpCompleteness:gr},totalQuestions:0,surface:"web",aiOverview:Ye,aiMode:Ut(v,e.includeAllSerpFeatures),...st?{fullSerpFeatures:st}:{},whatPeopleSaying:ot,tree:[],flat:[],videos:tt,forums:Ze,organicResults:ur,localPack:_e,entityIds:Lt(v,_e),stats:{seed:e.query,totalQuestions:0,maxDepthReached:0,durationMs:at,errorCount:0}}}var Cr="scraperlink~google-search-results-serp-scraper",ee="https://api.apify.com/v2";var T=class extends Error{constructor(n,r=null,o=!1,a=null){super(n);this.runId=r;this.retrySafe=o;this.terminalStatus=a;this.name="ApifyGoogleRunError"}runId;retrySafe;terminalStatus};function te(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)?e:null}function re(e){return typeof e=="string"&&e.trim()?e.trim():null}function Tr(e){try{return["http:","https:"].includes(new URL(e).protocol)}catch{return!1}}function xr(e){if(typeof e!="string")return e;try{return JSON.parse(e)}catch{throw new Error("Apify searchResults is not valid JSON")}}function jt(e,t){if(!Array.isArray(e)||e.length===0)throw new Error("Apify dataset contains no search results");let n=[];for(let[a,s]of e.entries()){let i=te(s);if(!i)throw new Error("Apify dataset contains an invalid row");let l=xr(i.searchResults??i),u=te(l),c=Array.isArray(l)?l:Array.isArray(u?.results)?u.results:Array.isArray(u?.organicResults)?u.organicResults:Array.isArray(u?.organic)?u.organic:u&&(u.position!==void 0||u.title!==void 0)?[u]:null;if(!c)throw new Error("Apify dataset search result shape is unsupported");let p=Number(i.page_number??u?.page_number??a+1);if(!Number.isInteger(p)||p<1||p>t)throw new Error("Apify dataset has an unexpected page number");n.push(...c.map(_=>({row:_,page:p})))}let r=n.map(({row:a,page:s},i)=>{let l=te(a),u=l?.position,c=re(l?.title),p=re(l?.url??l?.link);if(!l||!c||!p||!Tr(p)||u!==void 0&&(!Number.isInteger(u)||u<1))throw new Error(`Apify search result ${i+1} is missing a valid position, URL, or title`);let _=re(l.description??l.snippet),w=typeof u=="number"?u:i%10+1;return{position:s>1&&w<=10?w+(s-1)*10:w,url:p,title:c,description:_}});if(r.length>t*10)throw new Error("Apify returned more results than requested");let o={page1:r.filter(a=>a.position<=10).length,page2:r.filter(a=>a.position>10).length};if(o.page1===0)throw new Error("Apify did not return the first result page");return{results:r,pageResultCounts:o}}async function fe(e,t,n,r){let o=await n(e,{headers:{Authorization:`Bearer ${t}`},signal:r});if(!o.ok)throw new Error(`Apify API returned HTTP ${o.status}`);return o.json()}function He(e){let t=te(te(e)?.data),n=re(t?.id),r=re(t?.status);if(!n||!r)throw new Error("Apify returned an invalid run receipt");return{id:n,status:r}}async function Mr(e,t=fetch){let n=performance.now(),r=AbortSignal.timeout(9e4),o=e.signal?AbortSignal.any([e.signal,r]):r,a=e.runId,s=!1,i=null;try{if(!/^[A-Za-z0-9]+$/.test(a))throw new Error("Invalid Apify run ID");let l=He(await fe(`${ee}/actor-runs/${a}?waitForFinish=30`,e.token,t,o));for(;["READY","RUNNING","TIMING-OUT","ABORTING"].includes(l.status);)l=He(await fe(`${ee}/actor-runs/${a}?waitForFinish=30`,e.token,t,o));if(s=!0,i=l.status,l.status!=="SUCCEEDED")throw new Error(`Apify run ended with status ${l.status}`);let u=e.pages??1,c=await fe(`${ee}/actor-runs/${a}/dataset/items?format=json`,e.token,t,o);return{query:e.query,...jt(c,u),requestedPages:u,runId:a,elapsedMs:Math.round(performance.now()-n)}}catch(l){throw new T(l instanceof Error?l.message:String(l),a,s,i)}}async function Qt(e,t,n=fetch){let r=performance.now(),o=e.query.trim(),a=e.pages??1;if(!o||o.includes(`
|
|
6
|
-
`))throw new Error("Apify search requires one nonempty query");if(a!==1&&a!==2)throw new Error("Apify search supports one or two pages");let s=t.token.trim();if(!s)throw new Error("Apify API token is required");let i=t.maxChargeUsd??.01;if(!Number.isFinite(i)||i<=0)throw new Error("Invalid Apify charge cap");let l=AbortSignal.timeout(9e4),u=e.signal?AbortSignal.any([e.signal,l]):l,c=`${ee}/actors/${Cr}/runs?maxTotalChargeUsd=${i}&restartOnError=false&waitForFinish=30`,p=null,_=!1,w=null;try{let R=await n(c,{method:"POST",headers:{Authorization:`Bearer ${s}`,"Content-Type":"application/json"},body:JSON.stringify({keyword:o,limit:a===2?"20":"10",page:1,proxy_location:"us",include_merged:!1}),signal:u});if(!R.ok)throw new Error(`Apify run start returned HTTP ${R.status}`);let S=He(await R.json());if(p=S.id,_=!["READY","RUNNING","TIMING-OUT","ABORTING"].includes(S.status),_&&(w=S.status),S.status!=="SUCCEEDED"&&S.status!=="READY"&&S.status!=="RUNNING")throw new Error(`Apify run ended with status ${S.status}`);if(S.status==="SUCCEEDED"){let C=await fe(`${ee}/actor-runs/${p}/dataset/items?format=json`,s,n,u);return{query:o,...jt(C,a),requestedPages:a,runId:p,elapsedMs:Math.round(performance.now()-r)}}return{...await Mr({runId:p,query:o,token:s,pages:a,signal:u},n),elapsedMs:Math.round(performance.now()-r)}}catch(R){throw R instanceof T?R:new T(R instanceof Error?R.message:String(R),p,_,w)}}var Wt="apify_google_serp",Yt="apify",Nr="apify_google_serp_scraperlink";async function Xt(e,t,n,r){let o=Nt();await B({receiptId:e,costState:"estimated",units:t,unitType:o.unitType,amountUsdNanos:de(o,t),rateSource:o.source,providerProduct:o.providerProduct,billingTier:o.billingTier,billableEvent:o.billableEvent,measurementSource:"actor_run_succeeded",rateNumerator:o.numeratorUsdNanos,rateDenominator:o.denominatorUnits,ratePolicyVersion:o.policyVersion,rateEffectiveAt:o.effectiveAt,providerStatus:r,providerDurationMs:n})}async function Zt(e){let t=performance.now(),n=await $({runId:e.operationRunId,attemptNumber:e.attemptNumber,method:Wt,provider:Yt,parentAttemptId:e.parentAttemptId,edgeKind:e.edgeKind,retryKind:e.retryKind,normalizedFlags:{page:1,pages:e.options.pages??1,proxyLocation:"us"}}),r=await H({attemptId:n,provider:Yt,providerProduct:Nr,billingTier:"store_pay_per_event",accountScope:e.accountScope,method:Wt,providerStatus:"before_actor_start"}),o=null,a=null;try{return o=await Qt(e.options,{token:e.token},e.fetchImpl),await U(r.id,o.runId),await Xt(r.id,o.requestedPages,o.elapsedMs,"succeeded_estimated"),{...o,attemptId:n}}catch(s){throw a=s,o?Object.assign(new T("Apify cost tracking failed after run completion",o.runId),{attemptId:n}):(s instanceof T&&s.runId&&(await U(r.id,s.runId),s.terminalStatus==="SUCCEEDED"?await Xt(r.id,e.options.pages??1,Math.round(performance.now()-t),"succeeded_dataset_unavailable"):await le({receiptId:r.id,units:e.options.pages??1,unitType:"page",providerStatus:"run_outcome_or_cost_pending"})),Object.assign(s instanceof Error?s:new Error(String(s)),{attemptId:n}))}finally{await z({attemptId:n,status:a?e.options.signal?.aborted?"timed_out":"failed":"succeeded",errorCode:a?"apify_search_failed":null,durationMs:Math.round(performance.now()-t)})}}function er(e,t=e.requestedPages){let n=e.results.map(s=>{let i=new URL(s.url).hostname.replace(/^www\./,"");return{position:s.position,title:s.title,url:s.url,rawUrl:s.url,resolvedUrl:s.url,linkType:"plain",resolutionStatus:"not_needed",domain:i,cite:null,snippet:s.description,isRedditStyle:i==="reddit.com",inlineRating:null}}),r=t===2&&e.pageResultCounts.page2>0?2:1,o=t===2&&r===1,a=L(t,[{page:1,attempted:!0,captured:!0,resultCount:e.pageResultCounts.page1,requestOffset:0,providerRequestIdObserved:!0},...t===2?[{page:2,attempted:!0,captured:!o,resultCount:e.pageResultCounts.page2,requestOffset:10,providerRequestIdObserved:!0,...o?{failureCode:"empty_page"}:{}}]:[]]);return{seed:e.query,location:null,extractedAt:new Date().toISOString(),diagnostics:{phaseTimings:{providerConnectMs:e.elapsedMs,totalServerMs:e.elapsedMs},pagination:{requestedPages:t,capturedPages:r,page2Status:t===2?o?"failed":"captured":"not_requested",page1OrganicCount:e.pageResultCounts.page1,page2OrganicCount:e.pageResultCounts.page2},completionStatus:"serp_only",problem:null,resultQuality:o?"partial":"complete",degradedResult:o,degradationReasons:o?["page_two_missing"]:[],retryRecommended:!1,serpCompleteness:{pagination:a,features:{localPack:"not_requested",forums:"not_requested",videos:"not_requested",aiOverview:"not_requested",whatPeopleSaying:"not_requested"},queryIntegrity:ie({query:e.query,finalQuery:e.query,locationResolutionSource:"not_requested",outboundProviderMethod:"default_search"}),failureStage:null}},totalQuestions:0,surface:"web",aiOverview:{detected:!1,text:null,citations:[],shareUrl:null},aiMode:{detected:!1,text:null,citations:[]},whatPeopleSaying:[],tree:[],flat:[],videos:[],forums:[],organicResults:n,localPack:[],entityIds:{entities:[],kgIds:[],cids:[],gcids:[]},stats:{seed:e.query,totalQuestions:0,maxDepthReached:0,durationMs:e.elapsedMs,errorCount:0}}}var Dr=12e3;async function Ue(e){if(e.mode==="full")return Hr(e);if(!e.token)throw new Error("Apify search is not configured");let t=performance.now(),n;for(let o=1;o<=2;o++){let a=AbortSignal.any([e.signal,AbortSignal.timeout(Dr)]),s=e.pages===2&&o===2?1:e.pages;try{let i=await Zt({options:{query:e.query,pages:s,signal:a},token:e.token,operationRunId:e.operationRunId,accountScope:e.accountScope,attemptNumber:o,...n?{parentAttemptId:n,edgeKind:"same_method_retry",retryKind:"same_method"}:{}});console.info(JSON.stringify({event:"serp_provider_succeeded",provider:"apify",operationRunId:e.operationRunId,attemptNumber:o,durationMs:i.elapsedMs,resultCount:i.results.length,requestedPages:e.pages,providerRequestedPages:i.requestedPages,deliveredPages:i.pageResultCounts.page2>0?2:1,runId:i.runId}));let l=er(i,e.pages);return l.stats.durationMs=Math.round(performance.now()-t),l.diagnostics.phaseTimings&&(l.diagnostics.phaseTimings.totalServerMs=l.stats.durationMs),{result:l,mode:"light",deliveryProvider:"apify",deliveredPages:i.pageResultCounts.page2>0?2:1}}catch(i){n=i&&typeof i=="object"&&"attemptId"in i?String(i.attemptId):void 0;let l=i instanceof T?i:null;if(console.warn(JSON.stringify({event:"serp_provider_failed",provider:"apify",operationRunId:e.operationRunId,attemptNumber:o,runId:l?.runId??null,requestedPages:e.pages,attemptPages:s,retrySafe:l?.retrySafe===!0,error:i instanceof Error?i.message:String(i)})),e.signal.aborted||!n||l?.retrySafe!==!0)throw i}}if(!n)throw new Error("Search attempt receipt is missing");let r={query:e.query,gl:"us",hl:"en",device:"desktop",pages:1,includeAllSerpFeatures:!1,includeLocalPack:!1,includeForums:!1,includeVideos:!1,includeAiOverview:!1,includeWhatPeopleSaying:!1,signal:e.signal};console.info(JSON.stringify({event:"serp_provider_fallback",provider:"brightdata",operationRunId:e.operationRunId,attemptNumber:3,requestedPages:e.pages,providerRequestedPages:1}));try{let o=await qe(r,process.env,fetch,void 0,{attemptNumber:3,parentAttemptId:n});if(o.organicResults.length===0)throw new Error("Search returned no organic results");return e.pages===2&&qr(o),o.stats.durationMs=Math.round(performance.now()-t),o.diagnostics.phaseTimings&&(o.diagnostics.phaseTimings.totalServerMs=o.stats.durationMs),console.info(JSON.stringify({event:"serp_provider_succeeded",provider:"brightdata",operationRunId:e.operationRunId,attemptNumber:3,durationMs:o.stats.durationMs,resultCount:o.organicResults.length,requestedPages:e.pages,providerRequestedPages:1,deliveredPages:1,resultQuality:o.diagnostics.resultQuality})),{result:o,mode:"light",deliveryProvider:"brightdata",deliveredPages:1}}catch(o){throw console.warn(JSON.stringify({event:"serp_provider_failed",provider:"brightdata",operationRunId:e.operationRunId,attemptNumber:3,error:o instanceof Error?o.message:String(o)})),o}}function qr(e){let t=e.organicResults.length,n=e.diagnostics.pagination;n&&(n.requestedPages=2,n.capturedPages=1,n.page2Status="failed",n.page2OrganicCount=0),e.diagnostics.resultQuality="partial",e.diagnostics.degradedResult=!0,e.diagnostics.degradationReasons=["page_two_missing"];let r=e.diagnostics;if(r.serpCompleteness){let o=r.serpCompleteness.pagination.evidence.find(a=>a.page===1);r.serpCompleteness.pagination={...L(2,[{page:1,attempted:!0,captured:!0,resultCount:t,requestOffset:0,providerRequestIdObserved:o?.providerRequestIdObserved??!1},{page:2,attempted:!0,captured:!1,resultCount:0,requestOffset:10,providerRequestIdObserved:!1,failureCode:"provider_error"}]),providerRequestCount:r.serpCompleteness.pagination.providerRequestCount}}}async function Hr(e){if(e.pages===2){let o=await Ue({...e,mode:"light",pages:2}),a=Object.fromEntries(["localPack","forums","videos","aiOverview","aiMode","whatPeopleSaying","entityIds","paaPreview","topStories"].map(i=>[i,"unsupported"]));o.result.fullSerpFeatures={status:a,paaPreview:[],additional:[],pages:[]};let s=o.result.diagnostics;return s.serpCompleteness&&(s.serpCompleteness.features={localPack:"unsupported",forums:"unsupported",videos:"unsupported",aiOverview:"unsupported",whatPeopleSaying:"unsupported"}),{...o,mode:"full"}}let t=performance.now(),n={query:e.query,gl:"us",hl:"en",device:"desktop",pages:1,includeAllSerpFeatures:!0,includeLocalPack:!0,includeForums:!0,includeVideos:!0,includeAiOverview:!0,includeWhatPeopleSaying:!0,signal:e.signal},r=await qe(n);if(r.organicResults.length===0)throw new Error("Full Search returned no organic results");return r.stats.durationMs=Math.round(performance.now()-t),r.diagnostics.phaseTimings&&(r.diagnostics.phaseTimings.totalServerMs=r.stats.durationMs),{result:r,mode:"full",deliveryProvider:"brightdata",deliveredPages:r.diagnostics.pagination?.capturedPages===2?2:1}}async function Be(e,t,n=3){for(let r=1;r<=n;r++){try{let o=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(t),signal:AbortSignal.timeout(1e4)});if(o.ok)return;console.warn(`[webhook] attempt ${r} \u2192 ${o.status} from ${e}`)}catch(o){console.warn(`[webhook] attempt ${r} failed:`,o instanceof Error?o.message:o)}r<n&&await new Promise(o=>setTimeout(o,1e3*r*2))}console.error(`[webhook] gave up after ${n} attempts for ${e}`)}function Ur(e){return e instanceof Error?e.message:String(e)}function Br(e,t){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)}function Fr(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function Gr(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function Lr(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function Vr(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function $r(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function Fe(e){let t=Ur(e);return e instanceof pt?{error_code:"request_aborted",error_type:"request_aborted",message:N("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof dt||Fr(t)?{error_code:"captcha_exhausted",error_type:"captcha",message:N("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Tt(e,t)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:N("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof mt?{error_code:"location_mismatch",error_type:"location_mismatch",message:N("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Gr(t)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:N("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Lr(t)?{error_code:"proxy_unavailable",error_type:"connection",message:N("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:$r(t)?{error_code:"browser_session_interrupted",error_type:"extraction",message:N("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Br(e,t)?{error_code:"harvest_timeout",error_type:"timeout",message:N("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:Vr(t)?{error_code:"service_unavailable",error_type:"service_unavailable",message:Re,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:Re,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function Ge(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function he(e,t={}){let n=ct({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:t.retryAfterSeconds,chargeStatus:t.chargeStatus,details:t.details});return{error:n.message,...n}}function Le(e,t={}){let{error:n,...r}=he(e,t);return r}function Io(e,t={}){let n=Fe(e);return{body:he(n,t),status:n.httpStatus}}import{createHash as zr}from"crypto";var Kr=1e9;function Jr(e){return zr("sha256").update(JSON.stringify(e)).digest("hex")}function ye(e){return`harvest:${e}`}async function Do(e){let t=ye(e.jobId);try{return(await Et({id:t,userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,idempotencyKey:`job:${e.jobId}`,requestFingerprint:Jr(e.normalizedFlags),jobId:e.jobId,requestId:e.requestId??null,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id}catch(n){return console.warn(JSON.stringify({event:"operation_run_start_failed",job_id:e.jobId,tool:e.tool,message:n instanceof Error?n.message:String(n)})),null}}async function $e(e,t,n){if(e)try{await kt({runId:e,status:t,failureClass:n})}catch(r){console.warn(JSON.stringify({event:"operation_run_finish_failed",operation_run_id:e,message:r instanceof Error?r.message:String(r)}))}}async function qo(e,t,n,r){if(!(!e||!t))try{await Ot({runId:e,billingTable:"billing_debits",billingKey:t,relation:n,amountMc:r})}catch(o){console.warn(JSON.stringify({event:"operation_billing_link_failed",operation_run_id:e,relation:n,message:o instanceof Error?o.message:String(o)}))}}function Ve(e){return e==="timeout"?"timed_out":e==="request_aborted"?"cancelled":e==="browser_session_interrupted"?"interrupted":e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"||e==="location_mismatch"||e==="error"?"failed":"succeeded"}function jr(e){return e==="bright_data_browser_api"?"brightdata":"kernel"}function tr(e,t,n=0){let r=new Map,o=null;return async a=>{if(e)try{if(a.type==="started"){let i=n+a.attemptNumber;if(i>1&&!o){let p=(await It(e))?.attempts.at(-1);o=p?.id==null?null:String(p.id)}let l=await $({runId:e,attemptNumber:i,method:a.method,provider:a.provider,parentAttemptId:i===1?null:o,edgeKind:i===1?null:n>0&&a.attemptNumber===1?"durable_resume":a.retryKind==="provider_fallback"?"provider_fallback":"same_method_retry",retryKind:i===1?null:n>0&&a.attemptNumber===1?"durable_resume":a.retryKind,retryReason:i===1?null:n>0&&a.attemptNumber===1?"durable_worker_resume":a.retryReason,retryOwner:i===1?null:n>0&&a.attemptNumber===1?"durable_reconciler":"harvest",normalizedFlags:{maxAttempts:a.maxAttempts,maxQuestions:a.maxQuestions,hasLocation:!!a.location,plannedProvider:a.provider,planReason:a.planReason??null,planTimeoutMs:a.planTimeoutMs??null},startedAt:a.startedAt}),u=await H({attemptId:l,provider:a.provider,providerProduct:a.provider==="brightdata"?"brightdata_browser_api":"kernel_browser",billingTier:a.provider==="brightdata"?Z("https://www.google.com/search"):null,accountScope:t,method:a.method,providerStatus:"started"});r.set(a.attemptNumber,{attemptId:l,receiptId:u.id,plannedProvider:a.provider,observedProvider:null,providerMismatch:!1,method:a.method,startedAtMs:Date.parse(a.startedAt),billingMode:null}),o=l;return}let s=r.get(a.attemptNumber);if(!s)throw new Error(`Missing operation attempt state for attempt ${a.attemptNumber}`);if(a.type==="provider_session_started"){let i=jr(a.provider);if(s.observedProvider=i,s.billingMode=a.billingMode??null,i!==s.plannedProvider){s.providerMismatch=!0,await B({receiptId:s.receiptId,costState:"unavailable",providerStatus:"provider_mismatch",errorCode:"provider_mismatch",errorMessage:`Planned ${s.plannedProvider}; observed ${i}`,finalizedAt:a.observedAt});let l=i==="brightdata"?"brightdata_browser":"kernel_browser",u=await H({attemptId:s.attemptId,provider:i,providerProduct:i==="brightdata"?"brightdata_browser_api":"kernel_browser",billingTier:i==="brightdata"?Z("https://www.google.com/search"):null,accountScope:t,method:l,providerStatus:"observed_provider_mismatch"});s.receiptId=u.id,s.method=l}await U(s.receiptId,a.providerSessionId);return}if(await z({attemptId:s.attemptId,status:s.providerMismatch?"failed":Ve(a.outcome),errorCode:s.providerMismatch?"provider_mismatch":Ve(a.outcome)==="succeeded"?null:a.outcome,errorMessage:a.error,durationMs:a.durationMs,endedAt:a.completedAt}),(s.observedProvider??s.plannedProvider)==="kernel"){let i=s.billingMode==="headful",l=s.billingMode==="headless"||s.billingMode==="headful"?St(a.durationMs,i):null;await B({receiptId:s.receiptId,costState:"estimated",units:a.durationMs/1e3,unitType:"wall_open_second",amountUsdNanos:l==null?null:Math.round(l*Kr),providerProduct:"kernel_browser",billingTier:s.billingMode,billableEvent:"provider_active_gb_second",measurementSource:"wall_open_time_proxy",rateSource:l!=null?`kernel_${s.billingMode}_usd_per_second:${i?vt:wt}`:null,providerStatus:s.providerMismatch?"provider_mismatch":a.outcome,providerDurationMs:a.durationMs,errorCode:s.providerMismatch?"provider_mismatch":Ve(a.outcome)==="succeeded"?null:a.outcome,errorMessage:a.error,finalizedAt:a.completedAt})}}catch(s){console.warn(JSON.stringify({event:"operation_attempt_telemetry_failed",operation_run_id:e,attempt_number:a.attemptNumber,message:s instanceof Error?s.message:String(s)}))}}}function rr(e,t,n=null,r="production-default",o=0){let a=tr(n,r,o);return async s=>{if(await a(s),s.type==="started"){await gt({jobId:e,userId:t,attemptNumber:s.attemptNumber,maxAttempts:s.maxAttempts,query:s.query,location:s.location,maxQuestions:s.maxQuestions,startedAt:s.startedAt});return}if(s.type==="provider_session_started"){await ft({jobId:e,attemptNumber:s.attemptNumber,browserProvider:s.provider,providerSessionId:s.providerSessionId,observedAt:s.observedAt});return}await ht({jobId:e,attemptNumber:s.attemptNumber,outcome:s.outcome,kernelSessionId:s.kernelSessionId,questionCount:s.questionCount,durationMs:s.durationMs,error:s.error,willRetry:s.willRetry,kernelDeleteStarted:s.cleanup.kernelDeleteStarted,kernelDeleteSucceeded:s.cleanup.kernelDeleteSucceeded,kernelDeleteError:s.cleanup.kernelDeleteError,browserCloseSucceeded:s.cleanup.browserCloseSucceeded,browserCloseError:s.cleanup.browserCloseError,browserProvider:s.cleanup.provider??null,providerSessionId:s.cleanup.providerSessionId??null,providerLookupStatus:s.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:s.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:s.cleanup.providerDisconnectMessage??null,providerReasonSource:s.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:s.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:s.cleanup.providerDisconnectMessage??null,debug:s.debug,completedAt:s.completedAt})}}var or=2,ze=0;function nr(e){if(!e||typeof e!="object")return 0;let t=e;return typeof t.totalQuestions=="number"?t.totalQuestions:Array.isArray(t.flat)?t.flat.length:0}function sr(e){return Te.paa_base+Math.max(1,e)*Te.paa}async function ar(e){ze++;try{let t=typeof e.options=="string"?JSON.parse(e.options):e.options,n={value:null},r=null,o=t.serpOnly&&!t.serpIdentity,a=ye(e.id),s=process.env.APIFY_API_TOKEN?.trim();if(o&&(t.pages===2?!s:!zt()||t.mode!=="full"&&!s))throw new Error("Search providers are not configured");let i=await _t({op:t.serpOnly?"serp":"paa",userId:Number(e.user_id),headlessSentOut:n,operationRunId:a,operationAccountScope:"production-default"},()=>o?Ue({query:t.query??e.query,pages:t.pages===2?2:1,token:s,mode:t.mode,operationRunId:a,accountScope:"production-default",signal:AbortSignal.timeout(lt(t.maxQuestions??30,!0,t.mode,t.pages).serverMs)}).then(u=>(r=u,u.result)):Pt({...t,kernelApiKey:ut(),headless:!0,format:"json",outputDir:"/tmp/paa-output-api",onAttemptEvent:rr(e.id,e.user_id,a)}));if(o&&!r)throw new Error("Search provider billing receipt is missing");await yt(e.id,i),await $e(a,"succeeded");let l=await Ae(e.id,e.user_id);if(typeof t.billingHoldMc=="number"&&t.billingDebitKey){let u=t.serpOnly?o?Me(r.deliveredPages,r.deliveryProvider):xe(n.value):sr(nr(i));await Oe(Number(e.user_id),t.billingDebitKey,u,t.serpOnly?"serp_refund":"paa_refund",t.serpOnly?"SERP search settlement":"PAA harvest settlement",t.serpOnly?"serp_search":"paa_harvest")}else if(!t.serpOnly&&typeof t.billingHoldMc=="number"){let u=sr(nr(i)),c=t.billingHoldMc-u;c>0?await ae(e.user_id,c,"paa_refund","overestimate refund"):c<0&&await ke(e.user_id,-c,"paa",t.query??e.query)}else if(t.serpOnly&&typeof t.billingHoldMc=="number"){let u=o?Me(r.deliveredPages,r.deliveryProvider):xe(n.value),c=t.billingHoldMc-u;c>0?await ae(e.user_id,c,"serp_refund","headless-mode pricing settle"):c<0&&await ke(e.user_id,-c,"serp",t.query??e.query)}e.callback_url&&await Be(e.callback_url,{job_id:e.id,status:"done",result:Ct(i),attempts:Ce(l)})}catch(t){console.error("[harvest/worker] failed",{jobId:e.id,error:t instanceof Error?t.message:String(t)});let n=Fe(t);await $e(ye(e.id),n.error_code==="harvest_timeout"?"timed_out":n.terminalStatus,n.error_code);let r=typeof e.options=="string"?JSON.parse(e.options):e.options,o=typeof r.billingHoldMc=="number"&&r.billingHoldMc>0;await Ee(e.id,Ge(n),Le(n,{chargeStatus:o?"refund_pending":"not_charged"}));let a=await Ae(e.id,e.user_id);try{o&&(r.billingDebitKey?await Oe(Number(e.user_id),r.billingDebitKey,0,r.serpOnly?"serp_refund":"paa_refund","failed call",r.serpOnly?"serp_search":"paa_harvest"):await ae(e.user_id,r.billingHoldMc,"refund","failed call"),await Ee(e.id,Ge(n),Le(n,{chargeStatus:"refunded"})))}catch{}e.callback_url&&await Be(e.callback_url,{job_id:e.id,status:"failed",...he(n),attempts:Ce(a)})}finally{ze--}}async function Qr(){let e=await Pe();if(!e)return{claimed:!1};let t=Date.now();return await ar(e),{claimed:!0,jobId:e.id,completed:!0,durationMs:Date.now()-t}}async function en(e){let t=[];for(let n=0;n<e.maxJobs&&!(Date.now()>=e.deadlineMs);n++){let r=await Qr();if(t.push(r),!r.claimed)break}return t}function tn(){setInterval(async()=>{if(ze>=or)return;let e=await Pe();e&&ar(e)},2e3),console.log(`[worker] started \u2014 polling every 2s, max ${or} concurrent`)}export{Z as a,de as b,_r as c,to as d,zt as e,Ue as f,Fe as g,Ge as h,he as i,Le as j,Io as k,ye as l,Do as m,$e as n,qo as o,rr as p,Be as q,Qr as r,en as s,tn as t};
|
package/dist/chunk-XJ6PJWT5.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
var o="0.93.2";export{o as a};
|