mcp-scraper 0.89.1 → 0.89.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -15
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/{analytics-repository-J25XR5E7.js → analytics-repository-5CAFUXYD.js} +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-FFPD2DOF.js → chunk-2O3UYISB.js} +1 -1
- package/dist/chunk-3GP5CYZX.js +1 -1
- package/dist/chunk-4FROKQJN.js +1 -1
- package/dist/{chunk-YUPLOOTB.js → chunk-CYOEMHTB.js} +1 -1
- package/dist/{chunk-RYNHNEF3.js → chunk-D4NOJAUN.js} +15 -15
- package/dist/{chunk-IJ2AO7AO.js → chunk-DMCYIWB3.js} +1 -1
- package/dist/{chunk-ZHORX44A.js → chunk-EN66HBAW.js} +1 -1
- package/dist/{chunk-PRWZNHQB.js → chunk-HYQDXTS7.js} +1 -1
- package/dist/{chunk-6QO4F2KC.js → chunk-J7EVLE6H.js} +3 -3
- package/dist/{chunk-57AHYL2O.js → chunk-K7KMLT33.js} +1 -1
- package/dist/{chunk-CCYWSJNG.js → chunk-OBDMJCCF.js} +4 -4
- package/dist/{chunk-645LQBE7.js → chunk-OXFWXWQG.js} +3 -3
- package/dist/{chunk-MEM4D2QN.js → chunk-QN3QHTVK.js} +88 -88
- package/dist/chunk-QPWPR5XG.js +3 -3
- package/dist/chunk-RNDORDA2.js +1 -0
- package/dist/chunk-S7RA23GR.js +1 -0
- package/dist/{chunk-DGBLGFNJ.js → chunk-VEJQ4XPI.js} +1 -1
- package/dist/{chunk-D6IQJP64.js → chunk-Z27SYWYD.js} +1 -1
- package/dist/{chunk-63T4W2LX.js → chunk-Z66Z2HV4.js} +87 -83
- package/dist/{db-JJWJLAVQ.js → db-NLGNTQC3.js} +1 -1
- package/dist/{extract-bundle-TTSEGWYF.js → extract-bundle-TXH3VNXH.js} +1 -1
- package/dist/{gmail-service-34PIJRLK.js → gmail-service-J2DWSQJX.js} +1 -1
- package/dist/index.cjs +49 -37
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/index.js +1 -1
- package/dist/{lead-list-enrichment-repository-VJBBMBCG.js → lead-list-enrichment-repository-G7Z5DQZR.js} +1 -1
- package/dist/{location-data-repository-W3LPT6G3.js → location-data-repository-TEXZ3I2N.js} +1 -1
- package/dist/{server-UEY5YQL6.js → server-RORLM2FG.js} +719 -712
- package/dist/{site-extract-repository-7W5ZUNPR.js → site-extract-repository-MBSCAP4D.js} +1 -1
- package/dist/worker-SUCJKWU2.js +1 -0
- package/package.json +17 -131
- package/dist/chunk-5H3MCTTP.js +0 -1
- package/dist/chunk-FXEWHK54.js +0 -1
- package/dist/worker-ZOTOKG56.js +0 -1
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import{s as _}from"./chunk-
|
|
1
|
+
import{s as _}from"./chunk-Z66Z2HV4.js";import{AsyncLocalStorage as N}from"async_hooks";var C="MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST",d=new N;function D(e,t){return d.run(e,t)}function l(){return d.getStore()}import{randomUUID as T}from"crypto";var p=166667e-10,L=.0001333336,h=new Set(["serp","fb_search","fb_ad","instagram"]),A=.00111,R=4,g=4e-4;function n(e,t){let r=process.env[e]?.trim();if(!r)return t;let o=Number(r);return Number.isFinite(o)&&o>=0?o:t}var m=n("NANGO_USD_PER_CONNECTION_MONTH",1),U=n("NANGO_USD_PER_FUNCTION_RUN",1e-4),O=n("NANGO_USD_PER_PROXY_REQUEST",1e-4),S=n("NANGO_USD_PER_COMPUTE_SEC",2e-4),b=n("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_USD_PER_GB",8);function u(e,t){return Math.max(0,e)/1e3*(t?L:p)}function i(e,t){return e==="fal_wizper"?Math.max(0,t)/R*A:e==="deepinfra_qwen"?Math.max(0,t)/1e3*g:e==="openrouter"||e==="mcp_memory_video"||e==="mcp_memory_ai"?Math.max(0,t):e==="nango_connection"?Math.max(0,t)*m:e==="nango_function_run"?Math.max(0,t)*U:e==="nango_proxy_request"?Math.max(0,t)*O:e==="nango_compute"?Math.max(0,t)*S:e==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api"?Math.max(0,t)/1e9*b:0}var E=!1,s=null;async function c(){if(!E)return s||(s=x().finally(()=>{s=null}),s)}async function x(){let e=_(),t=await e.execute(`
|
|
2
2
|
SELECT
|
|
3
|
-
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('
|
|
4
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('
|
|
3
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('\u006b\u0065\u0072\u006e\u0065\u006c_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
4
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('\u006b\u0065\u0072\u006e\u0065\u006c_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
5
5
|
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
|
|
6
6
|
AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
|
|
7
7
|
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
8
8
|
AS ready
|
|
9
9
|
`);if(Number(t.rows[0]?.ready??0)===1){E=!0;return}await e.execute(`
|
|
10
|
-
CREATE TABLE IF NOT EXISTS
|
|
10
|
+
CREATE TABLE IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log (
|
|
11
11
|
id TEXT PRIMARY KEY,
|
|
12
|
-
|
|
12
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_session_id TEXT,
|
|
13
13
|
op TEXT,
|
|
14
14
|
source TEXT NOT NULL,
|
|
15
15
|
probe_run_id TEXT,
|
|
@@ -26,7 +26,7 @@ import{s as _}from"./chunk-63T4W2LX.js";import{AsyncLocalStorage as N}from"async
|
|
|
26
26
|
error TEXT,
|
|
27
27
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
28
28
|
)
|
|
29
|
-
`),await e.execute("CREATE INDEX IF NOT EXISTS
|
|
29
|
+
`),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_op ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(op)"),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_probe ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(probe_run_id)"),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_created ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(created_at)");try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN proxy_source TEXT")}catch{}try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN proxy_type TEXT")}catch{}try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN method TEXT")}catch{}await e.execute(`
|
|
30
30
|
CREATE TABLE IF NOT EXISTS vendor_usage_log (
|
|
31
31
|
id TEXT PRIMARY KEY,
|
|
32
32
|
op TEXT,
|
|
@@ -54,20 +54,20 @@ import{s as _}from"./chunk-63T4W2LX.js";import{AsyncLocalStorage as N}from"async
|
|
|
54
54
|
success INTEGER NOT NULL DEFAULT 0,
|
|
55
55
|
error TEXT,
|
|
56
56
|
http_only INTEGER NOT NULL DEFAULT 0,
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_used INTEGER NOT NULL DEFAULT 0,
|
|
58
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_sessions INTEGER NOT NULL DEFAULT 0,
|
|
59
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_seconds_total REAL NOT NULL DEFAULT 0,
|
|
60
|
+
\u006b\u0065\u0072\u006e\u0065\u006c_tier_observed TEXT,
|
|
61
|
+
est_\u006b\u0065\u0072\u006e\u0065\u006c_cost_usd_headless REAL NOT NULL DEFAULT 0,
|
|
62
|
+
est_\u006b\u0065\u0072\u006e\u0065\u006c_cost_usd_headful REAL NOT NULL DEFAULT 0,
|
|
63
63
|
vendor_cost_usd REAL NOT NULL DEFAULT 0,
|
|
64
64
|
charged_credits REAL NOT NULL DEFAULT 0,
|
|
65
65
|
margin_usd_headless_starter REAL NOT NULL DEFAULT 0,
|
|
66
66
|
margin_usd_headful_starter REAL NOT NULL DEFAULT 0,
|
|
67
67
|
notes TEXT
|
|
68
68
|
)
|
|
69
|
-
`),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function w(e){try{await c();let t=l(),r=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO
|
|
70
|
-
(id,
|
|
71
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[T(),e
|
|
69
|
+
`),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function w(e){try{await c();let t=l(),r=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO \u006b\u0065\u0072\u006e\u0065\u006c_session_log
|
|
70
|
+
(id, \u006b\u0065\u0072\u006e\u0065\u006c_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
|
|
71
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[T(),e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId??null,t?.op??null,e.source,t?.probeRunId??null,t?.userId??null,a(e.stealth),a(e.headlessSent),a(e.proxyUsed),e.proxySource??null,e.proxyType??null,e.fallback?1:0,new Date(e.openedAtMs).toISOString(),new Date(e.closedAtMs).toISOString(),r,u(r,!1),u(r,!0),e.error??null,t?.subOp??null]})}catch(t){console.warn("[cost-telemetry] record\u004b\u0065\u0072\u006e\u0065\u006cSession failed:",t instanceof Error?t.message:String(t))}}async function F(e){try{await c();let t=l();return(await _().execute({sql:`INSERT OR IGNORE INTO vendor_usage_log
|
|
72
72
|
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
|
|
73
73
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[T(),e.op??t?.op??null,e.probeRunId??t?.probeRunId??null,e.userId??t?.userId??null,e.vendor,e.model??null,e.units,e.unitType,i(e.vendor,e.units),e.error??null,e.method??t?.subOp??null,e.sourceKey??null,e.providerDurationMs??null,a(e.providerCaptcha),e.providerStatus??null]})).rowsAffected===1}catch(t){return console.warn("[cost-telemetry] recordVendorUsage failed:",t instanceof Error?t.message:String(t)),!1}}function a(e){return e==null?null:e?1:0}export{C as a,D as b,l as c,h as d,i as e,c as f,w as g,F as h};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{b as O}from"./chunk-KJQXUZ4Y.js";import{s as N}from"./chunk-
|
|
1
|
+
import{b as O}from"./chunk-KJQXUZ4Y.js";import{s as N}from"./chunk-Z66Z2HV4.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS location_data_imports (
|
|
3
3
|
id TEXT PRIMARY KEY,
|
|
4
4
|
dataset_kind TEXT NOT NULL,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{s as r}from"./chunk-
|
|
1
|
+
import{s as r}from"./chunk-Z66Z2HV4.js";import{createHash as b,randomUUID as O}from"crypto";var p="mcp-scraper/lead-list-enrichment.requested",A=new Set(["complete","partial","empty","failed","cancelled"]),U=new Set(["complete","failed","cancelled"]),T=10,g=360,S=60,h=null,m=null;function o(e,t,s){let n=e.trim();if(!n||n.length>s)throw new Error(`${t} is required and must be ${s} characters or fewer`);return n}function d(e,t){if(!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer`);return e}function u(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function R(e){let t=new WeakSet,s=n=>{if(n===null||typeof n=="string"||typeof n=="boolean")return n;if(typeof n=="number"){if(!Number.isFinite(n))throw new Error("canonical fingerprint input must contain finite numbers");return Object.is(n,-0)?0:n}if(Array.isArray(n))return n.map(s);if(typeof n=="object"){if(t.has(n))throw new Error("canonical fingerprint input must not be cyclic");t.add(n);let i={};for(let _ of Object.keys(n).sort()){let c=n[_];c!==void 0&&(i[_]=s(c))}return t.delete(n),i}throw new Error(`canonical fingerprint input contains unsupported ${typeof n}`)};return JSON.stringify(s(e))}function y(e){return b("sha256").update(R(e)).digest("hex")}function I(e){return b("sha256").update(R(e)).digest("hex")}function j(){return{mapsAttempts:0,pageAttempts:0,pageSuccesses:0,serpSearches:0}}function E(e){let t=u(e.input_json);if(!t)throw new Error(`lead list enrichment job ${String(e.id??"")} has invalid durable input`);return{id:String(e.id),userId:Number(e.user_id),idempotencyKey:String(e.idempotency_key),requestFingerprint:String(e.request_fingerprint),input:t,status:String(e.status),totalRows:Number(e.total_rows??0),doneRows:Number(e.done_rows??0),failedRows:Number(e.failed_rows??0),cancelledRows:Number(e.cancelled_rows??0),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},concurrencyLockId:e.concurrency_lock_id==null?null:String(e.concurrency_lock_id),debitKey:String(e.debit_key),holdMc:Number(e.hold_mc??0),billedMc:e.billed_mc==null?null:Number(e.billed_mc),result:u(e.result_json),csvArtifact:u(e.csv_artifact_json),xlsxArtifact:u(e.xlsx_artifact_json),warnings:u(e.warnings_json)??[],cancelRequestedAt:e.cancel_requested_at==null?null:String(e.cancel_requested_at),errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:e.retryable==null?null:Number(e.retryable)===1,createdAt:String(e.created_at??""),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function w(e){return{jobId:String(e.job_id),ordinal:Number(e.ordinal),inputDigest:String(e.input_digest),status:String(e.status),result:u(e.result_json),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:Number(e.retryable??0)===1,durationMs:Number(e.duration_ms??0),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function M(e){return{id:String(e.id),jobId:String(e.job_id),eventName:p,status:String(e.status),attempts:Number(e.attempts??0),claimToken:e.locked_by==null?null:String(e.locked_by),lockedUntil:e.locked_until==null?null:String(e.locked_until),nextAttemptAt:e.next_attempt_at==null?null:String(e.next_attempt_at),lastError:e.last_error==null?null:String(e.last_error),createdAt:String(e.created_at??""),dispatchedAt:e.dispatched_at==null?null:String(e.dispatched_at),updatedAt:String(e.updated_at??"")}}async function a(){let e=r();return h!==e&&(h=e,m=null),m||(m=(async()=>{await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_jobs (
|
|
2
2
|
id TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id), idempotency_key TEXT NOT NULL,
|
|
3
3
|
request_fingerprint TEXT NOT NULL, input_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'initializing',
|
|
4
4
|
total_rows INTEGER NOT NULL DEFAULT 0, done_rows INTEGER NOT NULL DEFAULT 0,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as m}from"./chunk-YQZGZBB4.js";import{b as n,da as p,ea as d,g as o,ga as b,h as i,j as l,m as u,o as c}from"./chunk-
|
|
1
|
+
import{a as m}from"./chunk-YQZGZBB4.js";import{b as n,da as p,ea as d,g as o,ga as b,h as i,j as l,m as u,o as c}from"./chunk-Z66Z2HV4.js";function f(e){return e instanceof Error?e.message:String(e)}function y(e,t){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)}function g(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function S(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function E(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function h(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function v(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function x(e){let t=f(e);return e instanceof u?{error_code:"request_aborted",error_type:"request_aborted",message:o("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof l||g(t)?{error_code:"captcha_exhausted",error_type:"captcha",message:o("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:m(e,t)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:o("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof c?{error_code:"location_mismatch",error_type:"location_mismatch",message:o("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:S(t)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:o("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:E(t)?{error_code:"proxy_unavailable",error_type:"connection",message:o("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:v(t)?{error_code:"browser_session_interrupted",error_type:"extraction",message:o("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:y(e,t)?{error_code:"harvest_timeout",error_type:"timeout",message:o("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:h(t)?{error_code:"service_unavailable",error_type:"service_unavailable",message:n,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:n,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function w(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function _(e,t={}){let r=i({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:t.retryAfterSeconds,chargeStatus:t.chargeStatus,details:t.details});return{error:r.message,...r}}function A(e,t={}){let{error:r,...a}=_(e,t);return a}function C(e,t={}){let r=x(e);return{body:_(r,t),status:r.httpStatus}}function D(e,t){return async r=>{if(r.type==="started"){await p({jobId:e,userId:t,attemptNumber:r.attemptNumber,maxAttempts:r.maxAttempts,query:r.query,location:r.location,maxQuestions:r.maxQuestions,startedAt:r.startedAt});return}if(r.type==="provider_session_started"){await d({jobId:e,attemptNumber:r.attemptNumber,browserProvider:r.provider,providerSessionId:r.providerSessionId,observedAt:r.observedAt});return}await b({jobId:e,attemptNumber:r.attemptNumber,outcome:r.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:r.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:r.questionCount,durationMs:r.durationMs,error:r.error,willRetry:r.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:r.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:r.cleanup.browserCloseSucceeded,browserCloseError:r.cleanup.browserCloseError,browserProvider:r.cleanup.provider??null,providerSessionId:r.cleanup.providerSessionId??null,providerLookupStatus:r.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:r.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:r.cleanup.providerDisconnectMessage??null,providerReasonSource:r.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:r.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:r.cleanup.providerDisconnectMessage??null,debug:r.debug,completedAt:r.completedAt})}}async function I(e,t,r=3){for(let a=1;a<=r;a++){try{let s=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(t),signal:AbortSignal.timeout(1e4)});if(s.ok)return;console.warn(`[webhook] attempt ${a} \u2192 ${s.status} from ${e}`)}catch(s){console.warn(`[webhook] attempt ${a} failed:`,s instanceof Error?s.message:s)}a<r&&await new Promise(s=>setTimeout(s,1e3*a*2))}console.error(`[webhook] gave up after ${r} attempts for ${e}`)}export{x as a,w as b,_ as c,A as d,C as e,D as f,I as g};
|