mcp-scraper 0.97.0 → 0.98.0

package/dist/chunk-UNEJYZ4T.js

@@ -1,102 +0,0 @@
-import{a as N}from"./chunk-WNUXMPVX.js";import{b as O}from"./chunk-KJQXUZ4Y.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
-        CREATE TABLE IF NOT EXISTS location_data_imports (
-          id           TEXT PRIMARY KEY,
-          dataset_kind TEXT NOT NULL,
-          source_url   TEXT,
-          sha256       TEXT NOT NULL,
-          row_count    INTEGER NOT NULL DEFAULT 0,
-          group_count  INTEGER NOT NULL DEFAULT 0,
-          state_count  INTEGER NOT NULL DEFAULT 0,
-          source_row_count   INTEGER NOT NULL DEFAULT 0,
-          rejected_row_count INTEGER NOT NULL DEFAULT 0,
-          zip_count          INTEGER NOT NULL DEFAULT 0,
-          covered_states_json TEXT NOT NULL DEFAULT '[]',
-          coverage_complete  INTEGER NOT NULL DEFAULT 0,
-          status       TEXT NOT NULL,
-          error        TEXT,
-          created_at   TEXT NOT NULL DEFAULT (datetime('now')),
-          completed_at TEXT
-        )
-      `);let o=new Set((await t.execute("PRAGMA table_info(location_data_imports)")).rows.map(r=>String(r.name))),a=[["source_row_count","INTEGER NOT NULL DEFAULT 0"],["rejected_row_count","INTEGER NOT NULL DEFAULT 0"],["zip_count","INTEGER NOT NULL DEFAULT 0"],["covered_states_json","TEXT NOT NULL DEFAULT '[]'"],["coverage_complete","INTEGER NOT NULL DEFAULT 0"]];for(let[r,n]of a)o.has(r)||await t.execute(`ALTER TABLE location_data_imports ADD COLUMN ${r} ${n}`);await t.execute("CREATE INDEX IF NOT EXISTS location_data_imports_kind_created ON location_data_imports(dataset_kind, created_at DESC)"),await t.execute(`
-        CREATE TABLE IF NOT EXISTS location_zip_groups (
-          import_id     TEXT NOT NULL REFERENCES location_data_imports(id) ON DELETE CASCADE,
-          state_abbr    TEXT NOT NULL,
-          city_key      TEXT NOT NULL,
-          city          TEXT NOT NULL,
-          zips_json     TEXT NOT NULL,
-          counties_json TEXT NOT NULL,
-          PRIMARY KEY(import_id, state_abbr, city_key)
-        )
-      `),await t.execute("CREATE INDEX IF NOT EXISTS location_zip_groups_state ON location_zip_groups(import_id, state_abbr, city_key)"),await t.execute(`
-        CREATE TABLE IF NOT EXISTS location_census_places (
-          import_id           TEXT NOT NULL REFERENCES location_data_imports(id) ON DELETE CASCADE,
-          state_abbr          TEXT NOT NULL,
-          city_key            TEXT NOT NULL,
-          city                TEXT NOT NULL,
-          census_name         TEXT NOT NULL,
-          estimates_base_2020 INTEGER,
-          population_2020     INTEGER,
-          population_2021     INTEGER,
-          population_2022     INTEGER,
-          population_2023     INTEGER,
-          population_2024     INTEGER,
-          population_2025     INTEGER,
-          PRIMARY KEY(import_id, state_abbr, city_key)
-        )
-      `),await t.execute("CREATE INDEX IF NOT EXISTS location_census_places_state ON location_census_places(import_id, state_abbr, city_key)"),await t.execute(`
-        CREATE TABLE IF NOT EXISTS location_data_active (
-          dataset_kind TEXT PRIMARY KEY,
-          import_id    TEXT NOT NULL REFERENCES location_data_imports(id),
-          activated_at TEXT NOT NULL DEFAULT (datetime('now'))
-        )
-      `)})().catch(o=>{throw w=null,o})),await w}async function h(t){await R();let a=(await N().execute({sql:`
-      SELECT i.*
-      FROM location_data_active a
-      JOIN location_data_imports i ON i.id = a.import_id
-      WHERE a.dataset_kind = ? AND i.status = 'ready'
-      LIMIT 1
-    `,args:[t]})).rows[0];return a?C(a):null}async function Z(){return h(D)}async function st(){return await R(),(await N().execute(`
-    SELECT i.*
-    FROM location_data_active a
-    JOIN location_data_imports i ON i.id = a.import_id
-    WHERE i.status = 'ready'
-    ORDER BY i.dataset_kind ASC
-  `)).rows.map(o=>C(o))}async function Q(t){let o=f(t);if(!o)throw new Error("state must be a two-letter US state abbreviation");let a=await Z();if(!a)return{dataset:null,groups:new Map};let r=await N().execute({sql:`
-      SELECT city, city_key, state_abbr, zips_json, counties_json
-      FROM location_zip_groups
-      WHERE import_id = ? AND state_abbr = ?
-      ORDER BY city COLLATE NOCASE ASC
-    `,args:[a.id,o]}),n=new Map;for(let u of r.rows){let d=String(u.city_key);n.set(d,{city:String(u.city),cityKey:d,state:String(u.state_abbr),zips:JSON.parse(String(u.zips_json)),counties:JSON.parse(String(u.counties_json))})}return{dataset:a,groups:n}}async function at(t){await R();let o=X(t.sourceUrl),a=M("sha256").update(t.csv).digest("hex"),r=await Z();if(r?.sha256===a)return{dataset:r,duplicate:!0};let n=W(t.csv);J(n);let u=`loc_${z().replaceAll("-","")}`,d=N();await d.execute({sql:`
-      INSERT INTO location_data_imports (
-        id, dataset_kind, source_url, sha256, row_count, group_count, state_count,
-        source_row_count, rejected_row_count, zip_count, covered_states_json, coverage_complete, status
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 'loading')
-    `,args:[u,D,o,a,n.rowCount,n.groups.length,n.stateCount,n.sourceRowCount,n.rejectedRowCount,n.zipCount,JSON.stringify(n.coveredStates)]});try{for(let i=0;i<n.groups.length;i+=A){let _=n.groups.slice(i,i+A).map(e=>({sql:`
-          INSERT INTO location_zip_groups (
-            import_id, state_abbr, city_key, city, zips_json, counties_json
-          ) VALUES (?, ?, ?, ?, ?, ?)
-        `,args:[u,e.state,e.cityKey,e.city,JSON.stringify(e.zips),JSON.stringify(e.counties)]}));await d.batch(_,"write")}await d.batch([{sql:"UPDATE location_data_imports SET status = 'ready', completed_at = datetime('now'), error = NULL WHERE id = ? AND status = 'loading'",args:[u]},{sql:`
-          INSERT INTO location_data_active (dataset_kind, import_id, activated_at)
-          VALUES (?, ?, datetime('now'))
-          ON CONFLICT(dataset_kind) DO UPDATE SET import_id = excluded.import_id, activated_at = excluded.activated_at
-        `,args:[D,u]}],"write")}catch(i){let _=i instanceof Error?i.message:String(i);throw await d.execute({sql:"UPDATE location_data_imports SET status = 'failed', error = ?, completed_at = datetime('now') WHERE id = ?",args:[_.slice(0,1e3),u]}).catch(()=>{}),i}let p=(await d.execute({sql:"SELECT * FROM location_data_imports WHERE id = ? LIMIT 1",args:[u]})).rows[0];if(!p)throw new Error("Hosted location import completed without a durable metadata row.");return{dataset:C(p),duplicate:!1}}async function nt(t){await R();let o=f(t.state);if(!o)throw new Error("state must be a two-letter US state abbreviation");let a=F(o),r=X(t.sourceUrl),n=M("sha256").update(t.csv).digest("hex"),u=await h(a);if(u?.sha256===n)return{dataset:u,duplicate:!0};let d=$[o],E=B(t.csv,d),p=`loc_${z().replaceAll("-","")}`,i=N();await i.execute({sql:`
-      INSERT INTO location_data_imports (
-        id, dataset_kind, source_url, sha256, row_count, group_count, state_count,
-        source_row_count, rejected_row_count, zip_count, covered_states_json, coverage_complete, status
-      ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, 0, 0, ?, 1, 'loading')
-    `,args:[p,a,r,n,E.places.length,E.places.length,E.sourceRowCount,JSON.stringify([o])]});try{for(let s=0;s<E.places.length;s+=A){let l=E.places.slice(s,s+A).map(c=>({sql:`
-          INSERT INTO location_census_places (
-            import_id, state_abbr, city_key, city, census_name, estimates_base_2020,
-            population_2020, population_2021, population_2022, population_2023,
-            population_2024, population_2025
-          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `,args:[p,o,c.cityKey,c.city,c.censusName,c.estimatesBase2020,c.populations[2020],c.populations[2021],c.populations[2022],c.populations[2023],c.populations[2024],c.populations[2025]]}));await i.batch(l,"write")}await i.batch([{sql:"UPDATE location_data_imports SET status = 'ready', completed_at = datetime('now'), error = NULL WHERE id = ? AND status = 'loading'",args:[p]},{sql:`
-          INSERT INTO location_data_active (dataset_kind, import_id, activated_at)
-          VALUES (?, ?, datetime('now'))
-          ON CONFLICT(dataset_kind) DO UPDATE SET import_id = excluded.import_id, activated_at = excluded.activated_at
-        `,args:[a,p]}],"write")}catch(s){let l=s instanceof Error?s.message:String(s);throw await i.execute({sql:"UPDATE location_data_imports SET status = 'failed', error = ?, completed_at = datetime('now') WHERE id = ?",args:[l.slice(0,1e3),p]}).catch(()=>{}),s}let e=(await i.execute({sql:"SELECT * FROM location_data_imports WHERE id = ? LIMIT 1",args:[p]})).rows[0];if(!e)throw new Error("Hosted Census import completed without a durable metadata row.");return{dataset:C(e),duplicate:!1}}async function it(t){let o=f(t.state);if(!o)throw new Error("state must be a two-letter US state abbreviation");if(!Number.isSafeInteger(t.minPopulation)||t.minPopulation<0)throw new Error("minPopulation must be a non-negative integer");if(!Number.isSafeInteger(t.maxResults)||t.maxResults<1||t.maxResults>100)throw new Error("maxResults must be between 1 and 100");let a=await h(F(o)),n=t.includeZipGroups||!!t.zip?await Q(o):{dataset:null,groups:new Map};if(!a)return{markets:[],populationDataset:null,zipDataset:n.dataset,warnings:[]};let u=await N().execute({sql:`
-      SELECT *
-      FROM location_census_places
-      WHERE import_id = ? AND state_abbr = ?
-      ORDER BY city COLLATE NOCASE ASC
-    `,args:[a.id,o]}),d=b(t.cityQuery??""),E=t.zip?.trim(),p=`population_${t.populationYear}`,i=u.rows.map(e=>{let s=e[p];if(s==null)return null;let l=Number(s);if(!Number.isSafeInteger(l)||l<t.minPopulation)return null;let c=String(e.city),T=String(e.city_key);if(d&&!T.includes(d))return null;let m=n.groups.get(T),S=m?.zips??[];return E&&!S.includes(E)?null:{city:c,state:o,location:`${c}, ${o}`,cityKey:`${c}|${o}`,censusName:String(e.census_name),population:l,populationYear:t.populationYear,estimatesBase2020:e.estimates_base_2020==null?null:Number(e.estimates_base_2020),zips:t.includeZipGroups?S:[],counties:t.includeZipGroups?m?.counties??[]:[]}}).filter(e=>e!==null).sort((e,s)=>s.population-e.population||e.city.localeCompare(s.city)).slice(0,t.maxResults),_=[];return t.includeZipGroups&&n.dataset&&i.some(e=>e.zips.length===0)&&_.push("Some hosted Census places did not match the hosted ZIP city groups."),{markets:i,populationDataset:a,zipDataset:n.dataset,warnings:_}}export{D as a,K as b,P as c,k as d,B as e,W as f,R as g,h,Z as i,st as j,Q as k,at as l,nt as m,it as n};

package/dist/chunk-VVSHZV5S.js

@@ -1,73 +0,0 @@
-import{c as l}from"./chunk-4ZVO6GX4.js";import{a as _}from"./chunk-WNUXMPVX.js";var c=166667e-10,S="owner_confirmed_2026-10-12";var N=.00111,L=4,p=4e-4;function o(e,t){let r=process.env[e]?.trim();if(!r)return t;let s=Number(r);return Number.isFinite(s)&&s>=0?s:t}var R=o("NANGO_USD_PER_CONNECTION_MONTH",1),A=o("NANGO_USD_PER_FUNCTION_RUN",1e-4),U=o("NANGO_USD_PER_PROXY_REQUEST",1e-4),m=o("NANGO_USD_PER_COMPUTE_SEC",2e-4),g=o("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_USD_PER_GB",8),D=o("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_USD_PER_REQUEST",.002836168);function u(e,t){return Math.max(0,e)/1e3*c}function d(e,t){return e==="fal_wizper"?Math.max(0,t)/L*N:e==="deepinfra_qwen"?Math.max(0,t)/1e3*p:e==="openrouter"||e==="mcp_memory_video"||e==="mcp_memory_ai"?Math.max(0,t):e==="nango_connection"?Math.max(0,t)*R:e==="nango_function_run"?Math.max(0,t)*A:e==="nango_proxy_request"?Math.max(0,t)*U:e==="nango_compute"?Math.max(0,t)*m:e==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api"?Math.max(0,t)/1e9*g:0}import{randomUUID as i}from"crypto";var E=!1,n=null;async function T(){if(!E)return n||(n=O().finally(()=>{n=null}),n)}async function O(){let e=_(),t=await e.execute(`
-    SELECT
-      (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('\u006b\u0065\u0072\u006e\u0065\u006c_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
-      AND (SELECT COUNT(*) FROM pragma_table_info('\u006b\u0065\u0072\u006e\u0065\u006c_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
-      AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
-      AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
-      AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
-      AS ready
-  `);if(Number(t.rows[0]?.ready??0)===1){E=!0;return}await e.execute(`
-    CREATE TABLE IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log (
-      id                    TEXT PRIMARY KEY,
-      \u006b\u0065\u0072\u006e\u0065\u006c_session_id     TEXT,
-      op                    TEXT,
-      source                TEXT NOT NULL,
-      probe_run_id          TEXT,
-      user_id               INTEGER,
-      stealth               INTEGER,
-      headless_sent         INTEGER,
-      proxy_used            INTEGER,
-      fallback              INTEGER NOT NULL DEFAULT 0,
-      opened_at             TEXT,
-      closed_at             TEXT,
-      duration_ms           INTEGER NOT NULL DEFAULT 0,
-      est_cost_usd_headless REAL NOT NULL DEFAULT 0,
-      est_cost_usd_headful  REAL NOT NULL DEFAULT 0,
-      error                 TEXT,
-      created_at            TEXT NOT NULL DEFAULT (datetime('now'))
-    )
-  `),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_op ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(op)"),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_probe ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(probe_run_id)"),await e.execute("CREATE INDEX IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log_created ON \u006b\u0065\u0072\u006e\u0065\u006c_session_log(created_at)");try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN proxy_source TEXT")}catch{}try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN proxy_type TEXT")}catch{}try{await e.execute("ALTER TABLE \u006b\u0065\u0072\u006e\u0065\u006c_session_log ADD COLUMN method TEXT")}catch{}await e.execute(`
-    CREATE TABLE IF NOT EXISTS vendor_usage_log (
-      id           TEXT PRIMARY KEY,
-      op           TEXT,
-      probe_run_id TEXT,
-      user_id      INTEGER,
-      vendor       TEXT NOT NULL,
-      model        TEXT,
-      units        REAL NOT NULL DEFAULT 0,
-      unit_type    TEXT,
-      est_cost_usd REAL NOT NULL DEFAULT 0,
-      error        TEXT,
-      source_key   TEXT,
-      provider_duration_ms INTEGER,
-      provider_captcha INTEGER,
-      provider_status TEXT,
-      created_at   TEXT NOT NULL DEFAULT (datetime('now'))
-    )
-  `),await e.execute("CREATE INDEX IF NOT EXISTS vendor_usage_log_op ON vendor_usage_log(op)"),await e.execute("CREATE INDEX IF NOT EXISTS vendor_usage_log_probe ON vendor_usage_log(probe_run_id)"),await e.execute("CREATE INDEX IF NOT EXISTS vendor_usage_log_created ON vendor_usage_log(created_at)");try{await e.execute("ALTER TABLE vendor_usage_log ADD COLUMN method TEXT")}catch{}try{await e.execute("ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT")}catch{}try{await e.execute("ALTER TABLE vendor_usage_log ADD COLUMN provider_duration_ms INTEGER")}catch{}try{await e.execute("ALTER TABLE vendor_usage_log ADD COLUMN provider_captcha INTEGER")}catch{}try{await e.execute("ALTER TABLE vendor_usage_log ADD COLUMN provider_status TEXT")}catch{}await e.execute("CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL"),await e.execute(`
-    CREATE TABLE IF NOT EXISTS cost_probe_runs (
-      id                            TEXT PRIMARY KEY,
-      tool                          TEXT NOT NULL,
-      target                        TEXT,
-      started_at                    TEXT NOT NULL DEFAULT (datetime('now')),
-      wall_ms                       INTEGER NOT NULL DEFAULT 0,
-      success                       INTEGER NOT NULL DEFAULT 0,
-      error                         TEXT,
-      http_only                     INTEGER NOT NULL DEFAULT 0,
-      \u006b\u0065\u0072\u006e\u0065\u006c_used                   INTEGER NOT NULL DEFAULT 0,
-      \u006b\u0065\u0072\u006e\u0065\u006c_sessions               INTEGER NOT NULL DEFAULT 0,
-      \u006b\u0065\u0072\u006e\u0065\u006c_seconds_total          REAL NOT NULL DEFAULT 0,
-      \u006b\u0065\u0072\u006e\u0065\u006c_tier_observed          TEXT,
-      est_\u006b\u0065\u0072\u006e\u0065\u006c_cost_usd_headless  REAL NOT NULL DEFAULT 0,
-      est_\u006b\u0065\u0072\u006e\u0065\u006c_cost_usd_headful   REAL NOT NULL DEFAULT 0,
-      vendor_cost_usd               REAL NOT NULL DEFAULT 0,
-      charged_credits               REAL NOT NULL DEFAULT 0,
-      margin_usd_headless_starter   REAL NOT NULL DEFAULT 0,
-      margin_usd_headful_starter    REAL NOT NULL DEFAULT 0,
-      notes                         TEXT
-    )
-  `),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function h(e){try{await T();let t=l(),r=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO \u006b\u0065\u0072\u006e\u0065\u006c_session_log
-        (id, \u006b\u0065\u0072\u006e\u0065\u006c_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId??null,t?.op??null,e.source,t?.probeRunId??null,t?.userId??null,a(e.stealth),a(e.headlessSent),a(e.proxyUsed),e.proxySource??null,e.proxyType??null,e.fallback?1:0,new Date(e.openedAtMs).toISOString(),new Date(e.closedAtMs).toISOString(),r,u(r,!1),u(r,!0),e.error??null,t?.subOp??null]})}catch(t){console.warn("[cost-telemetry] record\u004b\u0065\u0072\u006e\u0065\u006cSession failed:",t instanceof Error?t.message:String(t))}}async function f(e){try{await T();let t=l();return(await _().execute({sql:`INSERT OR IGNORE INTO vendor_usage_log
-        (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.op??t?.op??null,e.probeRunId??t?.probeRunId??null,e.userId??t?.userId??null,e.vendor,e.model??null,e.units,e.unitType,d(e.vendor,e.units),e.error??null,e.method??t?.subOp??null,e.sourceKey??null,e.providerDurationMs??null,a(e.providerCaptcha),e.providerStatus??null]})).rowsAffected===1}catch(t){return console.warn("[cost-telemetry] recordVendorUsage failed:",t instanceof Error?t.message:String(t)),!1}}function a(e){return e==null?null:e?1:0}export{c as a,S as b,u as c,d,T as e,h as f,f as g};

package/dist/chunk-WHVK6FUB.js

@@ -1,11 +0,0 @@
-import{c as b}from"./chunk-U3NV72AL.js";import{g as y,i as f}from"./chunk-C7MGJUTU.js";var h={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},E=/error|message/i,x=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),T=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,N=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),O=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),L=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function k(e){return e.replace(L,"[browser-service]")}var D=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),j=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),q=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),F=new Set(["provider_session_api","client_runtime","extractor","unknown"]),U=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),M=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),$=512,z=320;function p(e,r){return typeof e=="string"&&r.has(e)?e:null}function H(e){return typeof e!="string"?null:U.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function Q(e){return typeof e!="string"||e.length<1||e.length>$||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function R(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function A(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function C(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let t=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return k(t).slice(0,z)}function w(e){return typeof e=="boolean"?e:null}function V(e){return{browser_provider:p(e.browser_provider,D),provider_session_id:Q(e.provider_session_id),provider_session_observed_at:A(e.provider_session_observed_at),provider_lookup_status:p(e.provider_lookup_status,j),provider_status:p(e.provider_status,q),provider_error_code:H(e.provider_error_code),provider_error_message:C(e.provider_error_message),provider_reason_source:p(e.provider_reason_source,F),provider_duration_ms:R(e.provider_duration_ms),provider_bandwidth_bytes:R(e.provider_bandwidth_bytes),provider_captcha:w(e.provider_captcha),provider_checked_at:A(e.provider_checked_at),provider_disconnect_observed:w(e.provider_disconnect_observed),provider_disconnect_message:C(e.provider_disconnect_message)}}function l(e,r=""){if(typeof e=="string"){let t=k(e);return E.test(r)&&(x.test(e)||T.test(e))?t=y("service_unavailable"):E.test(r)&&(t=f(t)),t}if(Array.isArray(e))return e.map(t=>l(t,r));if(e!==null&&typeof e=="object"){let t={};for(let[n,o]of Object.entries(e)){if(N.has(n))continue;let i=h[n]??n;if(O.has(n)){t[i]=null;continue}t[i]=l(o,n)}return t}return e}function Y(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return l(r);let t=r,n=l(t);return Object.keys(t).some(i=>M.has(i))?{...n,...V(t)}:n})}function X(e){let r=e?.diagnostics;if(!r)return e;let t=r.serpCompleteness,n=t?.queryIntegrity;return!r.debug&&!n?e:{...e,diagnostics:{...r,...r.debug?{debug:l(r.debug)}:{},...n?{serpCompleteness:{...t,queryIntegrity:{...n,outboundProviderMethod:"search"}}}:{}}}}var ee="strftime('%Y-%m-%dT%H:%M:%fZ','now')",_="julianday('now')";var oe=20,m="(json_extract(options,'$.executionOwner')='direct' AND json_extract(options,'$.serpOnly') IS NOT 1)",se=`NOT COALESCE(${m},0) AND json_extract(options,'$.syncExecutionProtocol') IS NOT 1 AND json_extract(options,'$.queueExecutionProtocol') IS NULL`,ie=`${m} AND julianday(json_extract(options,'$.paaLeaseExpiresAt')) > ${_}`,ae=`${m} AND (
-  (json_extract(options,'$.paaLeaseExpiresAt') IS NOT NULL AND COALESCE(julianday(json_extract(options,'$.paaLeaseExpiresAt')),0) <= ${_})
-  OR (json_extract(options,'$.paaLeaseExpiresAt') IS NULL AND julianday(started_at) <= julianday('now','-15 minutes'))
-)`;function ue(e){return{jobId:e.id,userId:e.user_id,leaseToken:typeof e.options.paaLeaseToken=="string"?e.options.paaLeaseToken:null,recoveryCount:Number(e.options.paaRecoveryCount??0),resultJson:e.result==null?null:JSON.stringify(e.result)}}function de(e,r){return!r||r.jobId!==e||!Number.isSafeInteger(r.userId)||r.userId<=0||!Number.isSafeInteger(r.recoveryCount)||r.recoveryCount<0||r.leaseToken!==null&&(typeof r.leaseToken!="string"||r.leaseToken.length===0)?{sql:"0",args:[]}:{sql:`user_id=? AND json_extract(options,'$.paaLeaseToken') IS ?
-    AND COALESCE(json_extract(options,'$.paaRecoveryCount'),0)=? AND json(result) IS json(?)`,args:[r.userId,r.leaseToken,r.recoveryCount,r.resultJson]}}function le(e,r,t){let n={userId:e.user_id,tool:"harvest_paa",jobId:e.id,sql:"0",args:[]},o=e.options.requestFingerprint;if(e.options.executionProtocol!==1||e.options.executionOwner!=="direct"||e.options.serpOnly===!0||typeof o!="string"||!/^[a-f0-9]{64}$/.test(o)||!r||t.userId!==e.user_id||t.key!==e.options.billingDebitKey||t.amountMc!==e.options.billingHoldMc)return n;let i=b(t);return{...n,sql:`EXISTS (SELECT 1 FROM jobs j WHERE j.id=? AND j.user_id=? AND j.query=? AND j.status='running'
-      AND json_extract(j.options,'$.executionOwner')='direct' AND json_extract(j.options,'$.serpOnly') IS NOT 1
-      AND json_extract(j.options,'$.executionProtocol')=1 AND json_extract(j.options,'$.requestFingerprint')=?
-      AND json_extract(j.options,'$.paaLeaseToken')=?
-      AND julianday(json_extract(j.options,'$.paaLeaseExpiresAt')) > ${_}
-      AND json_extract(j.options,'$.billingDebitKey')=? AND json_extract(j.options,'$.billingHoldMc')=?
-      AND json_extract(j.options,'$.billingSettledAt') IS NULL AND (${i.sql}))`,args:[e.id,e.user_id,e.query,o,r,t.key,t.amountMc,...i.args]}}import{createHash as B}from"crypto";var G=["observed_present","observed_absent","incomplete","not_requested","unknown","unsupported"],J=["provider_transport","invalid_payload","serp_readiness","query_integrity","pagination","optional_feature_parsing","unknown"],P=class extends Error{stage;failureCode;constructor(r,t,n){super(n),this.name="SerpExtractionError",this.stage=r,this.failureCode=t}};function g(e){return B("sha256").update(e).digest("hex")}function W(e){return e.normalize("NFKC").toLocaleLowerCase("en-US").split(/[^\p{L}\p{N}]+/u).filter(Boolean).slice(0,64)}function _e(e){try{let r=new URL(e);return r.hash="",r.pathname.length>1&&(r.pathname=r.pathname.replace(/\/+$/,"")),r.toString()}catch{return e.trim()}}function ge(e,r){let t=e===2?[1,2]:[1],n=r.filter(s=>s.attempted).map(s=>s.page),o=r.filter(s=>s.captured&&s.resultCount>0).map(s=>s.page),i=r.find(s=>s.failureCode);return{status:t.every(s=>o.includes(s))?"complete":"incomplete",requestedPages:t,attemptedPages:n,capturedPages:o,pageResultCounts:t.map(s=>({page:s,resultCount:r.find(S=>S.page===s)?.resultCount??0})),providerRequestCount:n.length,evidence:r,failureCode:i?.failureCode??null}}function Se(e){return e.requested?e.moduleObserved?!e.rawShapeValid||e.rawCount>0&&e.mappedCount===0?"incomplete":e.mappedCount>0?"observed_present":"observed_absent":"unknown":"not_requested"}function me(e){let r=g(e.query),t=e.finalQuery==null?null:g(e.finalQuery),n=e.locationResolutionSource==="unresolved";return{status:t!==null&&t!==r?"query_mismatch":n?"unresolved_location":"matched",queryHash:r,normalizedQueryTokens:W(e.query),canonicalLocation:e.canonicalLocation??null,locationResolutionSource:e.locationResolutionSource,locationAmbiguousCandidates:Math.max(0,Math.trunc(e.locationAmbiguousCandidates??0)),uulePresent:!!e.uule,uuleHash:e.uule?g(e.uule):null,outboundProviderMethod:e.outboundProviderMethod,finalQueryHash:t}}function ve(e){return g(JSON.stringify(Object.fromEntries(Object.entries(e).filter(([r,t])=>r!=="requestFingerprint"&&t!==void 0).sort(([r],[t])=>r.localeCompare(t)))))}function c(e){return e&&typeof e=="object"&&!Array.isArray(e)?e:{}}function v(e){return typeof e=="number"&&Number.isFinite(e)&&e>=0?Math.trunc(e):0}function u(e,r){return typeof e=="string"&&e.length<=r?e:null}function ye(e){let r=c(e),t=c(r.pagination),n=c(r.features),o=c(r.queryIntegrity),i=a=>Array.isArray(a)?a.filter(d=>d===1||d===2).slice(0,2):[],s=i(t.requestedPages);if(s.length===0)return null;let S=new Set(G),I=new Set(J);return{pagination:{status:t.status==="complete"?"complete":"incomplete",requestedPages:s,attemptedPages:i(t.attemptedPages),capturedPages:i(t.capturedPages),pageResultCounts:Array.isArray(t.pageResultCounts)?t.pageResultCounts.slice(0,2).map(a=>{let d=c(a);return{page:d.page===2?2:1,resultCount:v(d.resultCount)}}):[],providerRequestCount:v(t.providerRequestCount),failureCode:u(t.failureCode,100)},features:Object.fromEntries(["localPack","forums","videos","aiOverview","whatPeopleSaying","aiMode","paaPreview","entityIds"].map(a=>[a,S.has(String(n[a]))?n[a]:"unknown"])),queryIntegrity:{status:["matched","unresolved_location","query_mismatch"].includes(String(o.status))?o.status:"query_mismatch",queryHash:u(o.queryHash,64)??"",normalizedQueryTokens:Array.isArray(o.normalizedQueryTokens)?o.normalizedQueryTokens.slice(0,64).flatMap(a=>typeof a=="string"?[a.slice(0,100)]:[]):[],canonicalLocation:u(o.canonicalLocation,500),locationResolutionSource:u(o.locationResolutionSource,100)??"unknown",locationAmbiguousCandidates:v(o.locationAmbiguousCandidates),uulePresent:o.uulePresent===!0,uuleHash:u(o.uuleHash,64),outboundProviderMethod:"search",finalQueryHash:u(o.finalQueryHash,64)},failureStage:I.has(String(r.failureStage))?r.failureStage:null}}export{l as a,Y as b,X as c,ee as d,oe as e,m as f,se as g,ie as h,ae as i,ue as j,de as k,le as l,P as m,_e as n,ge as o,Se as p,me as q,ve as r,ye as s};

package/dist/chunk-WNUXMPVX.js

@@ -1 +0,0 @@
-import{createClient as t}from"@libsql/client/http";var n=process.env.TURSO_DATABASE_URL??"file:./paa-api.db",l=process.env.TURSO_AUTH_TOKEN,e=null;function o(){return e||(e=t({url:n,authToken:l})),e}export{o as a};