mcp-scraper 0.88.2 → 0.89.1

package/dist/chunk-IJ2AO7AO.js

@@ -0,0 +1,102 @@
+import{b as O}from"./chunk-KJQXUZ4Y.js";import{s as N}from"./chunk-63T4W2LX.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
+        CREATE TABLE IF NOT EXISTS location_data_imports (
+          id           TEXT PRIMARY KEY,
+          dataset_kind TEXT NOT NULL,
+          source_url   TEXT,
+          sha256       TEXT NOT NULL,
+          row_count    INTEGER NOT NULL DEFAULT 0,
+          group_count  INTEGER NOT NULL DEFAULT 0,
+          state_count  INTEGER NOT NULL DEFAULT 0,
+          source_row_count   INTEGER NOT NULL DEFAULT 0,
+          rejected_row_count INTEGER NOT NULL DEFAULT 0,
+          zip_count          INTEGER NOT NULL DEFAULT 0,
+          covered_states_json TEXT NOT NULL DEFAULT '[]',
+          coverage_complete  INTEGER NOT NULL DEFAULT 0,
+          status       TEXT NOT NULL,
+          error        TEXT,
+          created_at   TEXT NOT NULL DEFAULT (datetime('now')),
+          completed_at TEXT
+        )
+      `);let o=new Set((await t.execute("PRAGMA table_info(location_data_imports)")).rows.map(r=>String(r.name))),a=[["source_row_count","INTEGER NOT NULL DEFAULT 0"],["rejected_row_count","INTEGER NOT NULL DEFAULT 0"],["zip_count","INTEGER NOT NULL DEFAULT 0"],["covered_states_json","TEXT NOT NULL DEFAULT '[]'"],["coverage_complete","INTEGER NOT NULL DEFAULT 0"]];for(let[r,n]of a)o.has(r)||await t.execute(`ALTER TABLE location_data_imports ADD COLUMN ${r} ${n}`);await t.execute("CREATE INDEX IF NOT EXISTS location_data_imports_kind_created ON location_data_imports(dataset_kind, created_at DESC)"),await t.execute(`
+        CREATE TABLE IF NOT EXISTS location_zip_groups (
+          import_id     TEXT NOT NULL REFERENCES location_data_imports(id) ON DELETE CASCADE,
+          state_abbr    TEXT NOT NULL,
+          city_key      TEXT NOT NULL,
+          city          TEXT NOT NULL,
+          zips_json     TEXT NOT NULL,
+          counties_json TEXT NOT NULL,
+          PRIMARY KEY(import_id, state_abbr, city_key)
+        )
+      `),await t.execute("CREATE INDEX IF NOT EXISTS location_zip_groups_state ON location_zip_groups(import_id, state_abbr, city_key)"),await t.execute(`
+        CREATE TABLE IF NOT EXISTS location_census_places (
+          import_id           TEXT NOT NULL REFERENCES location_data_imports(id) ON DELETE CASCADE,
+          state_abbr          TEXT NOT NULL,
+          city_key            TEXT NOT NULL,
+          city                TEXT NOT NULL,
+          census_name         TEXT NOT NULL,
+          estimates_base_2020 INTEGER,
+          population_2020     INTEGER,
+          population_2021     INTEGER,
+          population_2022     INTEGER,
+          population_2023     INTEGER,
+          population_2024     INTEGER,
+          population_2025     INTEGER,
+          PRIMARY KEY(import_id, state_abbr, city_key)
+        )
+      `),await t.execute("CREATE INDEX IF NOT EXISTS location_census_places_state ON location_census_places(import_id, state_abbr, city_key)"),await t.execute(`
+        CREATE TABLE IF NOT EXISTS location_data_active (
+          dataset_kind TEXT PRIMARY KEY,
+          import_id    TEXT NOT NULL REFERENCES location_data_imports(id),
+          activated_at TEXT NOT NULL DEFAULT (datetime('now'))
+        )
+      `)})().catch(o=>{throw w=null,o})),await w}async function h(t){await R();let a=(await N().execute({sql:`
+      SELECT i.*
+      FROM location_data_active a
+      JOIN location_data_imports i ON i.id = a.import_id
+      WHERE a.dataset_kind = ? AND i.status = 'ready'
+      LIMIT 1
+    `,args:[t]})).rows[0];return a?C(a):null}async function Z(){return h(D)}async function st(){return await R(),(await N().execute(`
+    SELECT i.*
+    FROM location_data_active a
+    JOIN location_data_imports i ON i.id = a.import_id
+    WHERE i.status = 'ready'
+    ORDER BY i.dataset_kind ASC
+  `)).rows.map(o=>C(o))}async function Q(t){let o=f(t);if(!o)throw new Error("state must be a two-letter US state abbreviation");let a=await Z();if(!a)return{dataset:null,groups:new Map};let r=await N().execute({sql:`
+      SELECT city, city_key, state_abbr, zips_json, counties_json
+      FROM location_zip_groups
+      WHERE import_id = ? AND state_abbr = ?
+      ORDER BY city COLLATE NOCASE ASC
+    `,args:[a.id,o]}),n=new Map;for(let u of r.rows){let d=String(u.city_key);n.set(d,{city:String(u.city),cityKey:d,state:String(u.state_abbr),zips:JSON.parse(String(u.zips_json)),counties:JSON.parse(String(u.counties_json))})}return{dataset:a,groups:n}}async function at(t){await R();let o=X(t.sourceUrl),a=M("sha256").update(t.csv).digest("hex"),r=await Z();if(r?.sha256===a)return{dataset:r,duplicate:!0};let n=W(t.csv);J(n);let u=`loc_${z().replaceAll("-","")}`,d=N();await d.execute({sql:`
+      INSERT INTO location_data_imports (
+        id, dataset_kind, source_url, sha256, row_count, group_count, state_count,
+        source_row_count, rejected_row_count, zip_count, covered_states_json, coverage_complete, status
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 'loading')
+    `,args:[u,D,o,a,n.rowCount,n.groups.length,n.stateCount,n.sourceRowCount,n.rejectedRowCount,n.zipCount,JSON.stringify(n.coveredStates)]});try{for(let i=0;i<n.groups.length;i+=A){let _=n.groups.slice(i,i+A).map(e=>({sql:`
+          INSERT INTO location_zip_groups (
+            import_id, state_abbr, city_key, city, zips_json, counties_json
+          ) VALUES (?, ?, ?, ?, ?, ?)
+        `,args:[u,e.state,e.cityKey,e.city,JSON.stringify(e.zips),JSON.stringify(e.counties)]}));await d.batch(_,"write")}await d.batch([{sql:"UPDATE location_data_imports SET status = 'ready', completed_at = datetime('now'), error = NULL WHERE id = ? AND status = 'loading'",args:[u]},{sql:`
+          INSERT INTO location_data_active (dataset_kind, import_id, activated_at)
+          VALUES (?, ?, datetime('now'))
+          ON CONFLICT(dataset_kind) DO UPDATE SET import_id = excluded.import_id, activated_at = excluded.activated_at
+        `,args:[D,u]}],"write")}catch(i){let _=i instanceof Error?i.message:String(i);throw await d.execute({sql:"UPDATE location_data_imports SET status = 'failed', error = ?, completed_at = datetime('now') WHERE id = ?",args:[_.slice(0,1e3),u]}).catch(()=>{}),i}let p=(await d.execute({sql:"SELECT * FROM location_data_imports WHERE id = ? LIMIT 1",args:[u]})).rows[0];if(!p)throw new Error("Hosted location import completed without a durable metadata row.");return{dataset:C(p),duplicate:!1}}async function nt(t){await R();let o=f(t.state);if(!o)throw new Error("state must be a two-letter US state abbreviation");let a=F(o),r=X(t.sourceUrl),n=M("sha256").update(t.csv).digest("hex"),u=await h(a);if(u?.sha256===n)return{dataset:u,duplicate:!0};let d=$[o],E=B(t.csv,d),p=`loc_${z().replaceAll("-","")}`,i=N();await i.execute({sql:`
+      INSERT INTO location_data_imports (
+        id, dataset_kind, source_url, sha256, row_count, group_count, state_count,
+        source_row_count, rejected_row_count, zip_count, covered_states_json, coverage_complete, status
+      ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, 0, 0, ?, 1, 'loading')
+    `,args:[p,a,r,n,E.places.length,E.places.length,E.sourceRowCount,JSON.stringify([o])]});try{for(let s=0;s<E.places.length;s+=A){let l=E.places.slice(s,s+A).map(c=>({sql:`
+          INSERT INTO location_census_places (
+            import_id, state_abbr, city_key, city, census_name, estimates_base_2020,
+            population_2020, population_2021, population_2022, population_2023,
+            population_2024, population_2025
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `,args:[p,o,c.cityKey,c.city,c.censusName,c.estimatesBase2020,c.populations[2020],c.populations[2021],c.populations[2022],c.populations[2023],c.populations[2024],c.populations[2025]]}));await i.batch(l,"write")}await i.batch([{sql:"UPDATE location_data_imports SET status = 'ready', completed_at = datetime('now'), error = NULL WHERE id = ? AND status = 'loading'",args:[p]},{sql:`
+          INSERT INTO location_data_active (dataset_kind, import_id, activated_at)
+          VALUES (?, ?, datetime('now'))
+          ON CONFLICT(dataset_kind) DO UPDATE SET import_id = excluded.import_id, activated_at = excluded.activated_at
+        `,args:[a,p]}],"write")}catch(s){let l=s instanceof Error?s.message:String(s);throw await i.execute({sql:"UPDATE location_data_imports SET status = 'failed', error = ?, completed_at = datetime('now') WHERE id = ?",args:[l.slice(0,1e3),p]}).catch(()=>{}),s}let e=(await i.execute({sql:"SELECT * FROM location_data_imports WHERE id = ? LIMIT 1",args:[p]})).rows[0];if(!e)throw new Error("Hosted Census import completed without a durable metadata row.");return{dataset:C(e),duplicate:!1}}async function it(t){let o=f(t.state);if(!o)throw new Error("state must be a two-letter US state abbreviation");if(!Number.isSafeInteger(t.minPopulation)||t.minPopulation<0)throw new Error("minPopulation must be a non-negative integer");if(!Number.isSafeInteger(t.maxResults)||t.maxResults<1||t.maxResults>100)throw new Error("maxResults must be between 1 and 100");let a=await h(F(o)),n=t.includeZipGroups||!!t.zip?await Q(o):{dataset:null,groups:new Map};if(!a)return{markets:[],populationDataset:null,zipDataset:n.dataset,warnings:[]};let u=await N().execute({sql:`
+      SELECT *
+      FROM location_census_places
+      WHERE import_id = ? AND state_abbr = ?
+      ORDER BY city COLLATE NOCASE ASC
+    `,args:[a.id,o]}),d=b(t.cityQuery??""),E=t.zip?.trim(),p=`population_${t.populationYear}`,i=u.rows.map(e=>{let s=e[p];if(s==null)return null;let l=Number(s);if(!Number.isSafeInteger(l)||l<t.minPopulation)return null;let c=String(e.city),T=String(e.city_key);if(d&&!T.includes(d))return null;let m=n.groups.get(T),S=m?.zips??[];return E&&!S.includes(E)?null:{city:c,state:o,location:`${c}, ${o}`,cityKey:`${c}|${o}`,censusName:String(e.census_name),population:l,populationYear:t.populationYear,estimatesBase2020:e.estimates_base_2020==null?null:Number(e.estimates_base_2020),zips:t.includeZipGroups?S:[],counties:t.includeZipGroups?m?.counties??[]:[]}}).filter(e=>e!==null).sort((e,s)=>s.population-e.population||e.city.localeCompare(s.city)).slice(0,t.maxResults),_=[];return t.includeZipGroups&&n.dataset&&i.some(e=>e.zips.length===0)&&_.push("Some hosted Census places did not match the hosted ZIP city groups."),{markets:i,populationDataset:a,zipDataset:n.dataset,warnings:_}}export{D as a,K as b,P as c,k as d,B as e,W as f,R as g,h,Z as i,st as j,Q as k,at as l,nt as m,it as n};

package/dist/chunk-KJQXUZ4Y.js

@@ -0,0 +1,4 @@
+function l(r){let t=[],e=[],n="",i=!1;for(let s=0;s<r.length;s+=1){let o=r[s],c=r[s+1];if(i){o==='"'&&c==='"'?(n+='"',s+=1):o==='"'?i=!1:n+=o;continue}o==='"'?i=!0:o===","?(e.push(n),n=""):o===`
+`?(e.push(n),t.push(e),e=[],n=""):o!=="\r"&&(n+=o)}return(n.length>0||e.length>0)&&(e.push(n),t.push(e)),t}function u(r){let t=l(r).filter(n=>n.some(i=>i.trim()!=="")),e=t[0]?.map(n=>n.trim())??[];return t.slice(1).map(n=>{let i={};for(let s=0;s<e.length;s+=1)i[e[s]]=n[s]??"";return i})}function f(r){if(r==null)return"";let t=String(r);return/[",\n\r]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function g(r,t){return[r.join(","),...t.map(e=>r.map(n=>f(e[n])).join(","))].join(`
+`)+`
+`}export{l as a,u as b,g as c};