mcp-scraper 0.95.2 → 0.96.1

package/dist/{chunk-ZH75KWBI.js → chunk-UNEJYZ4T.js}

@@ -1,4 +1,4 @@
-import{b as O}from"./chunk-KJQXUZ4Y.js";import{t as N}from"./chunk-7YOXSMHG.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
+import{a as N}from"./chunk-WNUXMPVX.js";import{b as O}from"./chunk-KJQXUZ4Y.js";import{createHash as M,randomUUID as z}from"crypto";var D="us_zip_groups",K="us_census_places",P=25e3,k=["AL","AK","AZ","AR","CA","CO","CT","DE","DC","FL","GA","HI","ID","IL","IN","IA","KS","KY","LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND","OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"],$={AL:"01",AK:"02",AZ:"04",AR:"05",CA:"06",CO:"08",CT:"09",DE:"10",DC:"11",FL:"12",GA:"13",HI:"15",ID:"16",IL:"17",IN:"18",IA:"19",KS:"20",KY:"21",LA:"22",ME:"23",MD:"24",MA:"25",MI:"26",MN:"27",MS:"28",MO:"29",MT:"30",NE:"31",NV:"32",NH:"33",NJ:"34",NM:"35",NY:"36",NC:"37",ND:"38",OH:"39",OK:"40",OR:"41",PA:"42",RI:"44",SC:"45",SD:"46",TN:"47",TX:"48",UT:"49",VT:"50",VA:"51",WA:"53",WV:"54",WI:"55",WY:"56"},x=new Set(k),A=200,H=2e3,v=null,w=null;function b(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim()}function Y(t){if(t==null)return[];try{let o=JSON.parse(String(t));return Array.isArray(o)?o.map(String):[]}catch{return[]}}function C(t){return{id:String(t.id),kind:String(t.dataset_kind),sourceUrl:t.source_url==null?null:String(t.source_url),sha256:String(t.sha256),rowCount:Number(t.row_count??0),groupCount:Number(t.group_count??0),stateCount:Number(t.state_count??0),sourceRowCount:Number(t.source_row_count??t.row_count??0),rejectedRowCount:Number(t.rejected_row_count??0),zipCount:Number(t.zip_count??0),coveredStates:Y(t.covered_states_json),coverageComplete:Number(t.coverage_complete??0)===1,status:String(t.status),createdAt:String(t.created_at),completedAt:t.completed_at==null?null:String(t.completed_at),error:t.error==null?null:String(t.error)}}function F(t){return`${K}:${t}`}function X(t){let o=t?.trim();if(!o)return null;if(o.length>H)throw new Error(`sourceUrl must be ${H} characters or fewer`);let a;try{a=new URL(o)}catch{throw new Error("sourceUrl must be a valid URL")}return a.username="",a.password="",a.search="",a.hash="",a.toString()}function q(t){let o=t.trim();return/^\d{1,5}$/.test(o)?o.padStart(5,"0"):null}function f(t){let o=t.trim().toUpperCase();return x.has(o)?o:null}function y(t,o){for(let a of o){let r=t[a];if(r!==void 0)return r}return""}function g(t){if(t===void 0||t.trim()==="")return null;let o=Number(t);return Number.isFinite(o)&&o>=0?Math.trunc(o):null}function V(t){return/^Nashville-Davidson metropolitan government/i.test(t)?"Nashville":t.replace(/\s+(city|town|village|municipality|borough)$/i,"").trim()}function B(t,o){let a=O(t.replace(/^\uFEFF/,""));if(a.length===0)throw new Error("Census place CSV contains no data rows.");let r=new Set(Object.keys(a[0]??{}).map(e=>e.trim())),u=["SUMLEV","STATE","NAME","POPESTIMATE2020","POPESTIMATE2021","POPESTIMATE2022","POPESTIMATE2023","POPESTIMATE2024","POPESTIMATE2025"].filter(e=>!r.has(e));if(u.length>0)throw new Error(`Census place CSV is missing required columns: ${u.join(", ")}.`);let d=a.filter(e=>e.SUMLEV?.trim()==="162");if(o){let e=o.padStart(2,"0"),s=d.filter(l=>l.STATE?.trim().padStart(2,"0")!==e);if(s.length>0){let l=[...new Set(s.map(c=>c.STATE?.trim()||"(blank)"))].slice(0,5);throw new Error(`Census place CSV STATE does not match requested FIPS ${e}; received ${l.join(", ")}.`)}}let E=d.map(e=>{let s=e.NAME?.trim()??"";if(!s)return null;let l=V(s),c=b(l);if(!c)return null;let T={2020:g(e.POPESTIMATE2020),2021:g(e.POPESTIMATE2021),2022:g(e.POPESTIMATE2022),2023:g(e.POPESTIMATE2023),2024:g(e.POPESTIMATE2024),2025:g(e.POPESTIMATE2025)};return Object.values(T).every(m=>m===null)?null:{city:l,cityKey:c,censusName:s,estimatesBase2020:g(e.ESTIMATESBASE2020),populations:T}}).filter(e=>e!==null),p=new Map,i=e=>e.populations[2025]??e.populations[2024]??e.populations[2023]??e.populations[2022]??e.populations[2021]??e.populations[2020]??e.estimatesBase2020??-1;for(let e of E){let s=p.get(e.cityKey);(!s||i(e)>i(s)||i(e)===i(s)&&e.censusName.localeCompare(s.censusName)<0)&&p.set(e.cityKey,e)}let _=[...p.values()].sort((e,s)=>e.city.localeCompare(s.city));if(_.length===0)throw new Error("Census place CSV did not contain any valid place population rows.");return{sourceRowCount:a.length,places:_}}function W(t){let o=O(t.replace(/^\uFEFF/,""));if(o.length===0)throw new Error("Location ZIP CSV contains no data rows.");let a=o[0]??{},r=new Set(Object.keys(a).map(s=>s.trim().toLowerCase())),n=["state_abbr","state","state_id"].some(s=>r.has(s)),u=["city","primary_city"].some(s=>r.has(s)),d=["zipcode","zip","zip_code"].some(s=>r.has(s));if(!n||!u||!d)throw new Error("Location ZIP CSV must include state_abbr/state, city, and zipcode/zip columns.");let E=new Map,p=0,i=0,_=new Set,e=new Set;for(let s of o){let l=Object.fromEntries(Object.entries(s).map(([G,j])=>[G.trim().toLowerCase(),j])),c=f(y(l,["state_abbr","state","state_id"])),T=y(l,["city","primary_city"]).trim(),m=q(y(l,["zipcode","zip","zip_code"])),S=y(l,["county","county_name"]).trim();if(!c||!x.has(c)||!T||!m){i+=1;continue}let I=b(T);if(!I){i+=1;continue}let U=`${c}\0${I}`,L=E.get(U);L||(L={city:T,cityKey:I,state:c,zips:new Set,counties:new Set},E.set(U,L)),L.zips.add(m),S&&L.counties.add(S),_.add(c),e.add(m),p+=1}if(E.size===0)throw new Error("Location ZIP CSV did not contain any valid US city/ZIP rows.");return{sourceRowCount:o.length,rowCount:p,rejectedRowCount:i,zipCount:e.size,stateCount:_.size,coveredStates:[..._].sort(),groups:[...E.values()].map(s=>({city:s.city,cityKey:s.cityKey,state:s.state,zips:[...s.zips].sort(),counties:[...s.counties].sort()})).sort((s,l)=>s.state.localeCompare(l.state)||s.city.localeCompare(l.city))}}function J(t){let o=new Set(t.coveredStates),a=k.filter(n=>!o.has(n)),r=[];if(a.length>0&&r.push(`missing required states: ${a.join(", ")}`),t.zipCount<P&&r.push(`only ${t.zipCount} unique ZIPs; at least ${P} are required`),r.length>0)throw new Error(`Location ZIP snapshot is not nationwide and was not activated (${r.join("; ")}; ${t.rejectedRowCount} rejected rows).`)}async function R(){let t=N();v!==t&&(v=t,w=null),w||(w=(async()=>{await t.execute(`
         CREATE TABLE IF NOT EXISTS location_data_imports (
           id           TEXT PRIMARY KEY,
           dataset_kind TEXT NOT NULL,

package/dist/chunk-UXNR3K6I.js

@@ -0,0 +1 @@
+import{g as c,j as d,k as p,l as w,m}from"./chunk-7WPCGRD7.js";import{i as u}from"./chunk-C7MGJUTU.js";import{Inngest as x}from"inngest";var h=new x({id:"mcp-scraper"});function b(r){return Math.min(3600,Math.max(15,15*2**Math.min(8,Math.max(0,r-1))))}async function f(r){let t={claimed:r.rows.length,dispatched:0,failed:0},e=()=>{t.unknown=(t.unknown??0)+1},o=async a=>{let i=await r.read(a).catch(()=>null);return i?.id===a.id?i:null};for(let a of r.rows){let i=a.claimToken;if(!i||!await r.isCurrent(a,i).catch(()=>!1)){e();continue}try{await r.send(a)}catch(n){let l=(n instanceof Error?n.message:String(n)).slice(0,2e3);if(await r.retainRetry(a,i,l,b(a.attempts)).catch(async()=>{let s=await o(a);return s?.status==="failed"&&s.lastError===l}))t.failed+=1;else{let s=await o(a);s?.status==="dispatched"&&s.dispatchedAt?t.dispatched+=1:e()}continue}if(await r.acknowledge(a,i).catch(()=>!1))t.dispatched+=1;else{let n=await o(a);n?.status==="dispatched"&&n.dispatchedAt?t.dispatched+=1:e()}}return t}async function k(r){return f({rows:r,isCurrent:(t,e)=>p(t.id,e),send:async t=>{try{return await h.send({id:t.id,name:"mcp-scraper/extract.requested",data:{jobId:t.id}})}catch(e){throw Error(u(e instanceof Error?e.message:String(e)).slice(0,2e3))}},acknowledge:(t,e)=>w(t.id,e),retainRetry:(t,e,o,a)=>m(t.id,e,o,a),read:async t=>{let e=await c(t.id);return e?{id:e.id,status:e.dispatchedAt?"dispatched":e.dispatchClaimToken?"dispatching":e.dispatchError?"failed":"pending",dispatchedAt:e.dispatchedAt??null,lastError:e.dispatchError??null}:null}})}async function A(r){if((await c(r))?.dispatchedAt)return{claimed:0,dispatched:1,failed:0};let e=await d({jobId:r});return e.length?k(e):{claimed:0,dispatched:0,failed:0,unknown:1}}async function S(r=25){let t=await k(await d({limit:r}));return{checked:t.claimed,dispatched:t.dispatched,failed:t.failed,...t.unknown?{unknown:t.unknown}:{}}}export{h as a,f as b,A as c,S as d};

package/dist/{chunk-VREXSMC6.js → chunk-VVSHZV5S.js}

@@ -1,4 +1,4 @@
-import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-7YOXSMHG.js";var c=166667e-10,N=.0001333336,I=new Set(["serp","fb_search","fb_ad","instagram"]),p=.00111,L=4,R=4e-4;function o(e,r){let t=process.env[e]?.trim();if(!t)return r;let s=Number(t);return Number.isFinite(s)&&s>=0?s:r}var A=o("NANGO_USD_PER_CONNECTION_MONTH",1),U=o("NANGO_USD_PER_FUNCTION_RUN",1e-4),m=o("NANGO_USD_PER_PROXY_REQUEST",1e-4),g=o("NANGO_USD_PER_COMPUTE_SEC",2e-4),O=o("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_USD_PER_GB",8),D=o("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_USD_PER_REQUEST",.002836168);function u(e,r){return Math.max(0,e)/1e3*(r?N:c)}function d(e,r){return e==="fal_wizper"?Math.max(0,r)/L*p:e==="deepinfra_qwen"?Math.max(0,r)/1e3*R:e==="openrouter"||e==="mcp_memory_video"||e==="mcp_memory_ai"?Math.max(0,r):e==="nango_connection"?Math.max(0,r)*A:e==="nango_function_run"?Math.max(0,r)*U:e==="nango_proxy_request"?Math.max(0,r)*m:e==="nango_compute"?Math.max(0,r)*g:e==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api"?Math.max(0,r)/1e9*O:0}import{randomUUID as i}from"crypto";var E=!1,n=null;async function T(){if(!E)return n||(n=S().finally(()=>{n=null}),n)}async function S(){let e=_(),r=await e.execute(`
+import{c as l}from"./chunk-4ZVO6GX4.js";import{a as _}from"./chunk-WNUXMPVX.js";var c=166667e-10,S="owner_confirmed_2026-10-12";var N=.00111,L=4,p=4e-4;function o(e,t){let r=process.env[e]?.trim();if(!r)return t;let s=Number(r);return Number.isFinite(s)&&s>=0?s:t}var R=o("NANGO_USD_PER_CONNECTION_MONTH",1),A=o("NANGO_USD_PER_FUNCTION_RUN",1e-4),U=o("NANGO_USD_PER_PROXY_REQUEST",1e-4),m=o("NANGO_USD_PER_COMPUTE_SEC",2e-4),g=o("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_BROWSER_USD_PER_GB",8),D=o("\u0042\u0052\u0049\u0047\u0048\u0054\u0044\u0041\u0054\u0041_SERP_USD_PER_REQUEST",.002836168);function u(e,t){return Math.max(0,e)/1e3*c}function d(e,t){return e==="fal_wizper"?Math.max(0,t)/L*N:e==="deepinfra_qwen"?Math.max(0,t)/1e3*p:e==="openrouter"||e==="mcp_memory_video"||e==="mcp_memory_ai"?Math.max(0,t):e==="nango_connection"?Math.max(0,t)*R:e==="nango_function_run"?Math.max(0,t)*A:e==="nango_proxy_request"?Math.max(0,t)*U:e==="nango_compute"?Math.max(0,t)*m:e==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser_api"?Math.max(0,t)/1e9*g:0}import{randomUUID as i}from"crypto";var E=!1,n=null;async function T(){if(!E)return n||(n=O().finally(()=>{n=null}),n)}async function O(){let e=_(),t=await e.execute(`
     SELECT
       (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('\u006b\u0065\u0072\u006e\u0065\u006c_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
       AND (SELECT COUNT(*) FROM pragma_table_info('\u006b\u0065\u0072\u006e\u0065\u006c_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
@@ -6,7 +6,7 @@ import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-7YOXSMHG.js";
       AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
       AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
       AS ready
-  `);if(Number(r.rows[0]?.ready??0)===1){E=!0;return}await e.execute(`
+  `);if(Number(t.rows[0]?.ready??0)===1){E=!0;return}await e.execute(`
     CREATE TABLE IF NOT EXISTS \u006b\u0065\u0072\u006e\u0065\u006c_session_log (
       id                    TEXT PRIMARY KEY,
       \u006b\u0065\u0072\u006e\u0065\u006c_session_id     TEXT,
@@ -66,8 +66,8 @@ import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-7YOXSMHG.js";
       margin_usd_headful_starter    REAL NOT NULL DEFAULT 0,
       notes                         TEXT
     )
-  `),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function f(e){try{await T();let r=l(),t=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO \u006b\u0065\u0072\u006e\u0065\u006c_session_log
+  `),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function h(e){try{await T();let t=l(),r=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO \u006b\u0065\u0072\u006e\u0065\u006c_session_log
         (id, \u006b\u0065\u0072\u006e\u0065\u006c_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId??null,r?.op??null,e.source,r?.probeRunId??null,r?.userId??null,a(e.stealth),a(e.headlessSent),a(e.proxyUsed),e.proxySource??null,e.proxyType??null,e.fallback?1:0,new Date(e.openedAtMs).toISOString(),new Date(e.closedAtMs).toISOString(),t,u(t,!1),u(t,!0),e.error??null,r?.subOp??null]})}catch(r){console.warn("[cost-telemetry] record\u004b\u0065\u0072\u006e\u0065\u006cSession failed:",r instanceof Error?r.message:String(r))}}async function v(e){try{await T();let r=l();return(await _().execute({sql:`INSERT OR IGNORE INTO vendor_usage_log
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.\u006b\u0065\u0072\u006e\u0065\u006cSessionId??null,t?.op??null,e.source,t?.probeRunId??null,t?.userId??null,a(e.stealth),a(e.headlessSent),a(e.proxyUsed),e.proxySource??null,e.proxyType??null,e.fallback?1:0,new Date(e.openedAtMs).toISOString(),new Date(e.closedAtMs).toISOString(),r,u(r,!1),u(r,!0),e.error??null,t?.subOp??null]})}catch(t){console.warn("[cost-telemetry] record\u004b\u0065\u0072\u006e\u0065\u006cSession failed:",t instanceof Error?t.message:String(t))}}async function f(e){try{await T();let t=l();return(await _().execute({sql:`INSERT OR IGNORE INTO vendor_usage_log
         (id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.op??r?.op??null,e.probeRunId??r?.probeRunId??null,e.userId??r?.userId??null,e.vendor,e.model??null,e.units,e.unitType,d(e.vendor,e.units),e.error??null,e.method??r?.subOp??null,e.sourceKey??null,e.providerDurationMs??null,a(e.providerCaptcha),e.providerStatus??null]})).rowsAffected===1}catch(r){return console.warn("[cost-telemetry] recordVendorUsage failed:",r instanceof Error?r.message:String(r)),!1}}function a(e){return e==null?null:e?1:0}export{c as a,N as b,I as c,u as d,d as e,T as f,f as g,v as h};
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.op??t?.op??null,e.probeRunId??t?.probeRunId??null,e.userId??t?.userId??null,e.vendor,e.model??null,e.units,e.unitType,d(e.vendor,e.units),e.error??null,e.method??t?.subOp??null,e.sourceKey??null,e.providerDurationMs??null,a(e.providerCaptcha),e.providerStatus??null]})).rowsAffected===1}catch(t){return console.warn("[cost-telemetry] recordVendorUsage failed:",t instanceof Error?t.message:String(t)),!1}}function a(e){return e==null?null:e?1:0}export{c as a,S as b,u as c,d,T as e,h as f,f as g};

package/dist/chunk-WNUXMPVX.js

@@ -0,0 +1 @@
+import{createClient as t}from"@libsql/client/http";var n=process.env.TURSO_DATABASE_URL??"file:./paa-api.db",l=process.env.TURSO_AUTH_TOKEN,e=null;function o(){return e||(e=t({url:n,authToken:l})),e}export{o as a};

package/dist/chunk-WP3QS4CJ.js

@@ -0,0 +1 @@
+var e={schemaVersion:1,ratePolicy:{version:"serp-page-2026-09-24",effectiveDate:"2026-09-24T00:00:00.000Z"},costEvidence:{id:"serp-page-rate-basis-2026-09",status:"owner-approved"},operationCosts:{serp:{credits:20,unit:"per page (ordinary search)",label:"SERP search",notes:"search_serp light mode returns one page of organic positions, URLs, titles, and descriptions by default. Unfiltered one-page full mode returns available same-page SERP features. Either mode requests two pages only when pages:2 is explicit. Unfiltered light searches cost 20 credits per delivered page, or 35 credits per delivered page if the backup supplies the result. One-page full searches cost 35 credits. Setting recency to week or month applies Google's date filter through a managed browser and returns organic results only. It costs 35 credits for one page; filtered pages:2 follows Google's page-two control and costs 70 credits when both pages are delivered. Unfiltered searches use the managed browser only as the last backup. Unfiltered two-page full searches return organic results only for 40 credits, with rich features marked unsupported. If only page 1 is delivered, the search is marked partial and billed for one delivered page. A temporary 35-credit-per-requested-page hold is settled to the delivered rate. Location, language, device, and optional module fields do not change ordinary searches. Saved browser identity and capture_serp_snapshot searches retain their 60-credit base price; optional page snapshots add 1 credit per attempted URL."},paa:{credits:10,unit:"per question (+400 credit base)",label:"PAA harvest",notes:"Returns original-query PAA questions with answers and sources plus organic results and Google entity IDs. Optional same-page SERP modules are flag-controlled. Billed 400 credit base + 10 per question actually returned (unused estimate refunded)."},page_scrape:{credits:1,unit:"per page",label:"Page crawl / extract",notes:"Applies to both single-URL extraction and per-page site crawls."},listing_create:{credits:10,unit:"per new listing",label:"Directory or wiki listing creation",notes:"Fixed total price for a new Local Sourcebook directory listing or Transparent Commons wiki entity. Sourcebook acquisition is included."},listing_edit:{credits:2,unit:"per existing listing revision",label:"Directory or wiki listing edit",notes:"Fixed total price for refreshing an existing Local Sourcebook listing or revising an existing Transparent Commons entity."},url_map:{credits:5,unit:"per mapping operation",label:"Site URL mapping",notes:"Flat rate for the full /map-urls call regardless of URL count discovered."},yt_channel:{credits:2,unit:"per call",label:"YouTube search / channel harvest"},yt_transcription:{credits:200,unit:"base + 2/min of video",label:"YouTube transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_ad:{credits:7,unit:"per call",label:"Facebook search / ad lookup"},maps_search:{credits:5,unit:"per search",label:"Maps business search",notes:"Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses."},maps_place:{credits:60,unit:"per business",label:"Maps business lookup",notes:"Maps place runs reserve the 60-credit base plus the declared review maximum before browsing. Completed runs settle to the base plus unique saved reviews; stopped attempts are refunded."},maps_review:{credits:1,unit:"per review card",label:"Maps review",notes:"One credit per unique saved review on a completed Maps place run. The declared maximum is held before browsing; unused credits are refunded."},fb_search:{credits:8,unit:"per search",label:"Facebook ad library search",notes:"Browser automation to search Facebook Ads Library by keyword."},google_ads_search:{credits:6,unit:"per search",label:"Google Ads Transparency search",notes:"Browser automation to find advertisers in Google Ads Transparency Center by domain or name."},google_ads_intel:{credits:2,unit:"per call",label:"Google Ads Transparency advertiser intel",notes:"Lists and hydrates an advertiser's creatives with image URLs and video references."},google_ads_transcribe:{credits:200,unit:"base + 2/min of video",label:"Google ad video transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_transcribe:{credits:200,unit:"base + 2/min of video",label:"Facebook video / ad transcription",notes:"Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},fb_reels_inventory:{credits:4,unit:"per profile scan",label:"Facebook Reel URL inventory",notes:"Collects canonical Reel URLs from a Facebook profile. Logged-out scans are capped at 60 URLs; saved browser profiles can request larger inventories."},tiktok_transcribe:{credits:200,unit:"base + 2/min of video",label:"TikTok video transcription",notes:"Transcribes a public TikTok video from its playback media. Flat 200-credit base plus 2 credits per minute, reconciled after transcription."},instagram_profile:{credits:4,unit:"per profile scan",label:"Instagram profile content discovery",notes:"Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile."},instagram_media:{credits:4,unit:"per post or reel",label:"Instagram media download",notes:"Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files."},instagram_transcribe:{credits:200,unit:"base + 2/min of video",label:"Instagram media transcription",notes:"Whisper transcription of selected Instagram audio/video media. Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."},browser_minute:{credits:120,unit:"per minute of use",label:"Interactive browser session",notes:"Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window."},reddit_thread:{credits:30,unit:"per thread",label:"Reddit thread base lookup",notes:"Base lookup for the post itself. Comments billed separately per comment at reddit_comment rate. Refunded if the thread cannot be retrieved."},reddit_comment:{credits:2,unit:"per comment",label:"Reddit comment",notes:"Charged per comment actually extracted, billed down automatically if the balance runs out mid-thread."},video_analysis:{credits:6667,unit:"per 120 frames (max 480)",label:"Video breakdown (frame-by-frame + transcript)",notes:"Full video breakdown: samples frames across a video up to 60 minutes, analyzes visuals and audio, then produces a summary, pacing, topic outline, key points, hook, style, and replication guidance. Reserves 6667 Credits per requested 120-frame tier, up to 480 frames. Unused frame tiers and failed-run frame charges are refunded. Incurred analysis-model usage is billed separately, including usage before a failure; terminal status checks settle the original run."},trustpilot_reviews:{credits:5,unit:"per call",label:"Trustpilot review harvest",notes:"Base lookup. Reviews billed separately per card at trustpilot_review rate. Refunded if no reviews are found."},trustpilot_review:{credits:1,unit:"per review card",label:"Trustpilot review card",notes:"Charged per review actually extracted, billed down automatically if the balance runs out mid-page."},g2_reviews:{credits:5,unit:"per call",label:"G2 review harvest",notes:"Base lookup. Reviews billed separately per card at g2_review rate. Refunded if no reviews are found."},g2_review:{credits:2,unit:"per review card",label:"G2 review card",notes:"Charged per review actually extracted (each card carries up to 3 Q&A sections), billed down automatically if the balance runs out mid-page."},diff_page:{credits:1,unit:"per check",label:"Page change check",notes:"Same cost as a single page extract \u2014 one scrape is performed per check, then compared against your last stored snapshot for that URL."}},publicTiers:[{tier:"starter",label:"Starter",monthlyUsd:12,credits:8e4,concurrency:3},{tier:"growth",label:"Growth",monthlyUsd:40,credits:266667,concurrency:10},{tier:"scale",label:"Scale",monthlyUsd:100,credits:8e5,concurrency:20}],credits:{mcPerCredit:100,freeSignupCredits:0},inventory:{totalTools:367,scraperTools:241,memoryTools:126},concurrencyPack:{ratePolicyVersion:"2026-08-14.concurrency-pack-v1",monthlyUsd:5,currency:"usd",interval:"month",billingUnit:"pack",slotsPerPack:2},scaleCreditPack:{ratePolicyVersion:"2026-09-21.scale-credit-pack-v1",eligibleTier:"scale",incrementUsd:10,creditsPerIncrement:8e4,maxQuantity:100,currency:"usd",expiryMonths:3},addons:{memory:{monthlyUsd:19,quotaGb:5,freeQuotaGb:.01},scheduling:{billingMode:"credits",entitlement:"paid_plan_included",separateSubscriptionRequired:!1,chargeEvent:"started_occurrence",runBaseCredits:75,runBaseMc:7500,llmCostMultiplier:1.5,llmCostMultiplierBps:15e3,modes:{connection_sync:{runBaseCredits:75,modelCostBilling:"none",modelCostMultiplier:0},agent:{runBaseCredits:75,modelCostBilling:"actual_reported_cost",modelCostMultiplier:1.5,modelCostMultiplierBps:15e3}}},connectedAccounts:{schemaVersion:2,ratePolicyVersion:"2026-08-03.connected-usage-v3",effectiveAt:"2026-08-03T00:00:00.000Z",entitlement:"paid_plan_add_on",connection:{billingMode:"flat_recurring_credits",amountCredits:15e3,interval:"month",unit:"active_connection",proration:"none"},usage:{billingMode:"credits",settlementMode:"authoritative_receipt",rounding:"aggregate_then_ceil_to_mc",functionRun:{credits:2,mc:200,unit:"per_function_run"},proxyRequest:{credits:2,mc:200,unit:"per_proxy_request"},compute:{creditsPerSecond:5,mcPerSecond:500,sourceUnit:"millisecond",unit:"per_compute_second",formula:"ceil(computeMilliseconds * mcPerSecond / 1000)"}}}}};export{e as a};

package/dist/chunk-ZTMQEDIB.js

@@ -0,0 +1,41 @@
+import{a as ot}from"./chunk-U4WIJZCX.js";import{F as fe,a as Qe,c as Ve}from"./chunk-7WPCGRD7.js";import{g as Ke,h as Ze}from"./chunk-KXJCGV3H.js";import{c as st}from"./chunk-JMZIDHYQ.js";import{c as tt,e as nt,f as rt,g as it}from"./chunk-CNSVZTTT.js";import{c as et}from"./chunk-5BKSKLHX.js";import{h as Ge}from"./chunk-C7MGJUTU.js";import{a as se}from"./chunk-WNUXMPVX.js";import{createWriteStream as ae,mkdtempSync as Yt,mkdirSync as le,readFileSync as Wt,readdirSync as Gt,rmSync as Kt,statSync as Zt,writeFileSync as y}from"fs";import{createHash as W}from"crypto";import{join as l}from"path";import{tmpdir as Qt}from"os";import{createGzip as Vt}from"zlib";import{once as G}from"events";import{finished as ue}from"stream/promises";import{ZipFile as en}from"yazl";import tn from"p-limit";var vt=new Set(["browser_navigation_blocked","browser_navigation_failed","browser_timeout","browser_session_failed","browser_result_missing"]);function Mt(e){return e instanceof Error?e.code:void 0}function At(e){return e instanceof Error?e.message:String(e)}function at(e){let r=e?.match(/^browser_http_(\d+)$/);return r?Number(r[1]):null}function It(e){return e===404?"page_not_found":e===403?"page_forbidden":e===429?"page_rate_limited":e>=500&&e<=599?"page_server_error":"page_http_error"}function Rt(e){let r=At(e),i=Mt(e);if(ot(e,r))return{errorCode:"vendor_unavailable",httpStatus:null,retryable:!0};let a=at(i)??at(r.match(/^(browser_http_\d+)/)?.[1]);return a!=null?{errorCode:It(a),httpStatus:a,retryable:a===429||a>=500}:i==="browser_challenge_unresolved"||/^browser_challenge_unresolved/.test(r)?{errorCode:"bot_check_unresolved",httpStatus:null,retryable:!0}:i==="browser_response_too_large"||/exceeds \d+ byte limit/i.test(r)?{errorCode:"page_too_large",httpStatus:null,retryable:!1}:/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(r)?{errorCode:"browser_session_interrupted",httpStatus:null,retryable:!0}:vt.has(i??"")||/timeout|navigation_failed|ERR_/i.test(r)?{errorCode:"page_unreachable",httpStatus:null,retryable:!0}:{errorCode:"extraction_failed",httpStatus:null,retryable:!1}}function lt(e,r){switch(e){case"page_not_found":return`The page could not be found (HTTP ${r??404}). It may have been moved or deleted.`;case"page_forbidden":return`The site refused the request (HTTP ${r??403}).`;case"page_rate_limited":return"The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";case"page_server_error":return`The page's own server returned an error (HTTP ${r}). This is on the target site's side, not something a retry here can fix.`;case"page_http_error":return`The page returned an unexpected HTTP ${r} response.`;case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. A retry uses a fresh browser session.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"extraction_failed":return"The page could not be extracted. Check retryable before attempting it again; contact support if this persists for the same URL."}}function on(e,r={}){let i=lt(e.errorCode,e.httpStatus);return{...Ge({errorCode:e.errorCode,retryable:e.retryable,chargeStatus:r.chargeStatus,details:{...r.details??{},...e.httpStatus!=null?{http_status:e.httpStatus}:{}}}),message:i,error:i}}function an(e){return e.httpStatus===404?404:e.httpStatus===403?403:e.httpStatus===429?429:e.httpStatus!=null&&e.httpStatus>=500&&e.httpStatus<=599?500:e.httpStatus!=null?502:e.errorCode==="vendor_unavailable"?503:502}var Dt="wayback_capture_missing";function Me(e,r){if(e==null)return{failureCode:null,failureReason:null};if(e===Dt)return{failureCode:e,failureReason:r??null};let i=new Error(r??e);i.code=e;let a=Rt(i);return{failureCode:a.errorCode,failureReason:lt(a.errorCode,a.httpStatus)}}import{createHash as Ie}from"crypto";import{neon as $t}from"@neondatabase/serverless";import{waitUntil as Ut}from"@vercel/functions";var jt="jina",ye=null,ut=!1;function be(){return(process.env.JINA_EMBED_MODEL??"jina-embeddings-v5-omni-small").trim()}function _e(){return Number((process.env.JINA_EMBED_DIM??"1024").trim())}function Ae(){return!!(process.env.JINA_API_KEY?.trim()&&process.env.MEMORY_DATABASE_URL?.trim())}function oe(){if(ye)return ye;let e=process.env.MEMORY_DATABASE_URL?.trim();if(!e)throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");return ye=$t(e),ye}async function mt(){if(ut)return;let e=_e();await oe().query("CREATE EXTENSION IF NOT EXISTS vector"),await oe().query(`
+    CREATE TABLE IF NOT EXISTS commons_index_vectors (
+      document_id   TEXT PRIMARY KEY,
+      entity_id     TEXT NOT NULL,
+      document_type TEXT NOT NULL,
+      title         TEXT NOT NULL,
+      embedding     vector(${e}) NOT NULL,
+      model         TEXT NOT NULL,
+      updated_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+    )
+  `),await oe().query("CREATE INDEX IF NOT EXISTS commons_index_vectors_entity ON commons_index_vectors(entity_id)"),ut=!0}async function we(e){let r=process.env.JINA_API_KEY?.trim();if(!r)throw new Error("JINA_API_KEY is not set; Commons semantic search cannot embed.");if(!e.length)return[];let i=await fetch("https://api.jina.ai/v1/embeddings",{method:"POST",headers:{authorization:`Bearer ${r}`,"content-type":"application/json"},body:JSON.stringify({model:be(),dimensions:_e(),input:e.map(p=>({text:p.slice(0,8e3)}))}),signal:AbortSignal.timeout(6e4)});if(!i.ok)throw new Error(`Jina embedding request failed with HTTP ${i.status}: ${(await i.text()).slice(0,200)}`);let s=((await i.json()).data??[]).map(p=>p.embedding);if(s.length!==e.length)throw new Error(`Jina returned ${s.length} embeddings for ${e.length} inputs.`);return s}var Nt=60*6e4;async function Lt(e=50){if(!Ae())return{claimed:0,embedded:0,failed:0,remaining:0};await mt();let r=Math.max(1,Math.min(200,Math.floor(e))),a=(await se().execute({sql:`SELECT id, entity_id, document_type, title, text FROM commons_index_documents
+          WHERE embedding_status = 'queued' OR (embedding_status = 'failed' AND updated_at < ?)
+          ORDER BY updated_at ASC LIMIT ?`,args:[new Date(Date.now()-Nt).toISOString(),r]})).rows;if(!a.length)return{claimed:0,embedded:0,failed:0,remaining:await ct()};let s=0,p=0,M=be();try{let h=await we(a.map(S=>`${S.title}
+
+${S.text}`));for(let[S,_]of a.entries()){let C=`[${h[S].join(",")}]`;await oe().query(`INSERT INTO commons_index_vectors (document_id, entity_id, document_type, title, embedding, model, updated_at)
+         VALUES ($1, $2, $3, $4, $5::vector, $6, now())
+         ON CONFLICT (document_id) DO UPDATE SET entity_id = EXCLUDED.entity_id, document_type = EXCLUDED.document_type,
+           title = EXCLUDED.title, embedding = EXCLUDED.embedding, model = EXCLUDED.model, updated_at = now()`,[_.id,_.entity_id,_.document_type,_.title,C,M]),await se().execute({sql:`UPDATE commons_index_documents SET embedding_status = 'indexed', embedding_provider = ?, embedding_model = ?,
+              vector_ref = ?, indexed_at = ?, error = NULL WHERE id = ?`,args:[jt,M,_.id,new Date().toISOString(),_.id]}),s+=1}}catch(h){p=a.length-s;let S=(h instanceof Error?h.message:String(h)).slice(0,500);for(let _ of a.slice(s))await se().execute({sql:"UPDATE commons_index_documents SET embedding_status = 'failed', error = ?, updated_at = ? WHERE id = ?",args:[S,new Date().toISOString(),_.id]}).catch(()=>{})}return{claimed:a.length,embedded:s,failed:p,remaining:await ct()}}function dn(){Ae()&&Ut(Lt(10).catch(e=>{console.error("[commons/embeddings] immediate embed failed; the scheduled drain will retry",e instanceof Error?e.message:String(e))}))}async function ct(){let e=await se().execute("SELECT COUNT(*) AS n FROM commons_index_documents WHERE embedding_status IN ('queued', 'failed')");return Number(e.rows[0]?.n??0)}async function pn(e,r=40){let i=new Map;if(!Ae()||!e.trim())return i;await mt();let[a]=await we([e]);if(!a)return i;let s=await oe().query(`SELECT entity_id, MAX(1 - (embedding <=> $1::vector)) AS score
+     FROM commons_index_vectors GROUP BY entity_id ORDER BY score DESC LIMIT $2`,[`[${a.join(",")}]`,Math.max(1,Math.min(100,r))]);for(let p of s)i.set(String(p.entity_id),Number(p.score));return i}var Ot=500,Bt=.9,qt=1e4,Ft=5e4;function Se(e,r){let i=10**r;return Math.round(e*i)/i}function Jt(e,r){if(e.length===0||e.length!==r.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let i=0,a=0,s=0;for(let p=0;p<e.length;p++)i+=e[p]*r[p],a+=e[p]*e[p],s+=r[p]*r[p];return a===0||s===0?0:Math.max(-1,Math.min(1,i/(Math.sqrt(a)*Math.sqrt(s))))}function Ht(e){let r=Ie("sha256");for(let i of[...e].sort((a,s)=>a.url.localeCompare(s.url)))r.update(i.url),r.update("\0"),r.update(i.contentHash),r.update("\0");return r.digest("hex")}function dt(e){let r=e.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return r.length>=12?r:null}function zt(e){let r=e.map(h=>h.bodyMarkdown.split(/\n{2,}/).map(S=>S.trim()).filter(Boolean)),i=e.length>=4?Math.max(3,Math.ceil(e.length*.5)):null,a=new Map;for(let h of r){let S=new Set(h.map(dt).filter(_=>!!_));for(let _ of S)a.set(_,(a.get(_)??0)+1)}let s=new Set;if(i!=null)for(let[h,S]of a)S>=i&&s.add(h);let p=0;return{texts:e.map((h,S)=>{let _=[];for(let P of r[S]){let I=dt(P);if(I&&s.has(I)){p+=P.length;continue}_.push(P)}let C=_.join(`
+
+`).trim(),v=C.length>=60?C:h.bodyMarkdown;return`${h.title?.trim()||"Untitled page"}
+
+${v}`}),minimumPageCount:i,removedBlockSignatures:s.size,removedCharacters:p}}var Re=class{parent;constructor(r){this.parent=Array.from({length:r},(i,a)=>a)}find(r){let i=this.parent[r];return i!==r&&(this.parent[r]=this.find(i)),this.parent[r]}union(r,i){let a=this.find(r),s=this.find(i);a!==s&&(this.parent[s]=a)}};async function pt(e,r={}){let i=e.filter(o=>o.bodyMarkdown.trim().length>0).slice(0,Ot),a=Number.isFinite(r.threshold)?r.threshold:Bt,s=Number.isFinite(r.maxPairs)?r.maxPairs:qt,p=Math.max(0,Math.min(1,a)),M=Math.max(1,Math.min(Ft,Math.floor(s))),h=r.model??be(),S=r.dimensions??_e(),_=r.embedTexts??we,C=zt(i),v=[],P=new Map,I=[];for(let[o,f]of i.entries()){let w=f.contentHash||Ie("sha256").update(f.bodyMarkdown).digest("hex"),k=P.get(w);k==null&&(k=v.length,P.set(w,k),v.push(C.texts[o])),I.push(k)}let D=[];for(let o=0;o<v.length;o+=64)D.push(...await _(v.slice(o,o+64)));if(D.length!==v.length)throw new Error(`Embedding provider returned ${D.length} vectors for ${v.length} unique page bodies.`);let K=I.map(o=>D[o]),ce=new Re(i.length),L=[],O=[];for(let o=0;o<i.length;o++)for(let f=o+1;f<i.length;f++){let w=Jt(K[o],K[f]);L.push(w),!(w<p)&&(O.push({source:o,target:f,score:w}),ce.union(o,f))}O.sort((o,f)=>f.score-o.score||i[o.source].url.localeCompare(i[f.source].url)||i[o.target].url.localeCompare(i[f.target].url));let R=[...L].sort((o,f)=>o-f),U=o=>R.length?Se(R[Math.floor((R.length-1)*o)],6):null,j=new Map;for(let o=0;o<i.length;o++){let f=ce.find(o),w=j.get(f)??[];w.push(o),j.set(f,w)}let B=new Map,J=[...j.values()].filter(o=>o.length>1).sort((o,f)=>f.length-o.length||i[o[0]].url.localeCompare(i[f[0]].url)).map((o,f)=>{let w=`cluster-${String(f+1).padStart(3,"0")}`;for(let k of o)B.set(k,w);return{clusterId:w,pageCount:o.length,urls:o.map(k=>i[k].url).sort()}}),H=O.slice(0,M).map((o,f)=>{let w=i[o.source],k=i[o.target],q=Se(o.score,6);return{sourceUrl:w.url,sourceTitle:w.title,targetUrl:k.url,targetTitle:k.title,similarity:q,similarityPercent:Se(q*100,2),corpusPercentile:R.length<=1?100:Se((1-f/(R.length-1))*100,2),exactContentDuplicate:!!(w.contentHash&&w.contentHash===k.contentHash),sourceWordCount:w.wordCount,targetWordCount:k.wordCount,clusterId:B.get(o.source)??null}}),z=Ht(i),me=Ie("sha256").update(JSON.stringify({corpusSha256:z,model:h,dimensions:S,threshold:p,maxPairs:M,boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:C.minimumPageCount,removedBlockSignatures:C.removedBlockSignatures}})).digest("hex");return{generatedAt:new Date().toISOString(),provider:"jina",model:h,dimensions:S,threshold:p,requestedMaxPairs:M,comparedPages:i.length,possiblePairs:i.length*Math.max(0,i.length-1)/2,qualifyingPairs:O.length,returnedPairs:H.length,pairsTruncated:O.length>H.length,corpusSha256:z,analysisSha256:me,scoreDistribution:{min:U(0),p25:U(.25),median:U(.5),p75:U(.75),p90:U(.9),max:U(1)},boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:C.minimumPageCount,removedBlockSignatures:C.removedBlockSignatures,removedCharacters:C.removedCharacters},rows:H,clusters:J}}import{basename as Xt}from"path";function Y(e){return e.split("#")[0].replace(/\/$/,"")}function De(e){let r=e.replace(/^www\./,"").toLowerCase(),i=r.split(".");return i.length<=2?r:i.slice(-2).join(".")}function gt(e,r){try{let i=new URL(e);return Xt(i.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`image-${r}`}catch{return`image-${r}`}}function ht(){let e=new Map;return r=>{let i=r.replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)||"page",a=e.get(i)??0;return e.set(i,a+1),a?`${i}-${a}`:i}}function ft(e){let r=e==null?"":String(e);return/[",\r\n]/.test(r)?`"${r.replace(/"/g,'""')}"`:r}function yt(e){return{source_url:e.sourceUrl,source_title:e.sourceTitle,target_url:e.targetUrl,target_title:e.targetTitle,similarity:e.similarity,similarity_percent:e.similarityPercent,corpus_percentile:e.corpusPercentile,exact_content_duplicate:e.exactContentDuplicate,source_word_count:e.sourceWordCount,target_word_count:e.targetWordCount,cluster_id:e.clusterId}}var $e=20,Ue=500,nn=8,bt=10*1024*1024,_t=100*1024*1024,je=25e4;async function Fn(e,r={}){let i=e.options.downloadImages===!0;if(e.userId==null)throw new Error("site extract artifact owner is missing");let a=Qe(String(e.userId)),s=Yt(l(Qt(),"extract-bundle-"));le(s,{recursive:!0}),i&&le(l(s,"images"),{recursive:!0});let p=e.options.captureRenderedDom===!0,M=e.options.semanticSimilarity===!0;p&&le(l(s,"rendered-dom"),{recursive:!0});try{let h=[],S=[],_=[],C=ht(),v=[],P=e.options.waybackTimeline,I=new Map;for await(let n of fe(e.id))for(let t of n){I.set(Y(t.url),t.status),h.push({url:t.url,status:t.status,title:t.title,titleLength:t.titleLength,titlePixels:t.titlePixels,metaDescription:t.metaDescription,metaDescLength:t.metaDescLength,h1:t.h1,h1_2:t.h1_2,h2Count:t.h2Count,extractionStatus:t.extractionStatus,indexable:t.indexable,indexabilityReason:t.indexabilityReason,canonicalUrl:t.canonicalUrl?t.url:null,wordCount:t.wordCount,contentHash:t.contentHash,imagesMissingAlt:t.imagesMissingAlt,schemaTypes:t.schemaTypes.length>0?["present"]:[],outlinks:[]});let u=t.pageId??W("sha256").update(t.archivedUrl??t.url).digest("hex"),E=l(s,"pages",u);le(E,{recursive:!0});let d=t.contentRef?await a(t.contentRef):null,g=d?.html??null,c=d?.markdown??t.bodyMarkdown??null;if(M&&t.extractionStatus==="successful"&&c?.trim()&&S.push({url:t.url,title:t.title,bodyMarkdown:c,contentHash:t.contentHash,wordCount:t.wordCount}),p&&g!=null){let We=`rendered-dom/${C(t.url)}.html.txt`;y(l(s,We),g),_.push({url:t.url,path:We,bytes:Buffer.byteLength(g),truncated:t.renderedDomTruncated===!0,sanitized:t.renderedDomSanitized===!0})}let m=g==null?null:`pages/${u}/page.html`,b=c==null?null:`pages/${u}/page.md`,x=c==null||!t.archiveRequestedMonth?null:`pages/${t.archiveRequestedMonth}/${u}.md`;g!=null&&m&&y(l(s,m),g),c!=null&&b&&y(l(s,b),c),c!=null&&x&&(le(l(s,"pages",t.archiveRequestedMonth),{recursive:!0}),y(l(s,x),c));let{bodyMarkdown:A,contentRef:ne,discoveryLinks:re,...Ct}=t,ie={version:"site-export-page.v1",...Ct,pageId:u,content:{htmlPath:m,markdownPath:b,htmlBytes:g==null?0:Buffer.byteLength(g),markdownBytes:c==null?0:Buffer.byteLength(c),htmlSha256:g==null?null:W("sha256").update(g).digest("hex"),markdownSha256:c==null?null:W("sha256").update(c).digest("hex")}},Xe=JSON.stringify(ie,null,2),Ye=`pages/${u}/page.json`;y(l(s,Ye),Xe),v.push({pageId:u,url:t.archivedUrl??t.url,jsonPath:Ye,htmlPath:m,markdownPath:b,legacyMarkdownPath:x,jsonSha256:W("sha256").update(Xe).digest("hex"),htmlSha256:ie.content.htmlSha256,markdownSha256:ie.content.markdownSha256,htmlBytes:ie.content.htmlBytes,markdownBytes:ie.content.markdownBytes})}let D=Vt(),K=ae(l(s,"links.jsonl.gz"));D.pipe(K);let ce=async n=>{D.write(n)||await G(D,"drain")},L=ae(l(s,"schema.jsonl")),O=async n=>{L.write(n)||await G(L,"drain")},R=ae(l(s,"failures.jsonl")),U=async n=>{R.write(n)||await G(R,"drain")},j=new Map,B=new Map,J=new Map,H=new Map,z=new Map,me=new Set,o=0,f=0,w=0,k=0,q=0,xe=!1,Ee="";try{Ee=De(new URL(e.startUrl).hostname)}catch{Ee=""}let wt=tn(nn),Ne=[],Le=[],Oe=[],Z=0,de=0,ke=0,St=n=>ke+n>_t?!1:(ke+=n,!0);for await(let n of fe(e.id))for(let t of n){if(t.schema?.length&&await O(JSON.stringify({url:t.url,schema:t.schema})+`
+`),t.extractionStatus==="failed"){let{failureCode:d,failureReason:g}=Me(t.failureCode,t.failureReason);await U(JSON.stringify({url:t.url,status:t.status,via:t.via,failureCode:d,failureReason:g,fetchAttempts:t.fetchAttempts??[]})+`
+`)}if(t.imageLinks?.length)for(let[d,g]of t.imageLinks.entries()){let c=W("sha256").update(`${t.url}\0${g}`).digest("hex"),m={imageId:c,sourcePage:t.url,sourceUrl:g,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Le.push(m),!i)continue;if(d>=$e){m.status="skipped_page_limit",m.failureCode="page_image_limit",m.failureReason=`Only the first ${$e} image candidates are downloaded per page.`;continue}if(Z>=Ue){m.status="skipped_site_limit",m.failureCode="site_image_limit",m.failureReason=`Only the first ${Ue} image candidates are downloaded per site.`;continue}Z++;let b=`${c}-${gt(g,d)}`,x=l(s,"images");Ne.push(wt(()=>st(g,x,b,{expectedType:"image",maxBytes:bt,consumeBytes:St})).then(async A=>{let ne=Wt(A.savedPath),re=await Ze({ownerId:String(e.userId),jobId:e.id,imageId:c,filename:b,contentType:A.mimeType??"application/octet-stream",content:ne,sourceUrl:g,sourcePage:t.url});Oe.push(re),m.status="downloaded",m.path=`images/${b}`,m.artifactId=re.key,m.mimeType=A.mimeType,m.bytes=A.sizeBytes,m.sha256=W("sha256").update(ne).digest("hex")},A=>{de++,m.status="failed",m.failureCode="image_download_failed",m.failureReason=(A instanceof Error?A.message:String(A)).replace(/\s+/g," ").slice(0,500)}))}let u=Y(t.url),E={int:0,ext:0};for(let d of t.outlinks??[]){let g=(d.rel??"").split(/\s+/).includes("nofollow"),c=Y(d.href),m=I.get(c)??null;if(await ce(JSON.stringify({from:t.url,to:d.href,anchor:d.anchor,rel:d.rel,internal:d.internal,nofollow:g,targetStatus:m})+`
+`),k++,q>=je){xe=!0;continue}if(q++,d.internal){if(E.int++,o++,j.has(c)||j.set(c,new Set),j.get(c).add(u),d.anchor){B.has(c)||B.set(c,new Map);let b=B.get(c);b.set(d.anchor,(b.get(d.anchor)??0)+1)}!g&&(I.get(c)===200||!I.has(c))&&(J.has(u)||J.set(u,new Set),J.get(u).add(c)),m!=null&&m>=400&&(w++,me.add(t.url))}else{E.ext++;let b="";try{b=De(new URL(d.href).hostname)}catch{continue}if(!b||b===Ee)continue;f++;let x=z.get(b)??{links:0,nofollow:0,pages:new Set};x.links++,g&&x.nofollow++,x.pages.add(t.url),z.set(b,x)}}H.set(t.url,E)}let xt=Promise.all([ue(K),ue(L),ue(R)]);D.end(),L.end(),R.end(),await xt,await Promise.all(Ne);let X=new Map,Pe=Y(e.startUrl);X.set(Pe,0);let Te=[Pe];for(;Te.length>0;){let n=Te.shift(),t=X.get(n);for(let u of J.get(n)??[])X.has(u)||(X.set(u,t+1),Te.push(u))}let Q=new Map;for(let n of h){let t=Y(n.url),u=j.get(t)??new Set,E=B.get(t),d=E?[...E.entries()].sort((c,m)=>m[1]-c[1]).slice(0,3).map(c=>c[0]):[],g=H.get(n.url)??{int:0,ext:0};Q.set(n.url,{url:n.url,inlinks:u.size,uniqueInlinks:u.size,outlinksInternal:g.int,outlinksExternal:g.ext,crawlDepth:X.has(t)?X.get(t):null,orphan:u.size===0&&t!==Pe,topAnchors:d})}let Be=rt(h,Q,me),V=it(e.startUrl,h,Be,Q),ee=Ve(e),F={jobId:e.id,startUrl:e.startUrl,status:e.successfulUrls===0?"failed":e.failedUrls>0||e.remainingUrls>0||ee.creditTruncated?"partial":"complete",...ee,discovered:e.totalUrls,attempted:e.attemptedUrls,successful:e.successfulUrls,failed:e.failedUrls,remaining:e.remainingUrls,linkAnalysis:{truncated:xe,analyzedEdges:q,streamedEdges:k,edgeCap:je},generatedAt:new Date().toISOString()};V=["## Crawl delivery",`- Discovered: ${F.discovered}`,`- Attempted: ${F.attempted}`,`- Successful: ${F.successful}`,`- Failed: ${F.failed}`,`- Remaining: ${F.remaining}`,`- Requested page cap: ${ee.requestedMaxPages}`,`- Funded page cap: ${ee.effectiveMaxPages}`,`- Credit-truncated: ${ee.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${xe?` (capped at ${je})`:""}`,"",V].join(`
+`),r.imageAudit&&(V+=`
+
+${tt(r.imageAudit)}`);let pe=[...Q.values()],qe=n=>Math.round(n*10)/10,te={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},Fe=0,Je=0;for(let n of pe)Fe+=n.inlinks,Je+=n.outlinksInternal+n.outlinksExternal,n.inlinks===0?te.zero++:n.inlinks<=2?te.oneToTwo++:n.inlinks<=10?te.threeToTen++:te.elevenPlus++;let Ce=[...z.entries()].map(([n,t])=>({domain:n,links:t.links,nofollow:t.nofollow,pages:t.pages.size})).sort((n,t)=>t.links-n.links),ve={summary:{internal:{totalLinks:o,pages:h.length,orphans:pe.filter(n=>n.orphan).length,brokenInternal:w,avgInlinks:h.length?qe(Fe/h.length):0,avgOutlinks:h.length?qe(Je/h.length):0,distribution:te,topByInlinks:[...pe].sort((n,t)=>t.inlinks-n.inlinks).slice(0,20).map(n=>({url:n.url,inlinks:n.inlinks,outlinksInternal:n.outlinksInternal,outlinksExternal:n.outlinksExternal}))},external:{totalLinks:f,uniqueDomains:Ce.length,topDomains:Ce.slice(0,20)}},externalDomains:Ce},ge=ae(l(s,"pages.jsonl")),Et=new Map(v.map(n=>[n.pageId,n])),$=P?ae(l(s,"capture-matrix.jsonl")):null,He=new Set,ze=(n,t)=>{let u=t;try{u=new URL(t).href}catch{}return`${n??""}\0${u}`};for await(let n of fe(e.id))for(let t of n){let u=t.extractionStatus==="failed"?Me(t.failureCode,t.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:E,schema:d,outlinks:g,contentRef:c,...m}=t,b=Q.get(t.url),x=t.pageId?Et.get(t.pageId):void 0,A={...m,...t.extractionStatus==="failed"?u:{},representations:x?{jsonPath:x.jsonPath,htmlPath:x.htmlPath,markdownPath:x.markdownPath,jsonSha256:x.jsonSha256,htmlSha256:x.htmlSha256,markdownSha256:x.markdownSha256,htmlBytes:x.htmlBytes,markdownBytes:x.markdownBytes}:null,inlinks:b?.inlinks??0,crawlDepth:b?.crawlDepth??null,orphan:b?.orphan??!1};if(ge.write(JSON.stringify(A)+`
+`)||await G(ge,"drain"),$){let ne=ze(t.archiveRequestedMonth,t.originalUrl??t.url);He.add(ne);let re={requestedMonth:t.archiveRequestedMonth??null,captureTimestamp:t.archiveTimestamp??null,originalUrl:t.originalUrl??t.url,archivedUrl:t.archivedUrl??null,status:t.extractionStatus==="successful"?"captured":"failed",httpStatus:t.status,contentHash:t.contentHash||null,wordCount:t.wordCount,failureCode:u.failureCode,failureReason:u.failureReason};$.write(JSON.stringify(re)+`
+`)||await G($,"drain")}}if($&&P?.timeline?.urls){let n=P.timeline,t=n.urls,u=(()=>{if(n.months?.length)return[...new Set(n.months)].sort();if(!n.from||!n.to)return[];let E=[],d=new Date(`${n.from}-01T00:00:00Z`),g=new Date(`${n.to}-01T00:00:00Z`);for(;d<=g&&E.length<60;)E.push(`${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,"0")}`),d.setUTCMonth(d.getUTCMonth()+(n.intervalMonths??1));return E})();for(let E of u)for(let d of t)He.has(ze(E,d))||$.write(JSON.stringify({requestedMonth:E,captureTimestamp:null,originalUrl:d,archivedUrl:null,status:"missing",httpStatus:null,contentHash:null,wordCount:0,failureCode:"wayback_capture_missing",failureReason:"No successful HTML capture was available in the requested month."})+`
+`)||await G($,"drain")}let kt=ue(ge),Pt=$?ue($):Promise.resolve();if(ge.end(),$?.end(),await Promise.all([kt,Pt]),p&&y(l(s,"rendered-dom","manifest.jsonl"),_.map(n=>JSON.stringify(n)).join(`
+`)),M){let n=await pt(S,{threshold:Number(e.options.similarityThreshold??void 0),maxPairs:Number(e.options.similarityMaxPairs??void 0),embedTexts:r.embedSimilarityTexts}),t=n.rows.map(yt),u=t.length>0?Object.keys(t[0]):["source_url","source_title","target_url","target_title","similarity","similarity_percent","corpus_percentile","exact_content_duplicate","source_word_count","target_word_count","cluster_id"],E=[u.join(","),...t.map(m=>u.map(b=>ft(m[b]??null)).join(","))].join(`
+`)+`
+`,{rows:d,clusters:g,...c}=n;y(l(s,"similarity-table.csv"),E),y(l(s,"similarity.jsonl"),t.map(m=>JSON.stringify(m)).join(`
+`)),y(l(s,"similarity-table-schema.json"),JSON.stringify({tableNameSuggestion:`site_similarity_${new URL(e.startUrl).hostname.replace(/[^a-z0-9]+/gi,"_").replace(/^_+|_+$/g,"").toLowerCase()}`,defaultSort:[{column:"similarity",direction:"desc"}],columns:{source_url:"text",source_title:"text",target_url:"text",target_title:"text",similarity:"number",similarity_percent:"number",corpus_percentile:"number",exact_content_duplicate:"boolean",source_word_count:"integer",target_word_count:"integer",cluster_id:"text"}},null,2)),y(l(s,"similarity-summary.json"),JSON.stringify(c,null,2)),y(l(s,"content-clusters.json"),JSON.stringify(n.clusters,null,2)),V+=["","## Rendered content similarity",`- Compared pages: ${n.comparedPages}`,`- Raw cosine threshold: ${n.threshold}`,`- Corpus score distribution: min ${n.scoreDistribution.min??"n/a"} / median ${n.scoreDistribution.median??"n/a"} / p90 ${n.scoreDistribution.p90??"n/a"} / max ${n.scoreDistribution.max??"n/a"}`,`- Qualifying pairs: ${n.qualifyingPairs}`,`- Returned table rows: ${n.returnedPairs}${n.pairsTruncated?" (capped)":""}`,`- Multi-page clusters: ${n.clusters.length}`,`- Embedding model: ${n.model} (${n.dimensions} dimensions)`,`- Corpus boilerplate removed: ${n.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${n.boilerplateRemoval.removedCharacters} characters`,`- Corpus SHA-256: ${n.corpusSha256}`].join(`
+`)}y(l(s,"report.md"),V),y(l(s,"crawl-summary.json"),JSON.stringify(F,null,2)),y(l(s,"images-manifest.jsonl"),Le.map(n=>JSON.stringify(n)).join(`
+`)),y(l(s,"manifest.json"),JSON.stringify({version:"site-export.v1",jobId:e.id,startUrl:e.startUrl,generatedAt:F.generatedAt,outputProfile:{json:!0,html:!0,markdown:!0,imageManifest:!0,imageBytes:i},pages:[...v].sort((n,t)=>n.pageId.localeCompare(t.pageId)),imagesManifestPath:"images-manifest.jsonl",download:"This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."},null,2)),P&&y(l(s,"wayback-manifest.json"),JSON.stringify({rootUrl:P.rootUrl??e.startUrl,requested:P.timeline??null,maxPagesPerSnapshot:P.maxPagesPerSnapshot??null,captureMatrix:"capture-matrix.jsonl"},null,2)),y(l(s,"issues.json"),JSON.stringify(Be,null,2)),y(l(s,"link-metrics.jsonl"),pe.map(n=>JSON.stringify(n)).join(`
+`)),y(l(s,"link-report.md"),nt(ve)),y(l(s,"links-summary.json"),JSON.stringify(ve.summary,null,2)),y(l(s,"external-domains.json"),JSON.stringify(ve.externalDomains,null,2)),r.imageAudit&&(y(l(s,"images.jsonl"),r.imageAudit.rows.map(n=>JSON.stringify(n)).join(`
+`)),y(l(s,"images-summary.json"),JSON.stringify(r.imageAudit.summary,null,2))),r.seoAudit&&y(l(s,"seo-audit.json"),JSON.stringify(r.seoAudit,null,2)),r.branding&&y(l(s,"branding.json"),JSON.stringify(r.branding,null,2)),i&&y(l(s,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-de,failed:de,downloadedBytes:ke,perPageCap:$e,perSiteCap:Ue,perFileByteCap:bt,totalByteCap:_t},null,2));let T=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(p){T.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let n of _)T.push({name:n.path,type:"text/plain"})}M&&(T.push({name:"similarity-table.csv",type:"text/csv"}),T.push({name:"similarity.jsonl",type:"application/x-ndjson"}),T.push({name:"similarity-table-schema.json",type:"application/json"}),T.push({name:"similarity-summary.json",type:"application/json"}),T.push({name:"content-clusters.json",type:"application/json"})),P&&(T.push({name:"wayback-manifest.json",type:"application/json"}),T.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),r.imageAudit&&(T.push({name:"images.jsonl",type:"application/x-ndjson"}),T.push({name:"images-summary.json",type:"application/json"})),r.seoAudit&&T.push({name:"seo-audit.json",type:"application/json"}),r.branding&&T.push({name:"branding.json",type:"application/json"}),i&&T.push({name:"images-download-summary.json",type:"application/json"});let N=new en,Tt=Ke({ownerId:String(e.userId),jobId:e.id,createdAt:new Date,content:N.outputStream});for(let n of T)N.addFile(l(s,n.name),n.name);let he=[...v].sort((n,t)=>n.pageId.localeCompare(t.pageId));for(let n of he)n.markdownPath&&N.addFile(l(s,n.markdownPath),n.markdownPath);for(let n of he)n.legacyMarkdownPath&&N.addFile(l(s,n.legacyMarkdownPath),n.legacyMarkdownPath);for(let n of he)n.htmlPath&&N.addFile(l(s,n.htmlPath),n.htmlPath);for(let n of he)N.addFile(l(s,n.jsonPath),n.jsonPath);if(i&&Z>de){let n=l(s,"images");for(let t of Gt(n,{recursive:!0})){let u=l(n,t);Zt(u).isFile()&&N.addFile(u,`images/${t.split("\\").join("/")}`)}}return N.end(),[await et("export_packaging",()=>Tt),...Oe]}finally{Kt(s,{recursive:!0,force:!0})}}export{Rt as a,on as b,an as c,Me as d,Ae as e,Lt as f,dn as g,ct as h,pn as i,je as j,Fn as k};

package/dist/contracts-GUXGFCVV.js

@@ -0,0 +1 @@
+import{a,b,c,d,e,f}from"./chunk-HNNYXMSI.js";export{b as OperationIdempotencyConflictError,a as OperationInputError,d as operationFlagsJson,c as operationIdentityHash,e as requiredOperationString,f as validateOperationStoreGate};