mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-X54CQLK2.js

@@ -0,0 +1 @@
+import{g as _,i as c}from"./chunk-GMWKPIYX.js";var g={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},a=/error|message/i,w=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),E=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,k=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),S=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),f=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function b(e){return e.replace(f,"[browser-service]")}var R=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),m=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),h=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),I=new Set(["provider_session_api","client_runtime","extractor","unknown"]),y=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),T=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),P=512,A=320;function t(e,r){return typeof e=="string"&&r.has(e)?e:null}function O(e){return typeof e!="string"?null:y.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function D(e){return typeof e!="string"||e.length<1||e.length>P||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function p(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function l(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function u(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let o=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return b(o).slice(0,A)}function v(e){return typeof e=="boolean"?e:null}function N(e){return{browser_provider:t(e.browser_provider,R),provider_session_id:D(e.provider_session_id),provider_session_observed_at:l(e.provider_session_observed_at),provider_lookup_status:t(e.provider_lookup_status,m),provider_status:t(e.provider_status,h),provider_error_code:O(e.provider_error_code),provider_error_message:u(e.provider_error_message),provider_reason_source:t(e.provider_reason_source,I),provider_duration_ms:p(e.provider_duration_ms),provider_bandwidth_bytes:p(e.provider_bandwidth_bytes),provider_captcha:v(e.provider_captcha),provider_checked_at:l(e.provider_checked_at),provider_disconnect_observed:v(e.provider_disconnect_observed),provider_disconnect_message:u(e.provider_disconnect_message)}}function i(e,r=""){if(typeof e=="string"){let o=b(e);return a.test(r)&&(w.test(e)||E.test(e))?o=_("service_unavailable"):a.test(r)&&(o=c(o)),o}if(Array.isArray(e))return e.map(o=>i(o,r));if(e!==null&&typeof e=="object"){let o={};for(let[n,d]of Object.entries(e)){if(k.has(n))continue;let s=g[n]??n;if(S.has(n)){o[s]=null;continue}o[s]=i(d,n)}return o}return e}function L(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return i(r);let o=r,n=i(o);return Object.keys(o).some(s=>T.has(s))?{...n,...N(o)}:n})}function C(e){let r=e?.diagnostics;return r?.debug?{...e,diagnostics:{...r,debug:i(r.debug)}}:e}export{i as a,L as b,C as c};

package/dist/chunk-XLWNEVUZ.js

@@ -0,0 +1,27 @@
+import{c as be}from"./chunk-KJQXUZ4Y.js";import{b as T,l as ve}from"./chunk-3GP5CYZX.js";import{a as _e,c as ke,d as X,e as Re,f as Se}from"./chunk-WO3N5FH2.js";function xe(e){return e.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,"")}import{mkdir as J,readFile as Ye,stat as Xe,writeFile as z}from"fs/promises";import{existsSync as Ze}from"fs";import{homedir as et,platform as tt}from"os";import{dirname as Z,join as U}from"path";import{execFile as rt}from"child_process";function ae(e){return e?.trim()||process.env.MCP_SCRAPER_OUTPUT_DIR?.trim()||U(et(),"Downloads","mcp-scraper")}function Ce(){return new Date().toISOString().replace(/[:.]/g,"-")}function se(e){return xe(e).replace(/^-+|-+$/g,"").slice(0,80)||"run"}var V=class e{constructor(t,n,r,i,s,a,l){this.workflowId=t;this.title=n;this.runId=r;this.baseDir=i;this.runDir=s;this.startedAt=a;this.input=l}workflowId;title;runId;baseDir;runDir;startedAt;input;artifacts=[];static async create(t,n,r,i,s){let a=new Date().toISOString(),l=s?.trim()||`${Ce()}-${se(t)}-${Math.random().toString(36).slice(2,8)}`,c=String(r.keyword??r.query??r.domain??r.state??t),p=ae(i),u=U(p,"workflows",t,`${Ce()}-${se(c)}-${se(l).slice(0,20)}`);return await J(u,{recursive:!0}),new e(t,n,l,p,u,a,r)}async remember(t,n,r,i){let s=await Xe(r).then(a=>a.size).catch(()=>{});return this.artifacts.push({kind:t,label:n,path:r,bytes:s,rows:i}),await ve({title:`${this.title} ${n}`,content:`artifact: ${r}`,source:`mcp-scraper-workflow:${this.workflowId}:${this.runId}`}),r}async writeJson(t,n,r){let i=U(this.runDir,n);return await J(Z(i),{recursive:!0}),await z(i,JSON.stringify(r,null,2),"utf8"),this.remember("json",t,i)}async writeText(t,n,r,i="markdown"){let s=U(this.runDir,n);return await J(Z(s),{recursive:!0}),await z(s,r,"utf8"),this.remember(i,t,s)}async writeCsv(t,n,r,i){let s=U(this.runDir,n);return await J(Z(s),{recursive:!0}),await z(s,be(r,i),"utf8"),this.remember("csv",t,s,i.length)}async writeHtml(t,n,r){return this.writeText(t,n,r,"html_report")}async writeManifest(t,n,r,i){let s={workflow:this.workflowId,title:this.title,runId:this.runId,status:t,startedAt:this.startedAt,completedAt:t==="running"?null:new Date().toISOString(),input:this.input,artifacts:this.artifacts,warnings:r,errors:i,counts:n},a=U(this.runDir,"manifest.json");return await z(a,JSON.stringify(s,null,2),"utf8"),await nt(s,a,this.baseDir),a}};function Ae(e=ae()){return U(e,"workflows","index.json")}async function Pe(e){let t=Ae(e);if(!Ze(t))return{runs:[]};try{return JSON.parse(await Ye(t,"utf8"))}catch{return{runs:[]}}}async function nt(e,t,n){if(e.status==="running")return;let r=Ae(n),i=await Pe(n),s=e.artifacts.find(l=>l.kind==="html_report")?.path??null,a={workflow:e.workflow,runId:e.runId,status:e.status,startedAt:e.startedAt,reportPath:s,manifestPath:t,summary:`${e.title} (${e.status})`};i.runs=[a,...i.runs.filter(l=>l.runId!==e.runId)].slice(0,200),await J(Z(r),{recursive:!0}),await z(r,JSON.stringify(i,null,2),"utf8")}async function $e(e){return(await Pe(ae(e))).runs}async function Ee(e,t){let n=await $e(t);return e==="last"?n[0]??null:n.find(r=>r.runId===e)??null}async function it(e,t){let n=await Ee(e,t);if(!n?.reportPath)throw new Error(`No report found for "${e}"`);return tt()==="darwin"&&await new Promise((r,i)=>{rt("open",[n.reportPath],s=>s?i(s):r())}),n.reportPath}import{readFile as At}from"fs/promises";import{z as dr}from"zod";var F=class{constructor(t,n,r=fetch,i={}){this.apiUrl=t;this.apiKey=n;this.fetchImpl=r;this.extraHeaders=i}apiUrl;apiKey;fetchImpl;extraHeaders;async post(t,n,r=18e4){let i=await this.fetchImpl(`${this.apiUrl.replace(/\/$/,"")}${t}`,{method:"POST",headers:{...this.extraHeaders,"Content-Type":"application/json","x-api-key":this.apiKey},body:JSON.stringify(n),signal:AbortSignal.timeout(r)}),s=await i.json().catch(()=>({}));if(!i.ok){let a=typeof s.error=="string"?s.error:`Workflow API ${t} failed with ${i.status}`;throw new Error(a)}return s}};import{z as D}from"zod";function I(e){return String(e??"").replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#39;")}function ot(e){let t=e.rows.map(n=>`<tr>${e.columns.map(r=>`<td>${I(n[r])}</td>`).join("")}</tr>`).join(`
+`);return["<section>",`<h2>${I(e.title)}</h2>`,'<div class="table-wrap"><table>',`<thead><tr>${e.columns.map(n=>`<th>${I(n)}</th>`).join("")}</tr></thead>`,`<tbody>${t||`<tr><td colspan="${e.columns.length}">No rows</td></tr>`}</tbody>`,"</table></div>","</section>"].join(`
+`)}function P(e){let t=e.warnings?.length?`<section class="warnings"><h2>Warnings</h2><ul>${e.warnings.map(i=>`<li>${I(i)}</li>`).join("")}</ul></section>`:"",n=(e.sections??[]).map(i=>`<section><h2>${I(i.title)}</h2><p>${I(i.body)}</p></section>`).join(`
+`),r=(e.tables??[]).map(ot).join(`
+`);return["<!doctype html>",'<html lang="en">',"<head>",'<meta charset="utf-8">','<meta name="viewport" content="width=device-width, initial-scale=1">',`<title>${I(e.title)}</title>`,"<style>",':root{font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172026;background:#f6f7f8}',"body{margin:0;padding:32px}main{max-width:1180px;margin:0 auto;background:#fff;border:1px solid #d9dee3;border-radius:8px;padding:28px}","h1{font-size:30px;line-height:1.15;margin:0 0 8px}h2{font-size:18px;margin:28px 0 12px}p{line-height:1.55;color:#3d4952}.sub{color:#66737f;margin:0 0 20px}",".summary{background:#eef6f8;border-left:4px solid #12829a;padding:14px 16px;margin:20px 0}.warnings{background:#fff7e8;border-left:4px solid #c87b00;padding:12px 16px}",".table-wrap{overflow:auto;border:1px solid #dfe5ea;border-radius:8px}table{border-collapse:collapse;width:100%;font-size:13px}th,td{border-bottom:1px solid #edf0f2;padding:9px 10px;text-align:left;vertical-align:top}th{background:#f3f5f7;color:#31404c;position:sticky;top:0}tr:last-child td{border-bottom:0}","</style>","</head>","<body><main>",`<h1>${I(e.title)}</h1>`,e.subtitle?`<p class="sub">${I(e.subtitle)}</p>`:"",`<div class="summary">${I(e.summary)}</div>`,t,n,r,"</main></body></html>"].join(`
+`)}var st=D.object({keyword:D.string().min(1),domain:D.string().optional(),location:D.string().optional(),maxQuestions:D.number().int().min(1).max(200).default(40),includeSerp:D.boolean().default(!0),includePaa:D.boolean().default(!0),includeAiOverview:D.boolean().default(!0),returnPartial:D.boolean().default(!0)});function G(e){if(!e)return null;try{return new URL(e.includes("://")?e:`https://${e}`).hostname.replace(/^www\./,"").toLowerCase()}catch{return e.replace(/^https?:\/\//,"").replace(/^www\./,"").split("/")[0]?.toLowerCase()||null}}function ce(e){return G(e)??""}function at(e,t,n){let r=G(e.domain),i=[];for(let s of t?.organicResults??[]){let a=G(s.domain)??ce(s.url);i.push({surface:"organic",position:s.position,title:s.title,url:s.url,domain:a,question:"",answer_excerpt:s.snippet??"",is_target:r?a===r:!1,is_competitor:r?a!==r:!0})}for(let s of n?.flat??[]){let a=s.source_cite??"",l=G(s.source_site??"")??(a?ce(a):"");i.push({surface:"paa",position:"",title:s.source_title??s.source_site??"",url:a,domain:l,question:s.question,answer_excerpt:(s.answer??"").slice(0,240),is_target:r?l===r:!1,is_competitor:r?!!(l&&l!==r):!!l})}for(let[s,a]of(t?.aiOverview?.citations??[]).entries()){let l=a.href??"",c=l?ce(l):"";i.push({surface:"ai_overview",position:s+1,title:a.text??"",url:l,domain:c,question:"",answer_excerpt:"",is_target:r?c===r:!1,is_competitor:r?!!(c&&c!==r):!!c})}return i}function ct(e,t){let n=new Map;for(let r of e){let i=String(r.domain??"");if(!i||i===t)continue;let s=n.get(i)??{organic:[],paa:0,ai:0,urls:new Set};r.surface==="organic"&&r.position&&s.organic.push(Number(r.position)),r.surface==="paa"&&(s.paa+=1),r.surface==="ai_overview"&&(s.ai+=1),r.url&&s.urls.add(String(r.url)),n.set(i,s)}return[...n.entries()].map(([r,i])=>({domain:r,organic_best_position:i.organic.length?Math.min(...i.organic):"",organic_count:i.organic.length,paa_mentions:i.paa,ai_overview_citations:i.ai,source_url_count:i.urls.size})).sort((r,i)=>Number(r.organic_best_position||999)-Number(i.organic_best_position||999))}var We={id:"agent-packet",title:"Agent-Ready SEO Packet",description:"Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",inputSchema:st,createState:()=>({serp:null,paa:null,warnings:[]}),steps:[{id:"harvest-serp",title:"Harvest organic SERP + AI Overview",async run({input:e,state:t,ctx:n}){if(!e.includeSerp)return{state:t,output:{skipped:!0,reason:"includeSerp is false"}};try{let r=await n.client.post("/harvest/sync",{query:e.keyword,location:e.location,serpOnly:!0,includeAllSerpFeatures:!0,maxQuestions:1,format:"json"},62e4);return await n.artifacts.writeJson("SERP raw JSON","raw/serp.json",r),{state:{...t,serp:r},output:{organicResults:r.organicResults?.length??0,localPack:r.localPack?.length??0,aiOverviewDetected:!!r.aiOverview?.detected,aiOverviewCitations:r.aiOverview?.citations?.length??0}}}catch(r){let i=`SERP evidence unavailable: ${r instanceof Error?r.message:String(r)}`;return{state:{...t,warnings:[...t.warnings,i]},output:{error:i},warnings:[i]}}}},{id:"harvest-paa",title:"Harvest People Also Ask",async run({input:e,state:t,ctx:n}){if(!e.includePaa)return{state:t,output:{skipped:!0,reason:"includePaa is false"}};try{let r=await n.client.post("/harvest/sync",{query:e.keyword,location:e.location,maxQuestions:e.maxQuestions,format:"json"},735e3);return await n.artifacts.writeJson("PAA raw JSON","raw/paa.json",r),{state:{...t,paa:r},output:{paaQuestions:r.flat?.length??0}}}catch(r){let i=`PAA evidence unavailable: ${r instanceof Error?r.message:String(r)}`;return{state:{...t,warnings:[...t.warnings,i]},output:{error:i},warnings:[i]}}}},{id:"assemble",title:"Assemble evidence packet",async run({input:e,state:t,ctx:n}){let{serp:r,paa:i}=t,s=t.warnings;if(!r&&!i&&!e.returnPartial)throw new Error("No SEO evidence was collected");let a=at(e,r,i),l=G(e.domain),c=ct(a,l),p={input:e,serp:r??{status:"skipped"},paa:i??{status:"skipped"},target:{domain:l,organicPositions:a.filter(d=>d.surface==="organic"&&d.is_target).map(d=>Number(d.position)),citedInAiOverview:a.some(d=>d.surface==="ai_overview"&&d.is_target),citedInPaa:a.some(d=>d.surface==="paa"&&d.is_target)},competitors:c,warnings:s};await n.artifacts.writeJson("Evidence JSON","evidence.json",p),await n.artifacts.writeCsv("Sources CSV","sources.csv",["surface","position","title","url","domain","question","answer_excerpt","is_target","is_competitor"],a),await n.artifacts.writeCsv("Competitors CSV","competitors.csv",["domain","organic_best_position","organic_count","paa_mentions","ai_overview_citations","source_url_count"],c);let u=[`# SEO Evidence Brief: ${e.keyword}`,"",`Location: ${e.location??"not specified"}`,`Target domain: ${l??"not specified"}`,"","## Evidence Summary",`- Organic rows: ${a.filter(d=>d.surface==="organic").length}`,`- PAA rows: ${a.filter(d=>d.surface==="paa").length}`,`- AI Overview citations: ${a.filter(d=>d.surface==="ai_overview").length}`,`- Competitor domains: ${c.length}`,"","## Recommended Use","Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."].join(`
+`);await n.artifacts.writeText("Brief","brief.md",u),await n.artifacts.writeText("Agent tasks","tasks.md",["# Agent Tasks","","- [ ] Read `evidence.json` before writing recommendations.","- [ ] Compare the target domain against `competitors.csv`.","- [ ] Use `sources.csv` for citations and source-grounded page sections.","- [ ] Mark unsupported recommendations as assumptions."].join(`
+`)),await n.artifacts.writeText("Agent instructions","agent-instructions.md",["# Agent Instructions","","You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."].join(`
+`));let w=`${a.length} evidence rows and ${c.length} competitor domains collected.`,v=await n.artifacts.writeHtml("HTML report","report.html",P({title:"Agent-Ready SEO Packet",subtitle:`${e.keyword}${e.location?` \xB7 ${e.location}`:""}`,summary:w,warnings:s,tables:[{title:"Competitor Domains",columns:["domain","organic_best_position","organic_count","paa_mentions","ai_overview_citations","source_url_count"],rows:c.slice(0,50)},{title:"Evidence Sources",columns:["surface","position","title","domain","question","is_target"],rows:a.slice(0,100)}]})),f=s.length?"partial":"succeeded",b={sources:a.length,competitors:c.length};return await n.artifacts.writeManifest(f,b,s,[]),{state:t,output:{sources:a.length,competitors:c.length,status:f},summary:{title:"Agent-Ready SEO Packet",summary:w,status:f,counts:b,warnings:s,errors:[],reportPath:v}}}}]};import{z as q}from"zod";var lt=q.object({query:q.string().min(1),state:q.string().min(2).default("TN"),minPopulation:q.number().int().min(0).default(1e5),maxCities:q.number().int().min(1).max(100).default(25),maxResultsPerCity:q.number().int().min(1).max(50).default(20),concurrency:q.number().int().min(1).max(5).default(5),proxyMode:q.enum(["configured","none"]).default(T),saveCsv:q.boolean().default(!0)});function Q(e){let t=[];for(let n of e.cities){if(!n.results.length){t.push({source_query:e.query,source_location:n.location,city:n.city,state:n.state,population:n.population,result_position:null,business_name:null,review_stars:null,review_count:null,category:null,address:null,phone:null,website_url:null,place_url:null,cid:null,cid_decimal:null,result_status:n.status,error:n.error});continue}for(let r of n.results)t.push({source_query:e.query,source_location:n.location,city:n.city,state:n.state,population:n.population,result_position:r.position,business_name:r.name,review_stars:r.rating,review_count:r.reviewCount,category:r.category,address:r.address,phone:r.phone,website_url:r.websiteUrl,place_url:r.placeUrl,cid:r.cid,cid_decimal:r.cidDecimal,result_status:n.status,error:n.error})}return t}var K=["source_query","source_location","city","state","population","result_position","business_name","review_stars","review_count","category","address","phone","website_url","place_url","cid","cid_decimal","result_status","error"],Me={id:"directory",title:"Directory Workflow",description:"Select city markets and export Google Maps business candidates.",inputSchema:lt,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=await t.client.post("/directory/run",{...e,background:!1},9e5);await t.artifacts.writeJson("Directory evidence","evidence.json",n);let r=Q(n);await t.artifacts.writeCsv("Directory CSV","exports/directory.csv",K,r);let i=n.cities.map(u=>({city:u.city,state:u.state,population:u.population,status:u.status,result_count:u.resultCount,error:u.error??""})),s=r.filter(u=>u.business_name).slice(0,100).map(u=>({city:u.city,position:u.result_position,business:u.business_name,rating:u.review_stars,reviews:u.review_count,category:u.category,website:u.website_url})),a=`${n.selectedCityCount} cities processed with ${n.totalResultCount} Maps results.`;await t.artifacts.writeText("Summary","summary.md",`# Directory Workflow
+
+${a}
+`);let l=await t.artifacts.writeHtml("HTML report","report.html",P({title:"Directory Workflow",subtitle:`${e.query} \xB7 ${e.state}`,summary:a,warnings:n.warnings,tables:[{title:"Cities",columns:["city","state","population","status","result_count","error"],rows:i},{title:"Top Results",columns:["city","position","business","rating","reviews","category","website"],rows:s}]})),c=n.cities.some(u=>u.status==="failed")?"partial":"succeeded",p={cities:n.selectedCityCount,results:n.totalResultCount,rows:r.length};return await t.artifacts.writeManifest(c,p,n.warnings,[]),{title:"Directory Workflow",summary:a,status:c,counts:p,warnings:n.warnings,errors:[],reportPath:l}}};import{z as O}from"zod";var ut=O.object({query:O.string().min(1).describe('Business category, niche, or keyword to search on Google Maps, e.g. "roofers", "med spas", "dentists". Do not include the city here.'),location:O.string().min(1).describe('City / market to search, e.g. "Houston, TX" or "Austin, Texas".'),maxResults:O.number().int().min(1).max(50).default(25).describe("How many Maps businesses to collect for the market. Maximum 50."),enrichWebsites:O.boolean().default(!0).describe("Visit each business website (home + contact pages) to harvest email and social links. Uses the proxy/browser-backed extractor so blocked sites still resolve."),emailSearchFallback:O.enum(["off","serp_snippets"]).default("off").describe("After bounded website extraction finds no email, optionally run one billed SERP lookup and retain exact-domain public email occurrences from result titles/snippets."),hydrateReviewCounts:O.boolean().default(!0).describe("Deep-dive each profile to confirm the review count and booking URL that the Maps search list omits."),concurrency:O.number().int().min(1).max(4).default(3).describe("How many businesses to enrich in parallel. Keep low to respect per-account concurrency limits."),proxyMode:O.enum(["configured","none"]).default(T).describe("Proxy behavior for the Maps search. Leave unset for clean egress; country/region localization comes from gl/hl plus the city or region in the query.")}),mt=["position","business_name","review_stars","review_count","category","address","phone","website","email","email_2","email_domain_match","email_status","socials","booking_url","place_url","cid","source_location"];async function pt(e,t,n){let r=new Array(e.length),i=0;async function s(){for(;i<e.length;){let a=i;i+=1,r[a]=await n(e[a],a)}}return await Promise.all(Array.from({length:Math.min(t,e.length)},()=>s())),r}function dt(e){try{return new URL(e).hostname.toLowerCase().replace(/^www\./,"").replace(/\.$/,"")}catch{return""}}function Ie(e,t){let n=dt(t),r=e.slice(e.lastIndexOf("@")+1).toLowerCase().replace(/^www\./,"").replace(/\.$/,"");return!!(n&&r===n)}function Le(e,t){let n=new Set(e.map(r=>r.value.toLowerCase()));for(let r of t){if(e.length>=20)break;let i=r.value.toLowerCase();n.has(i)||(n.add(i),e.push(r))}}function De(e,t,n){let r=new Set(e.map(i=>i.url));for(let i of t){if(e.length>=n)break;r.has(i.url)||(r.add(i.url),e.push(i))}}function qe(e,t){let n=[...e],r=[],i={entityType:"business",name:t,organization:t};for(;n.length;){let s=ke(n,i);if(!s){r.push(...n);break}r.push(s);let a=n.findIndex(l=>l.value.toLowerCase()===s.value.toLowerCase());n.splice(a,1)}return r}var Oe={id:"get-leads",title:"Get Leads",description:"Build an outreach-ready local lead list: Google Maps search for a niche in a market, confirm review counts + booking URLs, then visit each business website (home + contact) to harvest email and social links. An opt-in SERP fallback can recover exact-domain public email occurrences after bounded pages are empty. Saves a leads CSV.",inputSchema:ut,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=[],r=[],i=await t.client.post("/maps/search",{query:e.query,location:e.location,maxResults:e.maxResults,proxyMode:e.proxyMode},18e4);await t.artifacts.writeJson("Maps search raw","raw/maps-search.json",i);let s=new Set,a=i.results.filter(o=>{let h=o.cid??o.placeUrl??o.name.toLowerCase();return s.has(h)?!1:(s.add(h),!0)}),l=await pt(a,e.concurrency,async o=>{let h=o.reviewCount??"",R="",m=o.websiteUrl??"";if(e.hydrateReviewCounts)try{let E=await t.client.post("/maps/place",{businessName:o.name,location:e.location,includeReviews:!1},18e4);h=E.reviewCount??h,R=E.bookingUrl??"",m=m||(E.website??"")}catch(E){n.push(`Review hydration failed for ${o.name}: ${E instanceof Error?E.message:String(E)}`)}let x="",S="",B=[],j=m?"no email found":"no website",N="",_=[],L=[],Y=[],we=[];if(e.enrichWebsites&&m){let E=X(m,Y,3),fe=!1,ne=new Set;for(;ne.size<3;){let C=E.find(W=>!ne.has(W));if(!C)break;ne.add(C),we.push(C);try{let W=await t.client.post("/extract-url",{url:C},12e4);fe=!0;let oe=W.contactEvidence??_e(W.bodyHtml??W.bodyMarkdown??"",C);if(Le(_,oe.emails),De(L,oe.socialProfiles,20),De(Y,oe.candidatePageUrls,20),E=X(m,Y,3),_.length)break}catch{E=X(m,Y,3);continue}}if(!_.length&&e.emailSearchFallback==="serp_snippets"){let C=Re(m);if(C)try{let W=await t.client.post("/harvest/sync",{query:C,serpOnly:!0,pages:1},18e4);Le(_,Se({websiteUrl:m,results:W.result?.organicResults??[]}))}catch(W){n.push(`Email snippet search failed for ${o.name}: ${W instanceof Error?W.message:String(W)}`)}}B=L.map(C=>C.url).slice(0,5);let he=qe(_,o.name),ye=he.filter(C=>Ie(C.value,m)),ie=ye.length?ye.slice(0,2):he.slice(0,1);x=ie[0]?.value??"",S=ie[1]?.value??"",x?(N=Ie(x,m)?"yes":"NO-verify",j=ie.some(C=>C.sourceType==="serp_snippet")?"serp snippet \u2014 public occurrence, verify":N==="yes"?"ok":"ok (off-domain \u2014 verify)"):fe?B.length&&(j="form/social only"):j="site unreachable"}return{row:{position:o.position,business_name:o.name,review_stars:o.rating??"",review_count:h,category:o.category??"",address:o.address??"",phone:o.phone??"",website:m,email:x,email_2:S,email_domain_match:N,email_status:j,socials:B.join(" ; "),booking_url:R,place_url:o.placeUrl,cid:o.cid??"",source_location:e.location},emailEvidence:qe(_,o.name),socialProfileEvidence:L,attemptedUrls:we}}),c=l.map(o=>o.row);await t.artifacts.writeCsv("Leads CSV","exports/leads.csv",mt,c),await t.artifacts.writeJson("Leads evidence","evidence.json",{input:e,search:{query:i.searchQuery,resultCount:i.resultCount},rows:l.map(o=>({...o.row,emailEvidence:o.emailEvidence,socialProfileEvidence:o.socialProfileEvidence,attemptedUrls:o.attemptedUrls}))});let p=c.filter(o=>o.email).length,u=c.filter(o=>o.email_domain_match==="yes").length,w=c.filter(o=>o.review_count).length,v=c.filter(o=>o.socials).length,f=c.filter(o=>o.website).length,b=`${c.length} ${e.query} in ${e.location}: ${p} emails (${u} domain-matched), ${w} review counts, ${v} with socials.`;await t.artifacts.writeText("Summary","summary.md",`# Get Leads
+
+${b}
+`),await t.artifacts.writeText("Agent tasks","tasks.md",["# Get Leads \u2014 next steps","","- [ ] Verify deliverability of the harvested emails before outreach (MX/SMTP check).",'- [ ] For "form/social only" rows, use the website contact form or the captured social profile.','- [ ] Treat "off-domain \u2014 verify" emails with care; they may belong to a web developer or parent brand.',"- [ ] Respect CAN-SPAM / GDPR \u2014 scraped contact data is not consent to email."].join(`
+`));let d=await t.artifacts.writeHtml("HTML report","report.html",P({title:"Get Leads",subtitle:`${e.query} \xB7 ${e.location}`,summary:b,warnings:n,tables:[{title:"Leads",columns:["position","business_name","review_stars","review_count","phone","website","email","email_2","email_status","socials","booking_url"],rows:c}]})),k=r.length?"failed":n.length?"partial":"succeeded",g={businesses:c.length,withWebsite:f,emails:p,domainMatchedEmails:u,reviewCounts:w,withSocials:v};return await t.artifacts.writeManifest(k,g,n,r),{title:"Get Leads",summary:b,status:k,counts:g,warnings:n,errors:r,reportPath:d}}};import{z as M}from"zod";var gt=M.object({query:M.string().min(1),state:M.string().min(2).default("TN"),minPopulation:M.number().int().min(0).default(1e5),maxCities:M.number().int().min(1).max(100).default(25),maxResultsPerCity:M.number().int().min(1).max(50).default(20),hydrateTop:M.number().int().min(0).max(10).default(5),maxReviews:M.number().int().min(0).max(500).default(50),concurrency:M.number().int().min(1).max(5).default(5),proxyMode:M.enum(["configured","none"]).default(T),returnPartial:M.boolean().default(!0)});async function wt(e,t,n){let r=new Array(e.length),i=0;async function s(){for(;i<e.length;){let a=i;i+=1,r[a]=await n(e[a],a)}}return await Promise.all(Array.from({length:Math.min(t,e.length)},()=>s())),r}function le(e){if(e==null||e==="")return null;let t=Number(String(e).replace(/[^\d.]/g,""));return Number.isFinite(t)?t:null}function Te(e){let t=e.filter(n=>n!==null).sort((n,r)=>n-r);return t.length?t[Math.floor(t.length/2)]??null:null}function je(e){let t=new Set(["the","and","for","with","that","this","was","were","are","you","our","they","had","have","not","but","from","very","great","good"]),n=new Map;for(let r of e)for(let i of r.toLowerCase().replace(/[^a-z0-9\s]/g," ").split(/\s+/))i.length<4||t.has(i)||n.set(i,(n.get(i)??0)+1);return[...n.entries()].sort((r,i)=>i[1]-r[1]).slice(0,8).map(([r,i])=>`${r} (${i})`).join("; ")}var Ue={id:"local-competitive-audit",title:"Local Competitive Audit",description:"Audit local Maps competitors, categories, review counts, and review themes across city markets.",inputSchema:gt,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=[],r=[],i=await t.client.post("/directory/run",{query:e.query,state:e.state,minPopulation:e.minPopulation,maxCities:e.maxCities,maxResultsPerCity:e.maxResultsPerCity,concurrency:e.concurrency,proxyMode:e.proxyMode,saveCsv:!0,background:!1},9e5);await t.artifacts.writeJson("Directory raw JSON","raw/directory-workflow.json",i);let s=Q(i);await t.artifacts.writeCsv("Base directory CSV","exports/directory.csv",K,s);let a=i.cities.flatMap(g=>g.results.slice(0,e.hydrateTop).map(o=>({city:g,result:o}))),l=new Set,c=a.filter(({city:g,result:o})=>{let h=o.cid??o.placeUrl??`${g.location}:${o.name.toLowerCase()}`;return l.has(h)?!1:(l.add(h),!0)}),p=await wt(c,3,async({city:g,result:o},h)=>{try{let R=await t.client.post("/maps/place",{businessName:o.name,location:g.location,includeReviews:e.maxReviews>0,maxReviews:Math.max(1,e.maxReviews)},18e4);return await t.artifacts.writeJson(`${o.name} profile`,`raw/maps-place-intel/${h+1}-${o.name.toLowerCase().replace(/[^a-z0-9]+/g,"-")}.json`,R),{city:g,result:o,detail:R,error:null}}catch(R){let m=R instanceof Error?R.message:String(R);return n.push(`Profile hydration failed for ${o.name} (${g.location}): ${m}`),{city:g,result:o,detail:null,error:m}}}),u=p.map(({city:g,result:o,detail:h,error:R})=>{let m=h?.reviews??[],x=m.filter(S=>S.ownerResponse).length;return{city:g.city,state:g.state,source_location:g.location,result_position:o.position,business_name:o.name,category:h?.category??o.category,review_stars:h?.rating??o.rating,review_count:h?.reviewCount??o.reviewCount,phone:o.phone,website_url:h?.website??o.websiteUrl,place_url:o.placeUrl,cid:o.cid,cid_decimal:o.cidDecimal,hydrated:h?"true":"false",reviews_status:h?.reviewsStatus??"",review_topics:(h?.reviewTopics??[]).map(S=>`${S.label} (${S.count})`).join("; "),owner_response_rate:m.length?(x/m.length).toFixed(2):"",error:R??""}}),w=p.map(({city:g,result:o,detail:h})=>{let R=h?.reviews??[],m=R.filter(S=>Number(S.stars)>=5&&S.text).map(S=>S.text),x=R.filter(S=>Number(S.stars)>0&&Number(S.stars)<=3&&S.text).map(S=>S.text);return{city:g.city,business_name:o.name,cid:o.cid,review_count:h?.reviewCount??o.reviewCount,average_rating:h?.rating??o.rating,topics:(h?.reviewTopics??[]).map(S=>`${S.label} (${S.count})`).join("; "),praise_terms:je(m),complaint_terms:je(x),positive_sample:m[0]?.slice(0,280)??"",negative_sample:x[0]?.slice(0,280)??""}}),v=i.cities.map(g=>{let o=g.results.map(_=>le(_.reviewCount)),h=g.results.map(_=>le(_.rating)),R=g.results.slice(0,3).map(_=>le(_.reviewCount)).filter(_=>_!==null),m=new Map;for(let _ of g.results)_.category&&m.set(_.category,(m.get(_.category)??0)+1);let x=Te(o),S=Te(h),B=R.length?Math.round(R.reduce((_,L)=>_+L,0)/R.length):null,j=Math.min(100,Math.round((B??0)/20+(S??0)*10+g.results.filter(_=>_.websiteUrl).length)),N=Math.max(0,100-j);return{city:g.city,state:g.state,population:g.population,result_count:g.resultCount,median_review_count:x??"",median_rating:S??"",top_three_average_review_count:B??"",top_categories:[...m.entries()].sort((_,L)=>L[1]-_[1]).slice(0,5).map(([_,L])=>`${_} (${L})`).join("; "),opportunity_score:N,difficulty_score:j}});await t.artifacts.writeCsv("Competitors CSV","competitors.csv",["city","state","source_location","result_position","business_name","category","review_stars","review_count","phone","website_url","place_url","cid","cid_decimal","hydrated","reviews_status","review_topics","owner_response_rate","error"],u),await t.artifacts.writeCsv("Review insights CSV","review-insights.csv",["city","business_name","cid","review_count","average_rating","topics","praise_terms","complaint_terms","positive_sample","negative_sample"],w),await t.artifacts.writeCsv("City summary CSV","city-summary.csv",["city","state","population","result_count","median_review_count","median_rating","top_three_average_review_count","top_categories","opportunity_score","difficulty_score"],v),await t.artifacts.writeJson("Audit evidence","evidence.json",{input:e,directory:i,hydrated:p,citySummaryRows:v,competitorRows:u,reviewInsightRows:w}),await t.artifacts.writeText("Agent tasks","tasks.md",["# Local Competitive Audit Tasks","","- [ ] Review city opportunity and difficulty scores as heuristics, not absolute truth.","- [ ] Use review topics and samples as customer-language evidence.","- [ ] Compare target GBP category, review count, and website quality against top competitors.","- [ ] Identify cities with low review-count leaders and weak website coverage."].join(`
+`));let f=`${i.selectedCityCount} cities, ${i.totalResultCount} Maps results, ${p.filter(g=>g.detail).length} hydrated profiles.`;await t.artifacts.writeText("Summary","summary.md",`# Local Competitive Audit
+
+${f}
+`);let b=await t.artifacts.writeHtml("HTML report","report.html",P({title:"Local Competitive Audit",subtitle:`${e.query} \xB7 ${e.state}`,summary:f,warnings:n,tables:[{title:"City Opportunity",columns:["city","state","population","result_count","median_review_count","median_rating","top_three_average_review_count","top_categories","opportunity_score","difficulty_score"],rows:v},{title:"Hydrated Competitors",columns:["city","result_position","business_name","category","review_stars","review_count","hydrated","review_topics"],rows:u.slice(0,100)},{title:"Review Insights",columns:["city","business_name","topics","praise_terms","complaint_terms"],rows:w.slice(0,100)}]})),d=n.length||i.cities.some(g=>g.status==="failed")?"partial":"succeeded",k={cities:i.selectedCityCount,mapsResults:i.totalResultCount,hydratedProfiles:p.filter(g=>g.detail).length};return await t.artifacts.writeManifest(d,k,n,r),{title:"Local Competitive Audit",summary:f,status:d,counts:k,warnings:n,errors:r,reportPath:b}}};import{z as y}from"zod";var ft=new Set(["about","after","also","because","been","best","both","from","have","into","more","most","near","only","over","than","that","their","them","then","there","these","they","this","what","when","where","which","while","with","your","will","would","should","could","does","were","cost","costs"]);function $(e){if(!e)return null;try{return new URL(e.includes("://")?e:`https://${e}`).hostname.replace(/^www\./,"").toLowerCase()}catch{return e.replace(/^https?:\/\//,"").replace(/^www\./,"").split("/")[0]?.toLowerCase()||null}}function A(e){return $(e)??""}function H(e){if(e==null||e==="")return null;let t=Number(String(e).replace(/[^\d.]/g,""));return Number.isFinite(t)?t:null}function ue(e){let t=e.filter(n=>n!==null).sort((n,r)=>n-r);return t.length?t[Math.floor(t.length/2)]??null:null}function me(e,t=12){let n=new Map;for(let r of e.toLowerCase().replace(/[^a-z0-9\s]/g," ").split(/\s+/))r.length<4||ft.has(r)||n.set(r,(n.get(r)??0)+1);return[...n.entries()].sort((r,i)=>i[1]-r[1]||r[0].localeCompare(i[0])).slice(0,t).map(([r,i])=>`${r} (${i})`).join("; ")}function ht(e){let t=e.toLowerCase();return/\b(cost|price|pricing|charge|expensive|cheap)\b/.test(t)?"cost":/\b(best|top|recommended|reviews?|compare|versus|vs)\b/.test(t)?"comparison":/\b(how|steps?|process|way to)\b/.test(t)?"process":/\b(why|worth|important|benefit)\b/.test(t)?"why":/\b(near me|city|local|nearby)\b/.test(t)?"local":/\b(can|does|do|is|are|should|will)\b/.test(t)?"decision":"definition"}function ee(e){return(e?.flat??[]).filter(t=>t.question).map((t,n)=>{let r=t.source_cite??"",i=$(t.source_site??"")??A(r);return{position:n+1,intent:ht(t.question??""),question:t.question??"",answer_excerpt:(t.answer??"").slice(0,320),source_title:t.source_title??"",source_domain:i,source_url:r}})}function He(e){let t=new Map;for(let n of e){let r=String(n.source_domain??n.domain??"");if(!r)continue;let i=t.get(r)??{questions:0,urls:new Set,intents:new Map};i.questions+=n.question?1:0,(n.source_url||n.url)&&i.urls.add(String(n.source_url??n.url)),n.intent&&i.intents.set(String(n.intent),(i.intents.get(String(n.intent))??0)+1),t.set(r,i)}return[...t.entries()].map(([n,r])=>({domain:n,question_mentions:r.questions,source_url_count:r.urls.size,top_intents:[...r.intents.entries()].sort((i,s)=>s[1]-i[1]).map(([i,s])=>`${i} (${s})`).join("; ")})).sort((n,r)=>Number(r.question_mentions)-Number(n.question_mentions)||String(n.domain).localeCompare(String(r.domain)))}function Be(e){return(e??"").replace(/\s+/g," ").split(/(?<=[.!?])\s+/).map(t=>t.trim()).filter(t=>t.length>20).slice(0,20)}function Ne(e){let t=e.toLowerCase();return/\bis\b|\bare\b|\bmeans\b|\brefers to\b/.test(t)?"definition":/\binclude\b|\bconsider\b|\bfactors?\b|\bcriteria\b/.test(t)?"criteria":/\bfirst\b|\bthen\b|\bsteps?\b|\bprocess\b/.test(t)?"process":/\bvs\b|\bthan\b|\bcompare\b|\bdifference\b/.test(t)?"comparison":/\bcost\b|\bprice\b|\baverage\b|\brange\b/.test(t)?"cost":"claim"}function te(e,t){return{position:t.position??"",domain:t.domain,url:t.url,serp_title:t.title??"",page_title:e.title??"",h1:e.h1??"",meta_description:e.metaDescription??"",word_count:e.wordCount??"",heading_count:e.headings?.length??0,schema_types:(e.schemaTypes??[]).join("; ")}}async function pe(e,t,n){let r=new Array(e.length),i=0;async function s(){for(;i<e.length;){let a=i;i+=1,r[a]=await n(e[a],a)}}return await Promise.all(Array.from({length:Math.min(Math.max(1,t),e.length)},()=>s())),r}var yt=y.enum(["configured","none"]),vt=y.object({query:y.string().min(1),location:y.string().optional(),state:y.string().optional(),minPopulation:y.number().int().min(0).default(1e5),maxCities:y.number().int().min(1).max(100).default(5),maxResultsPerCity:y.number().int().min(1).max(50).default(20),hydrateTop:y.number().int().min(0).max(10).default(5),maxReviews:y.number().int().min(0).max(500).default(25),concurrency:y.number().int().min(1).max(5).default(5),proxyMode:yt.default(T),returnPartial:y.boolean().default(!0)}).refine(e=>e.location||e.state,{message:"Either location or state is required for map-comparison"}),_t=y.object({keyword:y.string().min(1),domain:y.string().optional(),url:y.string().url().optional(),location:y.string().optional(),maxResults:y.number().int().min(1).max(20).default(10),maxQuestions:y.number().int().min(1).max(200).default(40),extractTop:y.number().int().min(0).max(10).default(5),includePaa:y.boolean().default(!0),includeAiOverview:y.boolean().default(!0),returnPartial:y.boolean().default(!0)}),bt=y.object({keyword:y.string().min(1),location:y.string().optional(),maxQuestions:y.number().int().min(1).max(300).default(80),depth:y.number().int().min(1).max(6).default(3),returnPartial:y.boolean().default(!0)}),kt=y.object({keyword:y.string().min(1),domain:y.string().optional(),url:y.string().url().optional(),location:y.string().optional(),maxQuestions:y.number().int().min(1).max(200).default(40),extractTop:y.number().int().min(0).max(8).default(3),returnPartial:y.boolean().default(!0)});function Rt(e,t,n){return n.map(r=>({source_query:t,source_location:e,city:e.split(",")[0]?.trim()??e,state:e.split(",")[1]?.trim()??"",population:"",result_position:r.position,business_name:r.name,review_stars:r.rating??"",review_count:r.reviewCount??"",category:r.category??"",address:r.address??"",phone:r.phone??"",website_url:r.websiteUrl??"",place_url:r.placeUrl??"",cid:r.cid??"",cid_decimal:r.cidDecimal??"",result_status:"ok",error:""}))}function Je(e){let t=new Map;for(let n of e){let r=String(n.source_location??n.location??"");if(!r)continue;let i=t.get(r)??[];i.push(n),t.set(r,i)}return[...t.entries()].map(([n,r])=>{let i=r.map(c=>H(c.review_count)),s=r.map(c=>H(c.review_stars)),a=r.slice(0,3).map(c=>H(c.review_count)).filter(c=>c!==null),l=new Map;for(let c of r){let p=String(c.category??"");p&&l.set(p,(l.get(p)??0)+1)}return{source_location:n,result_count:r.filter(c=>c.business_name).length,median_review_count:ue(i)??"",median_rating:ue(s)??"",top_three_average_review_count:a.length?Math.round(a.reduce((c,p)=>c+p,0)/a.length):"",top_categories:[...l.entries()].sort((c,p)=>p[1]-c[1]).slice(0,5).map(([c,p])=>`${c} (${p})`).join("; "),websites_present:r.filter(c=>c.website_url).length}})}function St(e){let t=new Map;for(let n of Je(e))t.set(String(n.source_location),H(n.top_three_average_review_count)??0);return e.filter(n=>n.business_name).map(n=>{let r=H(n.review_count)??0,i=t.get(String(n.source_location))??0,s=!n.website_url,a=H(n.result_position)??999;return{source_location:n.source_location,result_position:n.result_position,business_name:n.business_name,category:n.category,review_stars:n.review_stars,review_count:n.review_count,review_gap_to_top3_average:i?Math.max(0,i-r):"",website_url:n.website_url,place_url:n.place_url,comparison_note:a<=3?"visible leader":s?"ranking without website":r<i?"review-light competitor":"visible competitor"}})}function xt(e,t){return(e?.organicResults??[]).map(n=>{let r=n.url??"",i=$(n.domain??"")??A(r);return{position:n.position??"",title:n.title??"",url:r,domain:i,snippet:n.snippet??"",is_target:t?i===t:!1}})}function Ct(e,t){let n=new Set((e?.headings??[]).map(s=>s.text.toLowerCase().replace(/[^a-z0-9\s]/g," ").trim())),r=new Set((e?.headings??[]).flatMap(s=>s.text.toLowerCase().replace(/[^a-z0-9\s]/g," ").split(/\s+/).filter(Boolean))),i=[];for(let{source:s,page:a}of t)for(let l of a.headings??[]){if(l.level>3||!l.text)continue;let c=l.text.toLowerCase().replace(/[^a-z0-9\s]/g," ").trim(),p=c.split(/\s+/).filter(v=>v.length>3),u=p.filter(v=>r.has(v)).length;(n.has(c)||u>=Math.max(2,Math.ceil(p.length/2)))&&e||i.push({source_position:s.position??"",source_domain:$(s.domain??"")??A(s.url),source_url:s.url??"",heading_level:l.level,competitor_heading:l.text,target_coverage:e?"not found in target headings":"no target page extracted",terms:me(l.text,6)})}return i.slice(0,150)}async function de(e,t,n,r){return pe(t.filter(i=>i.url),2,async(i,s)=>{try{let a=await e.client.post("/extract-url",{url:i.url},18e4);return await e.artifacts.writeJson(`${r} ${s+1}`,`raw/extract-url/${r.toLowerCase()}-${s+1}.json`,a),{source:i,page:a}}catch(a){return n.push(`Page extraction failed for ${i.url}: ${a instanceof Error?a.message:String(a)}`),null}}).then(i=>i.filter(s=>s!==null))}var ze={id:"map-comparison",title:"Maps Comparison",description:"Compare Google Maps competitors by rank, reviews, stars, categories, websites, and profile/review signals.",inputSchema:vt,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=[],r,i=null,s=null;e.location?(s=await t.client.post("/maps/search",{query:e.query,location:e.location,maxResults:e.maxResultsPerCity,proxyMode:e.proxyMode},24e4),r=Rt(e.location,e.query,s.results),await t.artifacts.writeJson("Maps search raw JSON","raw/maps-search.json",s)):(i=await t.client.post("/directory/run",{query:e.query,state:e.state,minPopulation:e.minPopulation,maxCities:e.maxCities,maxResultsPerCity:e.maxResultsPerCity,concurrency:e.concurrency,proxyMode:e.proxyMode,saveCsv:!0,background:!1},9e5),r=Q(i),await t.artifacts.writeJson("Directory raw JSON","raw/directory-workflow.json",i),await t.artifacts.writeCsv("Directory CSV","exports/directory.csv",K,r),n.push(...i.warnings));let a=St(r),l=a.slice(0,e.hydrateTop*Math.max(1,e.location?1:e.maxCities)),p=(await pe(l,3,async(d,k)=>{try{let g=await t.client.post("/maps/place",{businessName:d.business_name,location:d.source_location,includeReviews:e.maxReviews>0,maxReviews:Math.max(1,e.maxReviews)},18e4);return await t.artifacts.writeJson(`${d.business_name} profile`,`raw/maps-place-intel/${k+1}-${String(d.business_name).toLowerCase().replace(/[^a-z0-9]+/g,"-")}.json`,g),{row:d,detail:g,error:""}}catch(g){let o=g instanceof Error?g.message:String(g);return n.push(`Profile hydration failed for ${d.business_name}: ${o}`),{row:d,detail:null,error:o}}})).map(({row:d,detail:k,error:g})=>({source_location:d.source_location,result_position:d.result_position,business_name:d.business_name,category:k?.category??d.category,review_stars:k?.rating??d.review_stars,review_count:k?.reviewCount??d.review_count,website_url:k?.website??d.website_url,review_topics:(k?.reviewTopics??[]).map(o=>`${o.label} (${o.count})`).join("; "),about_attributes:(k?.aboutAttributes??[]).map(o=>`${o.section}: ${o.attribute}`).join("; "),reviews_status:k?.reviewsStatus??"",error:g})),u=Je(r);await t.artifacts.writeCsv("Maps results CSV","maps-results.csv",["source_query","source_location","city","state","population","result_position","business_name","review_stars","review_count","category","address","phone","website_url","place_url","cid","cid_decimal","result_status","error"],r),await t.artifacts.writeCsv("Comparison CSV","map-comparison.csv",["source_location","result_position","business_name","category","review_stars","review_count","review_gap_to_top3_average","website_url","place_url","comparison_note"],a),await t.artifacts.writeCsv("Profile insights CSV","profile-insights.csv",["source_location","result_position","business_name","category","review_stars","review_count","website_url","review_topics","about_attributes","reviews_status","error"],p),await t.artifacts.writeJson("Maps comparison evidence","evidence.json",{input:e,directory:i,mapsSearch:s,rows:r,compareRows:a,profileRows:p,markets:u,warnings:n}),await t.artifacts.writeText("Brief","brief.md",[`# Maps Comparison: ${e.query}`,"",`Markets: ${u.map(d=>d.source_location).join(", ")}`,"","## How to Use","- Compare rank position against review count and category patterns.","- Treat review gaps and missing websites as opportunity signals, not guarantees.","- Use profile topics and attributes as evidence for local content and GBP improvements."].join(`
+`));let w=`${a.length} Maps competitors compared across ${u.length} market(s); ${p.filter(d=>!d.error).length} profiles hydrated.`,v=await t.artifacts.writeHtml("HTML report","report.html",P({title:"Maps Comparison",subtitle:`${e.query}${e.location?` \xB7 ${e.location}`:e.state?` \xB7 ${e.state}`:""}`,summary:w,warnings:n,tables:[{title:"Market Benchmarks",columns:["source_location","result_count","median_review_count","median_rating","top_three_average_review_count","top_categories","websites_present"],rows:u},{title:"Competitor Comparison",columns:["source_location","result_position","business_name","category","review_stars","review_count","review_gap_to_top3_average","comparison_note"],rows:a.slice(0,120)},{title:"Profile Insights",columns:["source_location","business_name","review_topics","about_attributes","error"],rows:p}]})),f=n.length?"partial":"succeeded",b={markets:u.length,competitors:a.length,hydratedProfiles:p.filter(d=>!d.error).length};return await t.artifacts.writeManifest(f,b,n,[]),{title:"Maps Comparison",summary:w,status:f,counts:b,warnings:n,errors:[],reportPath:v}}},Ve={id:"serp-comparison",title:"SERP Comparison",description:"Compare ranking pages, SERP features, PAA evidence, AI Overview citations, and page-level content gaps.",inputSchema:_t,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=[],r=$(e.domain??e.url??null),i=await t.client.post("/harvest/sync",{query:e.keyword,location:e.location,serpOnly:!0,includeAllSerpFeatures:!0,maxQuestions:1,format:"json"},62e4);await t.artifacts.writeJson("SERP raw JSON","raw/serp.json",i);let s=null;if(e.includePaa)try{s=await t.client.post("/harvest/sync",{query:e.keyword,location:e.location,maxQuestions:e.maxQuestions,includeAllSerpFeatures:!0,format:"json"},735e3),await t.artifacts.writeJson("PAA raw JSON","raw/paa.json",s)}catch(m){n.push(`PAA evidence unavailable: ${m instanceof Error?m.message:String(m)}`)}let a=xt(i,r).slice(0,e.maxResults),l=(i.organicResults??[]).slice(0,e.maxResults),c=e.url?{position:0,title:"Target page",url:e.url,domain:A(e.url)}:l.find(m=>r&&($(m.domain??"")??A(m.url))===r),p=l.filter(m=>m.url&&m.url!==c?.url).filter(m=>!r||($(m.domain??"")??A(m.url))!==r).slice(0,e.extractTop),u=c?await de(t,[c],n,"Target"):[],w=e.extractTop>0?await de(t,p,n,"Competitor"):[],v=u[0]?.page??null,f=[...u.map(({source:m,page:x})=>te(x,{url:m.url??"",domain:$(m.domain??"")??A(m.url),position:m.position,title:m.title})),...w.map(({source:m,page:x})=>te(x,{url:m.url??"",domain:$(m.domain??"")??A(m.url),position:m.position,title:m.title}))],b=Ct(v,w),d=ee(s),k=(i.aiOverview?.citations??[]).map((m,x)=>({citation_position:x+1,citation_text:m.text??"",url:m.href??"",domain:A(m.href??""),is_target:r?A(m.href??"")===r:!1}));await t.artifacts.writeCsv("Organic results CSV","organic-results.csv",["position","title","url","domain","snippet","is_target"],a),await t.artifacts.writeCsv("Page comparison CSV","page-comparison.csv",["position","domain","url","serp_title","page_title","h1","meta_description","word_count","heading_count","schema_types"],f),await t.artifacts.writeCsv("Content gaps CSV","content-gaps.csv",["source_position","source_domain","source_url","heading_level","competitor_heading","target_coverage","terms"],b),await t.artifacts.writeCsv("PAA questions CSV","paa-questions.csv",["position","intent","question","answer_excerpt","source_title","source_domain","source_url"],d),await t.artifacts.writeCsv("AI Overview citations CSV","ai-overview-citations.csv",["citation_position","citation_text","url","domain","is_target"],k),await t.artifacts.writeJson("SERP comparison evidence","evidence.json",{input:e,serp:i,paa:s,organic:a,pageRows:f,gaps:b,questions:d,aiRows:k,warnings:n}),await t.artifacts.writeText("Writer brief","brief.md",[`# SERP Comparison Brief: ${e.keyword}`,"",`Target: ${r??e.url??"not specified"}`,`Location: ${e.location??"not specified"}`,"","## Recommended Actions","- Use `content-gaps.csv` to decide which missing sections deserve coverage.","- Use `paa-questions.csv` for FAQ and answer-block candidates.","- Use `ai-overview-citations.csv` to see whether the target is cited in AI Overview evidence.","- Treat extracted page headings as evidence, not a complete semantic analysis."].join(`
+`));let g=`${a.length} organic results, ${f.length} extracted pages, ${b.length} heading gaps, ${d.length} PAA questions.`,o=await t.artifacts.writeHtml("HTML report","report.html",P({title:"SERP Comparison",subtitle:`${e.keyword}${e.location?` \xB7 ${e.location}`:""}`,summary:g,warnings:n,tables:[{title:"Organic Results",columns:["position","title","domain","is_target"],rows:a},{title:"Page Comparison",columns:["position","domain","h1","word_count","heading_count","schema_types"],rows:f},{title:"Content Gaps",columns:["source_position","source_domain","competitor_heading","target_coverage","terms"],rows:b.slice(0,80)},{title:"PAA Questions",columns:["position","intent","question","source_domain"],rows:d.slice(0,80)}]})),h=n.length?"partial":"succeeded",R={organic:a.length,pages:f.length,gaps:b.length,questions:d.length,aiCitations:k.length};return await t.artifacts.writeManifest(h,R,n,[]),{title:"SERP Comparison",summary:g,status:h,counts:R,warnings:n,errors:[],reportPath:o}}},Fe={id:"paa-expansion-brief",title:"PAA Expansion Brief",description:"Expand People Also Ask questions into an evidence-backed writer brief, section map, and source table.",inputSchema:bt,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=[],r=await t.client.post("/harvest/sync",{query:e.keyword,location:e.location,maxQuestions:e.maxQuestions,depth:e.depth,format:"json"},735e3);await t.artifacts.writeJson("PAA raw JSON","raw/paa.json",r);let i=ee(r),s=He(i),a=new Map;for(let w of i){let v=a.get(String(w.intent))??[];v.push(w),a.set(String(w.intent),v)}let l=[...a.entries()].map(([w,v])=>({recommended_section:w,question_count:v.length,sample_questions:v.slice(0,5).map(f=>f.question).join(" | "),source_domains:[...new Set(v.map(f=>f.source_domain).filter(Boolean))].slice(0,5).join("; "),terms:me(v.map(f=>`${f.question} ${f.answer_excerpt}`).join(" "),10)})).sort((w,v)=>Number(v.question_count)-Number(w.question_count));await t.artifacts.writeCsv("PAA questions CSV","paa-questions.csv",["position","intent","question","answer_excerpt","source_title","source_domain","source_url"],i),await t.artifacts.writeCsv("Source domains CSV","source-domains.csv",["domain","question_mentions","source_url_count","top_intents"],s),await t.artifacts.writeCsv("Section map CSV","section-map.csv",["recommended_section","question_count","sample_questions","source_domains","terms"],l),await t.artifacts.writeJson("PAA brief evidence","evidence.json",{input:e,paa:r,questions:i,sourceRows:s,sectionRows:l}),await t.artifacts.writeText("Writer brief","writer-brief.md",[`# PAA Expansion Brief: ${e.keyword}`,"",`Location: ${e.location??"not specified"}`,"","## Suggested Page Structure",...l.map(w=>`- ${w.recommended_section}: answer ${w.question_count} related question(s). Sample: ${w.sample_questions}`),"","## Writing Rules","- Answer the highest-frequency question in the first 60 words of each section.","- Use exact customer question language from `paa-questions.csv` for H2/H3 candidates.","- Use `source-domains.csv` to identify which source types Google is already rewarding.","- Do not invent citations; cite only rows that have a source URL."].join(`
+`));let c=`${i.length} PAA questions grouped into ${l.length} writing sections.`,p=await t.artifacts.writeHtml("HTML report","report.html",P({title:"PAA Expansion Brief",subtitle:`${e.keyword}${e.location?` \xB7 ${e.location}`:""}`,summary:c,warnings:n,tables:[{title:"Section Map",columns:["recommended_section","question_count","sample_questions","source_domains","terms"],rows:l},{title:"Questions",columns:["position","intent","question","source_domain"],rows:i.slice(0,120)},{title:"Source Domains",columns:["domain","question_mentions","source_url_count","top_intents"],rows:s}]})),u={questions:i.length,sections:l.length,sourceDomains:s.length};return await t.artifacts.writeManifest("succeeded",u,n,[]),{title:"PAA Expansion Brief",summary:c,status:"succeeded",counts:u,warnings:n,errors:[],reportPath:p}}},Ge={id:"ai-overview-language",title:"AI Overview Language Brief",description:"Turn AI Overview, citation, PAA, and ranking-page evidence into answer-block and citation-hook guidance.",inputSchema:kt,async run(e,t){await t.artifacts.writeManifest("running",{},[],[]);let n=[],r=$(e.domain??e.url??null),i=await t.client.post("/harvest/sync",{query:e.keyword,location:e.location,serpOnly:!0,includeAiOverview:!0,maxQuestions:1,format:"json"},62e4);await t.artifacts.writeJson("SERP raw JSON","raw/serp.json",i);let s=null;try{s=await t.client.post("/harvest/sync",{query:e.keyword,location:e.location,maxQuestions:e.maxQuestions,includeAiOverview:!0,format:"json"},735e3),await t.artifacts.writeJson("PAA raw JSON","raw/paa.json",s)}catch(o){n.push(`PAA evidence unavailable: ${o instanceof Error?o.message:String(o)}`)}let a=(i.aiOverview?.citations??[]).map((o,h)=>{let R=A(o.href??"");return{citation_position:h+1,citation_text:o.text??"",url:o.href??"",domain:R,is_target:r?R===r:!1}}),c=Be(i.aiOverview?.text).map((o,h)=>({position:h+1,claim_type:Ne(o),sentence:o,reusable_pattern:o.length>140?`${o.slice(0,140)}...`:o})),p=ee(s),u=(i.aiOverview?.citations??[]).filter(o=>o.href).map((o,h)=>({position:h+1,title:o.text,url:o.href,domain:A(o.href)})).slice(0,e.extractTop),v=(e.extractTop>0?await de(t,u,n,"Citation"):[]).map(({source:o,page:h})=>te(h,{url:o.url??"",domain:$(o.domain??"")??A(o.url),position:o.position,title:o.title})),f=[{block:"direct_answer",guidance:"Open with a 40-70 word answer that directly resolves the query before adding context.",evidence_basis:p[0]?.question??e.keyword},{block:"criteria_or_steps",guidance:"List the criteria, steps, or decision factors Google is already compressing into AI Overview language.",evidence_basis:c.filter(o=>["criteria","process"].includes(String(o.claim_type))).map(o=>o.sentence).slice(0,3).join(" | ")},{block:"citation_hook",guidance:"Add source-worthy details competitors can cite: definitions, numbers, examples, process details, and named entity relationships.",evidence_basis:a.map(o=>`${o.domain}: ${o.citation_text}`).slice(0,5).join(" | ")},{block:"faq_followups",guidance:"Use PAA phrasing for follow-up sections so the page answers adjacent questions in Google language.",evidence_basis:p.slice(0,5).map(o=>o.question).join(" | ")}];await t.artifacts.writeCsv("AI Overview citations CSV","ai-overview-citations.csv",["citation_position","citation_text","url","domain","is_target"],a),await t.artifacts.writeCsv("AI Overview claim patterns CSV","claim-patterns.csv",["position","claim_type","sentence","reusable_pattern"],c),await t.artifacts.writeCsv("Language guidance CSV","language-guidance.csv",["block","guidance","evidence_basis"],f),await t.artifacts.writeCsv("PAA questions CSV","paa-questions.csv",["position","intent","question","answer_excerpt","source_title","source_domain","source_url"],p),await t.artifacts.writeCsv("Extracted citation pages CSV","citation-pages.csv",["position","domain","url","serp_title","page_title","h1","meta_description","word_count","heading_count","schema_types"],v),await t.artifacts.writeJson("AI Overview language evidence","evidence.json",{input:e,serp:i,paa:s,citations:a,claimRows:c,languageRows:f,extractedRows:v,warnings:n}),await t.artifacts.writeText("Answer block template","answer-block-template.md",[`# AI Overview Language Brief: ${e.keyword}`,"",`AI Overview detected: ${i.aiOverview?.detected?"yes":"no"}`,`Target cited: ${r?a.some(o=>o.is_target)?"yes":"no":"target not specified"}`,"","## Direct Answer Block","Write one compact answer block that starts with the answer, not background. Keep it clear enough that Google could lift it as a standalone summary.","","## Suggested Follow-Up Blocks",...f.map(o=>`- ${o.block}: ${o.guidance}`),"","## Evidence to Mirror",...c.slice(0,8).map(o=>`- ${o.claim_type}: ${o.sentence}`),"","## Citation Hooks",...a.slice(0,8).map(o=>`- ${o.domain}: ${o.citation_text}`)].join(`
+`));let b=`${a.length} AI Overview citations, ${c.length} claim patterns, ${p.length} PAA questions, ${v.length} citation pages extracted.`,d=await t.artifacts.writeHtml("HTML report","report.html",P({title:"AI Overview Language Brief",subtitle:`${e.keyword}${e.location?` \xB7 ${e.location}`:""}`,summary:b,warnings:n,tables:[{title:"Language Guidance",columns:["block","guidance","evidence_basis"],rows:f},{title:"AI Overview Citations",columns:["citation_position","citation_text","domain","is_target"],rows:a},{title:"Claim Patterns",columns:["position","claim_type","sentence"],rows:c},{title:"PAA Follow-Ups",columns:["position","intent","question","source_domain"],rows:p.slice(0,60)}]})),k=n.length||!i.aiOverview?.detected?"partial":"succeeded",g={citations:a.length,claimPatterns:c.length,questions:p.length,extractedCitationPages:v.length};return await t.artifacts.writeManifest(k,g,n,[]),{title:"AI Overview Language Brief",summary:b,status:k,counts:g,warnings:n,errors:[],reportPath:d}}};var ge=[Me,Oe,We,Ue,ze,Ve,Fe,Ge];function br(){return ge.map(({id:e,title:t,description:n})=>({id:e,title:t,description:n}))}function re(e){let t=ge.find(n=>n.id===e);if(!t)throw new Error(`Unknown workflow "${e}". Available: ${ge.map(n=>n.id).join(", ")}`);return t}function kr(e){return re(e).steps?.length??1}function Rr(e){let t=re(e);return!!(t.steps&&t.steps.length>0)}function Qe(e){let t=e.apiKey?.trim()||process.env.MCP_SCRAPER_API_KEY?.trim();if(!t)throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");return t}function Ke(e){return e.apiUrl?.trim()||process.env.MCP_SCRAPER_API_URL?.trim()||"https://mcpscraper.dev"}async function Sr(e,t,n={}){let r=re(e),i=r.inputSchema.parse(t),s=Qe(n),a=Ke(n),l=await V.create(r.id,r.title,i,n.outputDir,n.runId),c=new F(a,s,n.fetchImpl,n.headers),p={runId:l.runId,startedAt:l.startedAt,client:c,artifacts:l,signal:n.signal};try{if(r.steps&&r.steps.length>0){let u=r.createState?r.createState(i):{},w=null;for(let v of r.steps){let f=await v.run({input:i,state:u,ctx:p});u=f.state,f.summary&&(w=f.summary)}if(!w)throw new Error(`Workflow "${e}" produced no terminal step summary`);return w}if(r.run)return await r.run(i,p);throw new Error(`Workflow "${e}" has neither steps nor a run() implementation`)}catch(u){let w=u instanceof Error?u.message:String(u);throw await l.writeText("Failure","summary.md",`# ${r.title}
+
+${w}
+`),await l.writeManifest("failed",{},[],[w]),u}}async function xr(e,t,n){let r=re(e),i=r.steps;if(!i||i.length===0)throw new Error(`Workflow "${e}" does not support stepwise execution`);if(n.stepIndex<0||n.stepIndex>=i.length)throw new Error(`Step index ${n.stepIndex} is out of range for "${e}" (${i.length} steps)`);let s=n.options??{},a=r.inputSchema.parse(t),l=Qe(s),c=Ke(s),p=await V.create(r.id,r.title,a,s.outputDir,s.runId),u=new F(c,l,s.fetchImpl,s.headers),w={runId:p.runId,startedAt:p.startedAt,client:u,artifacts:p,signal:s.signal},v=n.state??(r.createState?r.createState(a):{}),f=i[n.stepIndex],b=await f.run({input:a,state:v,ctx:w}),d=await Promise.all(p.artifacts.map(async k=>({...k,content:await At(k.path,"utf8").catch(()=>"")})));return{runId:p.runId,workflowId:r.id,title:r.title,stepIndex:n.stepIndex,stepId:f.id,stepTitle:f.title,totalSteps:i.length,isLast:n.stepIndex===i.length-1,output:b.output,state:b.state,warnings:b.warnings??[],summary:b.summary??null,artifacts:d}}export{xe as a,ae as b,$e as c,Ee as d,it as e,br as f,re as g,kr as h,Rr as i,Sr as j,xr as k};

package/dist/chunk-XPZVJIZ2.js

@@ -0,0 +1,100 @@
+import{s as r}from"./chunk-GMWKPIYX.js";import{createHash as b,randomUUID as O}from"crypto";var p="mcp-scraper/lead-list-enrichment.requested",A=new Set(["complete","partial","empty","failed","cancelled"]),U=new Set(["complete","failed","cancelled"]),T=10,g=360,S=60,h=null,m=null;function o(e,t,s){let n=e.trim();if(!n||n.length>s)throw new Error(`${t} is required and must be ${s} characters or fewer`);return n}function d(e,t){if(!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer`);return e}function u(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function R(e){let t=new WeakSet,s=n=>{if(n===null||typeof n=="string"||typeof n=="boolean")return n;if(typeof n=="number"){if(!Number.isFinite(n))throw new Error("canonical fingerprint input must contain finite numbers");return Object.is(n,-0)?0:n}if(Array.isArray(n))return n.map(s);if(typeof n=="object"){if(t.has(n))throw new Error("canonical fingerprint input must not be cyclic");t.add(n);let i={};for(let _ of Object.keys(n).sort()){let c=n[_];c!==void 0&&(i[_]=s(c))}return t.delete(n),i}throw new Error(`canonical fingerprint input contains unsupported ${typeof n}`)};return JSON.stringify(s(e))}function y(e){return b("sha256").update(R(e)).digest("hex")}function I(e){return b("sha256").update(R(e)).digest("hex")}function j(){return{mapsAttempts:0,pageAttempts:0,pageSuccesses:0,serpSearches:0}}function E(e){let t=u(e.input_json);if(!t)throw new Error(`lead list enrichment job ${String(e.id??"")} has invalid durable input`);return{id:String(e.id),userId:Number(e.user_id),idempotencyKey:String(e.idempotency_key),requestFingerprint:String(e.request_fingerprint),input:t,status:String(e.status),totalRows:Number(e.total_rows??0),doneRows:Number(e.done_rows??0),failedRows:Number(e.failed_rows??0),cancelledRows:Number(e.cancelled_rows??0),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},concurrencyLockId:e.concurrency_lock_id==null?null:String(e.concurrency_lock_id),debitKey:String(e.debit_key),holdMc:Number(e.hold_mc??0),billedMc:e.billed_mc==null?null:Number(e.billed_mc),result:u(e.result_json),csvArtifact:u(e.csv_artifact_json),xlsxArtifact:u(e.xlsx_artifact_json),warnings:u(e.warnings_json)??[],cancelRequestedAt:e.cancel_requested_at==null?null:String(e.cancel_requested_at),errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:e.retryable==null?null:Number(e.retryable)===1,createdAt:String(e.created_at??""),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function w(e){return{jobId:String(e.job_id),ordinal:Number(e.ordinal),inputDigest:String(e.input_digest),status:String(e.status),result:u(e.result_json),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:Number(e.retryable??0)===1,durationMs:Number(e.duration_ms??0),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function M(e){return{id:String(e.id),jobId:String(e.job_id),eventName:p,status:String(e.status),attempts:Number(e.attempts??0),claimToken:e.locked_by==null?null:String(e.locked_by),lockedUntil:e.locked_until==null?null:String(e.locked_until),nextAttemptAt:e.next_attempt_at==null?null:String(e.next_attempt_at),lastError:e.last_error==null?null:String(e.last_error),createdAt:String(e.created_at??""),dispatchedAt:e.dispatched_at==null?null:String(e.dispatched_at),updatedAt:String(e.updated_at??"")}}async function a(){let e=r();return h!==e&&(h=e,m=null),m||(m=(async()=>{await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_jobs (
+      id TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id), idempotency_key TEXT NOT NULL,
+      request_fingerprint TEXT NOT NULL, input_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'initializing',
+      total_rows INTEGER NOT NULL DEFAULT 0, done_rows INTEGER NOT NULL DEFAULT 0,
+      failed_rows INTEGER NOT NULL DEFAULT 0, cancelled_rows INTEGER NOT NULL DEFAULT 0,
+      maps_attempts INTEGER NOT NULL DEFAULT 0, page_attempts INTEGER NOT NULL DEFAULT 0,
+      page_successes INTEGER NOT NULL DEFAULT 0, serp_searches INTEGER NOT NULL DEFAULT 0,
+      concurrency_lock_id TEXT, debit_key TEXT NOT NULL,
+      hold_mc INTEGER NOT NULL DEFAULT 0, billed_mc INTEGER, result_json TEXT,
+      csv_artifact_json TEXT, xlsx_artifact_json TEXT, warnings_json TEXT NOT NULL DEFAULT '[]',
+      cancel_requested_at TEXT, error_code TEXT, error_message TEXT, retryable INTEGER,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')), started_at TEXT, completed_at TEXT,
+      updated_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE(user_id, idempotency_key)
+    )`);try{await e.execute("ALTER TABLE lead_list_enrichment_jobs ADD COLUMN serp_searches INTEGER NOT NULL DEFAULT 0")}catch{}await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_jobs_user_created ON lead_list_enrichment_jobs(user_id, created_at DESC)"),await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_jobs_status ON lead_list_enrichment_jobs(status, updated_at)"),await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_rows (
+      job_id TEXT NOT NULL REFERENCES lead_list_enrichment_jobs(id) ON DELETE CASCADE,
+      ordinal INTEGER NOT NULL, input_digest TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending',
+      result_json TEXT, maps_attempts INTEGER NOT NULL DEFAULT 0, page_attempts INTEGER NOT NULL DEFAULT 0,
+      page_successes INTEGER NOT NULL DEFAULT 0, serp_searches INTEGER NOT NULL DEFAULT 0,
+      error_code TEXT, error_message TEXT,
+      retryable INTEGER NOT NULL DEFAULT 0, duration_ms INTEGER NOT NULL DEFAULT 0,
+      started_at TEXT, completed_at TEXT, updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+      PRIMARY KEY(job_id, ordinal)
+    )`);try{await e.execute("ALTER TABLE lead_list_enrichment_rows ADD COLUMN serp_searches INTEGER NOT NULL DEFAULT 0")}catch{}await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_rows_job_status ON lead_list_enrichment_rows(job_id, status, ordinal)"),await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_outbox (
+      id TEXT PRIMARY KEY, job_id TEXT NOT NULL UNIQUE REFERENCES lead_list_enrichment_jobs(id) ON DELETE CASCADE,
+      event_name TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'held', attempts INTEGER NOT NULL DEFAULT 0,
+      locked_by TEXT, locked_until TEXT, next_attempt_at TEXT, last_error TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')), dispatched_at TEXT,
+      updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )`),await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_outbox_dispatch ON lead_list_enrichment_outbox(status, next_attempt_at, locked_until, created_at)")})()),m}async function l(e,t){await a();let s=t==null?await r().execute({sql:"SELECT * FROM lead_list_enrichment_jobs WHERE id = ? LIMIT 1",args:[e]}):await r().execute({sql:"SELECT * FROM lead_list_enrichment_jobs WHERE id = ? AND user_id = ? LIMIT 1",args:[e,t]});return s.rows[0]?E(s.rows[0]):null}async function C(e,t){await a();let s=await r().execute({sql:"SELECT * FROM lead_list_enrichment_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[e,t]});return s.rows[0]?E(s.rows[0]):null}async function F(e){await a();let t=o(e.id,"job id",200),s=o(e.idempotencyKey,"idempotency key",200),n=e.requestFingerprint??y(e.input);o(n,"request fingerprint",200);let i=o(e.debitKey,"debit key",500),_=d(e.holdMc,"holdMc");if(e.input.rows.length<1||e.input.rows.length>250)throw new Error("lead list job must contain 1 to 250 rows");let c=[{sql:`INSERT OR IGNORE INTO lead_list_enrichment_jobs (
+      id,user_id,idempotency_key,request_fingerprint,input_json,total_rows,concurrency_lock_id,debit_key,hold_mc
+    ) VALUES (?,?,?,?,?,?,?,?,?)`,args:[t,e.userId,s,n,JSON.stringify(e.input),e.input.rows.length,e.concurrencyLockId??null,i,_]},{sql:`INSERT OR IGNORE INTO lead_list_enrichment_outbox(id,job_id,event_name,status)
+      SELECT 'lead-list-enrichment:' || id,id,?,'held' FROM lead_list_enrichment_jobs
+      WHERE user_id=? AND idempotency_key=? AND request_fingerprint=?`,args:[p,e.userId,s,n]}];e.input.rows.forEach((D,x)=>c.push({sql:`INSERT OR IGNORE INTO lead_list_enrichment_rows(job_id,ordinal,input_digest)
+      SELECT id,?,? FROM lead_list_enrichment_jobs
+      WHERE user_id=? AND idempotency_key=? AND request_fingerprint=?`,args:[x,I(D),e.userId,s,n]}));let f=await r().batch(c,"write"),L=await C(e.userId,s);if(!L)throw new Error("lead list enrichment job could not be created");return{job:L,created:Number(f[0]?.rowsAffected??0)>0,conflict:L.requestFingerprint!==n}}async function H(e,t){return await a(),await r().execute({sql:"UPDATE lead_list_enrichment_jobs SET concurrency_lock_id=?, updated_at=datetime('now') WHERE id=? AND status='initializing'",args:[t,e]}),l(e)}async function W(e){return await a(),await r().batch([{sql:"UPDATE lead_list_enrichment_jobs SET status='queued',updated_at=datetime('now') WHERE id=? AND status='initializing'",args:[e]},{sql:`UPDATE lead_list_enrichment_outbox SET status='pending',next_attempt_at=NULL,updated_at=datetime('now')
+      WHERE job_id=? AND status='held' AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='queued')`,args:[e,e]}],"write"),l(e)}async function X(e){await a(),await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status='running',started_at=COALESCE(started_at,datetime('now')),
+      updated_at=datetime('now') WHERE id=? AND status IN ('queued','running') AND cancel_requested_at IS NULL`,args:[e]});let t=await l(e);return t?.status==="running"?t:null}async function P(e,t){return await a(),d(t,"row ordinal"),await r().execute({sql:`UPDATE lead_list_enrichment_rows SET status='running',started_at=COALESCE(started_at,datetime('now')),
+      updated_at=datetime('now') WHERE job_id=? AND ordinal=? AND status IN ('pending','running')
+      AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='running' AND cancel_requested_at IS NULL)`,args:[e,t,e]}),N(e,t)}async function N(e,t){await a();let s=await r().execute({sql:"SELECT * FROM lead_list_enrichment_rows WHERE job_id=? AND ordinal=? LIMIT 1",args:[e,t]});return s.rows[0]?w(s.rows[0]):null}async function J(e){return await a(),(await r().execute({sql:"SELECT * FROM lead_list_enrichment_rows WHERE job_id=? ORDER BY ordinal ASC",args:[e]})).rows.map(s=>w(s))}async function G(e){await a();let t=e.result.usage??j(),s=e.result.rowStatus==="failed"?"failed":"complete",n=await N(e.jobId,e.ordinal);return n&&U.has(n.status)?n:(await r().batch([{sql:`UPDATE lead_list_enrichment_rows SET status=?,result_json=?,maps_attempts=?,page_attempts=?,page_successes=?,serp_searches=?,
+      error_code=?,error_message=?,retryable=?,duration_ms=?,completed_at=datetime('now'),updated_at=datetime('now')
+      WHERE job_id=? AND ordinal=? AND status IN ('pending','running')
+      AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='running')`,args:[s,JSON.stringify(e.result),d(t.mapsAttempts,"mapsAttempts"),d(t.pageAttempts,"pageAttempts"),d(t.pageSuccesses,"pageSuccesses"),d(t.serpSearches,"serpSearches"),e.result.error?.code??null,e.result.error?.message??null,e.result.error?1:0,d(e.durationMs??0,"durationMs"),e.jobId,e.ordinal,e.jobId]},{sql:`UPDATE lead_list_enrichment_jobs SET
+      done_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status IN ('complete','failed','cancelled')),
+      failed_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='failed'),
+      cancelled_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='cancelled'),
+      maps_attempts=(SELECT COALESCE(SUM(maps_attempts),0) FROM lead_list_enrichment_rows WHERE job_id=?),
+      page_attempts=(SELECT COALESCE(SUM(page_attempts),0) FROM lead_list_enrichment_rows WHERE job_id=?),
+      page_successes=(SELECT COALESCE(SUM(page_successes),0) FROM lead_list_enrichment_rows WHERE job_id=?),
+      serp_searches=(SELECT COALESCE(SUM(serp_searches),0) FROM lead_list_enrichment_rows WHERE job_id=?),
+      updated_at=datetime('now') WHERE id=? AND status='running'`,args:[e.jobId,e.jobId,e.jobId,e.jobId,e.jobId,e.jobId,e.jobId,e.jobId]}],"write"),N(e.jobId,e.ordinal))}async function v(e,t){await a();let s=t==null?"":" AND user_id=?",n=[e];return t!=null&&n.push(t),await r().batch([{sql:`UPDATE lead_list_enrichment_jobs SET cancel_requested_at=COALESCE(cancel_requested_at,datetime('now')),
+      status=CASE WHEN status IN ('initializing','queued') THEN 'settling' ELSE status END,updated_at=datetime('now')
+      WHERE id=?${s} AND status IN ('initializing','queued','running')`,args:n},{sql:`UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
+      last_error='Lead list enrichment was cancelled before execution.',updated_at=datetime('now')
+      WHERE job_id=? AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND cancel_requested_at IS NOT NULL AND status='settling')`,args:[e,e]}],"write"),l(e,t)}async function $(e){await a();let t=await r().batch([{sql:`UPDATE lead_list_enrichment_rows SET status='cancelled',completed_at=datetime('now'),updated_at=datetime('now')
+      WHERE job_id=? AND status='pending' AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND cancel_requested_at IS NOT NULL)`,args:[e,e]},{sql:`UPDATE lead_list_enrichment_jobs SET
+      done_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status IN ('complete','failed','cancelled')),
+      failed_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='failed'),
+      cancelled_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='cancelled'),
+      status=CASE WHEN status='running' THEN 'settling' ELSE status END,updated_at=datetime('now')
+      WHERE id=? AND cancel_requested_at IS NOT NULL`,args:[e,e,e,e]}],"write");return Number(t[0]?.rowsAffected??0)}async function B(e){return await a(),await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status='settling',result_json=?,csv_artifact_json=?,xlsx_artifact_json=?,
+      warnings_json=?,updated_at=datetime('now') WHERE id=? AND status IN ('running','settling')`,args:[JSON.stringify(e.result),e.csvArtifact==null?null:JSON.stringify(e.csvArtifact),e.xlsxArtifact==null?null:JSON.stringify(e.xlsxArtifact),JSON.stringify(e.warnings??e.result.warnings),e.jobId]}),l(e.jobId)}async function K(e){await a();let t=await l(e.jobId);if(!t)throw new Error("lead list enrichment job not found");let s=d(e.billedMc,"billedMc");if(s>t.holdMc)throw new Error("billedMc cannot exceed the lead list enrichment hold");if(A.has(t.status)){if(t.status===e.status&&t.billedMc===s)return t;throw new Error("lead list enrichment job already has a conflicting terminal result")}await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status=?,billed_mc=?,result_json=COALESCE(?,result_json),
+      csv_artifact_json=CASE WHEN ?=1 THEN ? ELSE csv_artifact_json END,
+      xlsx_artifact_json=CASE WHEN ?=1 THEN ? ELSE xlsx_artifact_json END,
+      warnings_json=CASE WHEN ?=1 THEN ? ELSE warnings_json END,error_code=NULL,error_message=NULL,retryable=NULL,
+      completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status NOT IN ('complete','partial','empty','failed','cancelled')`,args:[e.status,s,e.result===void 0?null:JSON.stringify(e.result),e.csvArtifact===void 0?0:1,e.csvArtifact==null?null:JSON.stringify(e.csvArtifact),e.xlsxArtifact===void 0?0:1,e.xlsxArtifact==null?null:JSON.stringify(e.xlsxArtifact),e.warnings===void 0?0:1,JSON.stringify(e.warnings??[]),e.jobId]});let n=await l(e.jobId);if(!n||n.status!==e.status||n.billedMc!==s)throw new Error("lead list enrichment job completion conflicted");return n}async function z(e){await a();let t=await l(e.jobId);if(!t)throw new Error("lead list enrichment job not found");if(t.status!=="failed"&&A.has(t.status))throw new Error("completed lead list enrichment job cannot be failed");if(t.status==="failed")return t;let s=e.billedMc==null?null:d(e.billedMc,"billedMc");if(s!=null&&s>t.holdMc)throw new Error("billedMc cannot exceed the lead list enrichment hold");await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status='failed',billed_mc=COALESCE(?,billed_mc),error_code=?,error_message=?,retryable=?,
+      completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status NOT IN ('complete','partial','empty','failed','cancelled')`,args:[s,o(e.errorCode,"error code",200),o(e.errorMessage,"error message",2e3),e.retryable?1:0,e.jobId]});let n=await l(e.jobId);if(!n||n.status!=="failed")throw new Error("lead list enrichment job failure conflicted");return n}async function Y(e,t){await a();let s=await l(e);if(!s)throw new Error("lead list enrichment job not found");let n=d(t,"billedMc");if(n>s.holdMc)throw new Error("billedMc cannot exceed the lead list enrichment hold");await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET billed_mc=COALESCE(billed_mc,?),updated_at=datetime('now')
+      WHERE id=? AND status IN ('settling','failed','cancelled')`,args:[n,e]});let i=await l(e);if(!i||i.billedMc!==n)throw new Error("lead list enrichment settlement conflicts with durable amount");return i}async function Q(e=25){return await a(),(await r().execute({sql:`SELECT * FROM lead_list_enrichment_jobs WHERE status='settling' OR (status IN ('failed','cancelled') AND billed_mc IS NULL)
+      ORDER BY updated_at ASC LIMIT ?`,args:[Math.max(1,Math.min(100,Math.trunc(e)))]})).rows.map(s=>E(s))}async function V(e=25,t=T,s=g){return await a(),(await r().execute({sql:`SELECT j.* FROM lead_list_enrichment_jobs j JOIN lead_list_enrichment_outbox o ON o.job_id=j.id
+      WHERE j.status IN ('initializing','queued') AND j.billed_mc IS NULL AND (o.attempts>=? OR j.created_at<=datetime('now',?))
+      AND (o.status IN ('held','pending','failed','dispatched') OR (o.status='dispatching' AND o.locked_until<=datetime('now')))
+      ORDER BY j.updated_at ASC LIMIT ?`,args:[Math.max(1,Math.trunc(t)),`-${Math.max(15,Math.trunc(s))} minutes`,Math.max(1,Math.min(100,Math.trunc(e)))]})).rows.map(i=>E(i))}async function Z(e,t=T,s=g){await a();let n=await r().batch([{sql:`UPDATE lead_list_enrichment_jobs SET status='failed',billed_mc=CASE
+      WHEN EXISTS(SELECT 1 FROM billing_debits d WHERE d.idempotency_key=lead_list_enrichment_jobs.debit_key
+        AND d.user_id=lead_list_enrichment_jobs.user_id AND d.status IN ('applied','settled')) THEN NULL ELSE 0 END,
+      error_code='lead_list_dispatch_expired',
+      error_message='Lead list enrichment did not start before its dispatch deadline; the unused hold will be refunded.',retryable=1,
+      completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status IN ('initializing','queued') AND billed_mc IS NULL
+      AND EXISTS(SELECT 1 FROM lead_list_enrichment_outbox o WHERE o.job_id=lead_list_enrichment_jobs.id AND
+      (o.attempts>=? OR lead_list_enrichment_jobs.created_at<=datetime('now',?)))`,args:[e,Math.max(1,Math.trunc(t)),`-${Math.max(15,Math.trunc(s))} minutes`]},{sql:`UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
+      last_error='Lead list enrichment dispatch deadline exhausted.',updated_at=datetime('now') WHERE job_id=?
+      AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND error_code='lead_list_dispatch_expired')`,args:[e,e]}],"write");return Number(n[0]?.rowsAffected??0)===1?l(e):null}async function ee(e=25,t=S){return await a(),(await r().execute({sql:`SELECT * FROM lead_list_enrichment_jobs WHERE status='running' AND billed_mc IS NULL
+    AND updated_at<=datetime('now',?) ORDER BY updated_at ASC LIMIT ?`,args:[`-${Math.max(15,Math.trunc(t))} minutes`,Math.max(1,Math.min(100,Math.trunc(e)))]})).rows.map(n=>E(n))}async function te(e,t=S){await a();let s=await r().batch([{sql:`UPDATE lead_list_enrichment_jobs SET
+      status=CASE WHEN cancel_requested_at IS NULL THEN 'failed' ELSE 'settling' END,
+      error_code=CASE WHEN cancel_requested_at IS NULL THEN 'lead_list_worker_stale' ELSE NULL END,
+      error_message=CASE WHEN cancel_requested_at IS NULL
+        THEN 'Lead list enrichment stopped without a heartbeat; completed row checkpoints were preserved.' ELSE NULL END,
+      retryable=CASE WHEN cancel_requested_at IS NULL THEN 1 ELSE NULL END,
+      completed_at=CASE WHEN cancel_requested_at IS NULL THEN datetime('now') ELSE NULL END,updated_at=datetime('now')
+      WHERE id=? AND status='running' AND billed_mc IS NULL
+      AND updated_at<=datetime('now',?)`,args:[e,`-${Math.max(15,Math.trunc(t))} minutes`]},{sql:`UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
+      last_error='Lead list enrichment worker heartbeat expired.',updated_at=datetime('now') WHERE job_id=?
+      AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND (error_code='lead_list_worker_stale'
+        OR (status='settling' AND cancel_requested_at IS NOT NULL)))`,args:[e,e]}],"write");return Number(s[0]?.rowsAffected??0)===1?l(e):null}async function ne(e){await a();let t=o(e.workerId,"outbox worker id",160),s=Math.max(1,Math.min(100,Math.trunc(e.limit??25))),n=Math.max(30,Math.min(3600,Math.trunc(e.leaseSeconds??120))),i=`${t}:${O()}`;return(await r().batch([{sql:`UPDATE lead_list_enrichment_outbox SET status='dispatching',attempts=attempts+1,locked_by=?,
+      locked_until=datetime('now',?),updated_at=datetime('now') WHERE id IN (SELECT id FROM lead_list_enrichment_outbox
+      WHERE attempts<${T} AND created_at>datetime('now','-${g} minutes')
+      AND (status='pending' OR (status='failed' AND (next_attempt_at IS NULL OR next_attempt_at<=datetime('now')))
+      OR (status='dispatching' AND locked_until<=datetime('now'))) ORDER BY created_at ASC LIMIT ?)`,args:[i,`+${n} seconds`,s]},{sql:"SELECT * FROM lead_list_enrichment_outbox WHERE locked_by=? AND status='dispatching' ORDER BY created_at ASC",args:[i]}],"write"))[1]?.rows.map(c=>M(c))??[]}async function se(e,t){await a();let s=await r().execute({sql:`UPDATE lead_list_enrichment_outbox SET status='dispatched',locked_by=NULL,
+    locked_until=NULL,next_attempt_at=NULL,last_error=NULL,dispatched_at=datetime('now'),updated_at=datetime('now')
+    WHERE id=? AND status='dispatching' AND locked_by=?`,args:[e,t]});return Number(s.rowsAffected??0)>0}async function re(e){await a();let t=Math.max(1,Math.min(86400,Math.trunc(e.retryAfterSeconds??30))),s=await r().execute({sql:`UPDATE lead_list_enrichment_outbox SET status='failed',locked_by=NULL,
+    locked_until=NULL,next_attempt_at=datetime('now',?),last_error=?,updated_at=datetime('now')
+    WHERE id=? AND status='dispatching' AND locked_by=?`,args:[`+${t} seconds`,String(e.error).slice(0,2e3),e.id,e.claimToken]});return Number(s.rowsAffected??0)>0}export{T as a,g as b,S as c,R as d,y as e,I as f,a as g,l as h,C as i,F as j,H as k,W as l,X as m,P as n,N as o,J as p,G as q,v as r,$ as s,B as t,K as u,z as v,Y as w,Q as x,V as y,Z as z,ee as A,te as B,ne as C,se as D,re as E};

package/dist/chunk-YQZGZBB4.js

@@ -0,0 +1 @@
+import{a as o,b as e,c as i,d as t,e as s}from"./chunk-QPWPR5XG.js";var a="SERP browser service is not configured";function E(n,r){return n instanceof i||n instanceof t||n instanceof o||n instanceof e||n instanceof s||/^401\s+Invalid or disabled API key\s*$/i.test(r.trim())||r===a}export{E as a};

package/dist/chunk-Z2QGQJS2.js

@@ -0,0 +1 @@
+var o="0.89.0";export{o as a};

package/dist/db-F2MX63GI.js

@@ -0,0 +1 @@
+import{$ as M,$a as Ma,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,W as H,Wa as Ha,X as I,Xa as Ia,Y as J,Ya as Ja,Z as K,Za as Ka,_ as L,_a as La,aa as N,ab as Na,ba as O,bb as Oa,ca as P,cb as Pa,da as Q,db as Qa,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-GMWKPIYX.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,kb as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,yb as SiteAuditJobRowSchema,zb as SiteAuditPhaseLogRowSchema,qa as advanceWorkflowRunStep,oa as appendWorkflowArtifacts,P as attachHarvestAttemptProviderSession,Oa as cancelJob,D as checkRateLimit,za as checkpointDurablePaaJob,ya as checkpointJob,xa as claimDurablePaaJob,qb as claimMonthlyFreeRefresh,wa as claimPendingJob,Qa as claimPendingKpoJob,o as claimStripeEvent,Ba as completeDurablePaaJob,Ea as completeExpiredDurablePaaJob,Ka as completeJob,Ta as completeKpoJob,La as completeRunningJob,p as completeStripeEvent,la as completeWorkflowRunRecord,j as consumeCode,F as consumePasswordResetToken,va as countActiveJobsForUser,ua as countActiveUsers,K as createJob,L as createJobWithId,Pa as createKpoJob,E as createPasswordResetToken,M as createRunningJob,C as createUser,ga as createWorkflowRun,$ as createWorkflowSchedule,nb as creditMc,ob as creditMcIdempotent,J as deactivateUser,rb as debitMc,sb as debitMcIdempotent,da as deleteWorkflowSchedule,lb as expireOldLots,Ca as failDurablePaaJob,Fa as failExpiredDurablePaaJob,Ma as failJob,Ua as failKpoJob,Na as failRunningJob,q as failStripeEvent,ma as failWorkflowRunRecord,Q as finalizeExpiredHarvestAttempts,R as finishHarvestAttempt,s as generateApiKey,vb as getBillingLedgerSummary,h as getClient,cb as getConnectedAccountBillingState,d as getDb,Bb as getInboxMessage,V as getJob,Ra as getKpoJob,Wa as getKpoPhaseLog,ub as getLedger,gb as getMemoryPlan,hb as getMemoryProvisioned,N as getOrCreateRunningJob,X as getPageSnapshot,l as getRefresh,w as getUserByApiKey,v as getUserByEmail,x as getUserById,Ya as getUserByStripeCustomerId,y as getUserMemoryCreds,B as getUserStats,sa as getWorkflowArtifact,ha as getWorkflowRun,ia as getWorkflowRunByIdempotencyKey,pa as getWorkflowRunStepState,aa as getWorkflowSchedule,t as hashPassword,Eb as insertInboxMessage,pb as ledgerExistsForOperation,wb as ledgerExistsForStripePI,Fb as listActiveUserIds,ea as listDueWorkflowSchedules,U as listHarvestAttempts,T as listHarvestAttemptsPendingProviderTelemetry,Ab as listInboxMessages,W as listJobs,Xa as listKpoJobs,Ga as listRecentTerminalDurablePaaJobs,Ia as listRecoverableDurablePaaJobs,_ as listRequestEvents,Ha as listStaleRunningDurablePaaJobs,I as listUsers,eb as listUsersDueForConnectedAccountBillingReconciliation,ra as listWorkflowArtifacts,ja as listWorkflowRuns,ba as listWorkflowSchedules,Va as logKpoPhaseComplete,Z as logRequestEvent,Cb as markInboxMessageOpened,Ja as markJobBillingSettled,ka as markWorkflowRunRunning,fa as markWorkflowScheduleRan,f as migrate,mb as migrateExistingBalancesToLots,ca as patchWorkflowSchedule,i as putCode,k as putRefresh,xb as reconcileBalanceMc,ta as recordWorkflowWebhookDelivery,g as registerClient,Aa as renewDurablePaaLease,na as replaceWorkflowArtifacts,Da as requeueExpiredDurablePaaJob,jb as resolveUserByMemorySubscription,H as revokeApiKey,m as revokeRefresh,G as rotateApiKey,n as rotateRefresh,$a as setConcurrencyPackQuantity,ab as setConcurrencySubId,db as setConnectedAccountBillingState,e as setDbForTesting,_a as setExtraConcurrencySlots,fb as setMemoryPlan,ib as setMemoryProvisioned,A as setPassword,Za as setStripeCustomerId,bb as setSubscriptionTier,z as setUserMemoryCreds,tb as settleDebitMcIdempotent,Db as softDeleteInboxMessage,O as startHarvestAttempt,r as stripeEventAlreadyProcessed,S as updateHarvestAttemptProviderTelemetry,Sa as updateKpoJobState,Y as upsertPageSnapshot,u as verifyPassword};

package/dist/extract-bundle-SNUIHM3J.js

@@ -0,0 +1,26 @@
+import{c as Je,h as ke,i as Ke,j as Qe,l as Ve}from"./chunk-TXENITMS.js";import{b as Ge,c as Ye,h as Xe,j as Ze}from"./chunk-CFI6CXIV.js";import"./chunk-YQZGZBB4.js";import"./chunk-QPWPR5XG.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import"./chunk-TMB56NCA.js";import"./chunk-MZN4U5BL.js";import"./chunk-HE45FFBU.js";import{s as qe}from"./chunk-GMWKPIYX.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as bt,readdirSync as xt,rmSync as kt,statSync as _t,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Pt,join as a}from"path";import{tmpdir as Mt}from"os";import{createGzip as It}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as Ct}from"yazl";import At from"p-limit";import{createHash as _e}from"crypto";var pt=500,gt=.9,ht=1e4,ft=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function yt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function wt(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function et(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function St(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(et).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=et(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
+
+`).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
+
+${C}`}),minimumPageCount:n,removedBlockSignatures:b.size,removedCharacters:w}}var Pe=class{parent;constructor(s){this.parent=Array.from({length:s},(n,c)=>c)}find(s){let n=this.parent[s];return n!==s&&(this.parent[s]=this.find(n)),this.parent[s]}union(s,n){let c=this.find(s),b=this.find(n);c!==b&&(this.parent[b]=c)}};async function tt(r,s={}){let n=r.filter(i=>i.bodyMarkdown.trim().length>0).slice(0,pt),c=Number.isFinite(s.threshold)?s.threshold:gt,b=Number.isFinite(s.maxPairs)?s.maxPairs:ht,w=Math.max(0,Math.min(1,c)),$=Math.max(1,Math.min(ft,Math.floor(b))),f=s.model??Ke(),M=s.dimensions??Qe(),I=s.embedTexts??Ve,A=St(n),C=[],_=new Map,T=[];for(let[i,p]of n.entries()){let y=p.contentHash||_e("sha256").update(p.bodyMarkdown).digest("hex"),k=_.get(y);k==null&&(k=C.length,_.set(y,k),C.push(A.texts[i])),T.push(k)}let j=[];for(let i=0;i<C.length;i+=64)j.push(...await I(C.slice(i,i+64)));if(j.length!==C.length)throw new Error(`Embedding provider returned ${j.length} vectors for ${C.length} unique page bodies.`);let X=T.map(i=>j[i]),le=new Pe(n.length),L=[],O=[];for(let i=0;i<n.length;i++)for(let p=i+1;p<n.length;p++){let y=yt(X[i],X[p]);L.push(y),!(y<w)&&(O.push({source:i,target:p,score:y}),le.union(i,p))}O.sort((i,p)=>p.score-i.score||n[i.source].url.localeCompare(n[p.source].url)||n[i.target].url.localeCompare(n[p.target].url));let v=[...L].sort((i,p)=>i-p),U=i=>v.length?ge(v[Math.floor((v.length-1)*i)],6):null,D=new Map;for(let i=0;i<n.length;i++){let p=le.find(i),y=D.get(p)??[];y.push(i),D.set(p,y)}let N=new Map,z=[...D.values()].filter(i=>i.length>1).sort((i,p)=>p.length-i.length||n[i[0]].url.localeCompare(n[p[0]].url)).map((i,p)=>{let y=`cluster-${String(p+1).padStart(3,"0")}`;for(let k of i)N.set(k,y);return{clusterId:y,pageCount:i.length,urls:i.map(k=>n[k].url).sort()}}),F=O.slice(0,$).map((i,p)=>{let y=n[i.source],k=n[i.target],q=ge(i.score,6);return{sourceUrl:y.url,sourceTitle:y.title,targetUrl:k.url,targetTitle:k.title,similarity:q,similarityPercent:ge(q*100,2),corpusPercentile:v.length<=1?100:ge((1-p/(v.length-1))*100,2),exactContentDuplicate:!!(y.contentHash&&y.contentHash===k.contentHash),sourceWordCount:y.wordCount,targetWordCount:k.wordCount,clusterId:N.get(i.source)??null}}),H=wt(n),ue=_e("sha256").update(JSON.stringify({corpusSha256:H,model:f,dimensions:M,threshold:w,maxPairs:$,boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures}})).digest("hex");return{generatedAt:new Date().toISOString(),provider:"jina",model:f,dimensions:M,threshold:w,requestedMaxPairs:$,comparedPages:n.length,possiblePairs:n.length*Math.max(0,n.length-1)/2,qualifyingPairs:O.length,returnedPairs:F.length,pairsTruncated:O.length>F.length,corpusSha256:H,analysisSha256:ue,scoreDistribution:{min:U(0),p25:U(.25),median:U(.5),p75:U(.75),p90:U(.9),max:U(1)},boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures,removedCharacters:A.removedCharacters},rows:F,clusters:z}}var Rt=25,Me=20,Ie=500,Tt=8,nt=10*1024*1024,rt=100*1024*1024,Ce=25e4;function oe(r){return r.split("#")[0].replace(/\/$/,"")}function st(r){let s=r.replace(/^www\./,"").toLowerCase(),n=s.split(".");return n.length<=2?s:n.slice(-2).join(".")}function vt(r,s){try{let n=new URL(r);return Pt(n.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`image-${s}`}catch{return`image-${s}`}}function jt(){let r=new Map;return s=>{let n=s.replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)||"page",c=r.get(n)??0;return r.set(n,c+1),c?`${n}-${c}`:n}}async function*Ae(r){let s=qe(),n=0;for(;;){let c=await s.execute({sql:`SELECT rowid AS cursor_id, page
+            FROM site_extract_pages
+            WHERE job_id = ? AND rowid > ?
+            ORDER BY rowid
+            LIMIT ${Rt}`,args:[r,n]});if(!c.rows.length)return;if(n=Number(c.rows.at(-1)?.cursor_id??n),!Number.isSafeInteger(n)||n<=0)throw new Error("site extract page cursor is invalid");yield c.rows.map(b=>JSON.parse(String(b.page)))}}function Et(r){let s=r==null?"":String(r);return/[",\r\n]/.test(s)?`"${s.replace(/"/g,'""')}"`:s}function $t(r){return{source_url:r.sourceUrl,source_title:r.sourceTitle,target_url:r.targetUrl,target_title:r.targetTitle,similarity:r.similarity,similarity_percent:r.similarityPercent,corpus_percentile:r.corpusPercentile,exact_content_duplicate:r.exactContentDuplicate,source_word_count:r.sourceWordCount,target_word_count:r.targetWordCount,cluster_id:r.clusterId}}async function rn(r,s={}){let n=a(Mt(),`extract-bundle-${r.id}-${Date.now()}`),c=r.options.downloadImages===!0;if(r.userId==null)throw new Error("site extract artifact owner is missing");let b=Xe(String(r.userId));ie(n,{recursive:!0}),c&&ie(a(n,"images"),{recursive:!0});let w=r.options.captureRenderedDom===!0,$=r.options.semanticSimilarity===!0;w&&ie(a(n,"rendered-dom"),{recursive:!0});try{let f=[],M=[],I=[],A=jt(),C=[],_=r.options.waybackTimeline,T=new Map;for await(let t of Ae(r.id))for(let e of t){T.set(oe(e.url),e.status),f.push({url:e.url,status:e.status,title:e.title,titleLength:e.titleLength,titlePixels:e.titlePixels,metaDescription:e.metaDescription,metaDescLength:e.metaDescLength,h1:e.h1,h1_2:e.h1_2,h2Count:e.h2Count,extractionStatus:e.extractionStatus,indexable:e.indexable,indexabilityReason:e.indexabilityReason,canonicalUrl:e.canonicalUrl?e.url:null,wordCount:e.wordCount,contentHash:e.contentHash,imagesMissingAlt:e.imagesMissingAlt,schemaTypes:e.schemaTypes.length>0?["present"]:[],outlinks:[]});let o=e.pageId??G("sha256").update(e.archivedUrl??e.url).digest("hex"),x=a(n,"pages",o);ie(x,{recursive:!0});let m=e.contentRef?await b(e.contentRef):null,d=m?.html??null,l=m?.markdown??e.bodyMarkdown??null;if($&&e.extractionStatus==="successful"&&l?.trim()&&M.push({url:e.url,title:e.title,bodyMarkdown:l,contentHash:e.contentHash,wordCount:e.wordCount}),w&&d!=null){let Ne=`rendered-dom/${A(e.url)}.html.txt`;g(a(n,Ne),d),I.push({url:e.url,path:Ne,bytes:Buffer.byteLength(d),truncated:e.renderedDomTruncated===!0,sanitized:e.renderedDomSanitized===!0})}let u=d==null?null:`pages/${o}/page.html`,h=l==null?null:`pages/${o}/page.md`,S=l==null||!e.archiveRequestedMonth?null:`pages/${e.archiveRequestedMonth}/${o}.md`;d!=null&&u&&g(a(n,u),d),l!=null&&h&&g(a(n,h),l),l!=null&&S&&(ie(a(n,"pages",e.archiveRequestedMonth),{recursive:!0}),g(a(n,S),l));let{bodyMarkdown:R,contentRef:te,discoveryLinks:ne,...dt}=e,re={version:"site-export-page.v1",...dt,pageId:o,content:{htmlPath:u,markdownPath:h,htmlBytes:d==null?0:Buffer.byteLength(d),markdownBytes:l==null?0:Buffer.byteLength(l),htmlSha256:d==null?null:G("sha256").update(d).digest("hex"),markdownSha256:l==null?null:G("sha256").update(l).digest("hex")}},Le=JSON.stringify(re,null,2),Oe=`pages/${o}/page.json`;g(a(n,Oe),Le),C.push({pageId:o,url:e.archivedUrl??e.url,jsonPath:Oe,htmlPath:u,markdownPath:h,legacyMarkdownPath:S,jsonSha256:G("sha256").update(Le).digest("hex"),htmlSha256:re.content.htmlSha256,markdownSha256:re.content.markdownSha256,htmlBytes:re.content.htmlBytes,markdownBytes:re.content.markdownBytes})}let j=It(),X=se(a(n,"links.jsonl.gz"));j.pipe(X);let le=async t=>{j.write(t)||await Y(j,"drain")},L=se(a(n,"schema.jsonl")),O=async t=>{L.write(t)||await Y(L,"drain")},v=se(a(n,"failures.jsonl")),U=async t=>{v.write(t)||await Y(v,"drain")},D=new Map,N=new Map,z=new Map,F=new Map,H=new Map,ue=new Set,i=0,p=0,y=0,k=0,q=0,he=!1,fe="";try{fe=st(new URL(r.startUrl).hostname)}catch{fe=""}let it=At(Tt),Re=[],Te=[],ve=[],Z=0,ce=0,ye=0,at=t=>ye+t>rt?!1:(ye+=t,!0);for await(let t of Ae(r.id))for(let e of t){if(e.schema?.length&&await O(JSON.stringify({url:e.url,schema:e.schema})+`
+`),e.extractionStatus==="failed"){let{failureCode:m,failureReason:d}=ke(e.failureCode,e.failureReason);await U(JSON.stringify({url:e.url,status:e.status,via:e.via,failureCode:m,failureReason:d,fetchAttempts:e.fetchAttempts??[]})+`
+`)}if(e.imageLinks?.length)for(let[m,d]of e.imageLinks.entries()){let l=G("sha256").update(`${e.url}\0${d}`).digest("hex"),u={imageId:l,sourcePage:e.url,sourceUrl:d,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Te.push(u),!c)continue;if(m>=Me){u.status="skipped_page_limit",u.failureCode="page_image_limit",u.failureReason=`Only the first ${Me} image candidates are downloaded per page.`;continue}if(Z>=Ie){u.status="skipped_site_limit",u.failureCode="site_image_limit",u.failureReason=`Only the first ${Ie} image candidates are downloaded per site.`;continue}Z++;let h=`${l}-${vt(d,m)}`,S=a(n,"images");Re.push(it(()=>Je(d,S,h,{expectedType:"image",maxBytes:nt,consumeBytes:at})).then(async R=>{let te=bt(R.savedPath),ne=await Ye({ownerId:String(r.userId),jobId:r.id,imageId:l,filename:h,contentType:R.mimeType??"application/octet-stream",content:te,sourceUrl:d,sourcePage:e.url});ve.push(ne),u.status="downloaded",u.path=`images/${h}`,u.artifactId=ne.key,u.mimeType=R.mimeType,u.bytes=R.sizeBytes,u.sha256=G("sha256").update(te).digest("hex")},R=>{ce++,u.status="failed",u.failureCode="image_download_failed",u.failureReason=(R instanceof Error?R.message:String(R)).replace(/\s+/g," ").slice(0,500)}))}let o=oe(e.url),x={int:0,ext:0};for(let m of e.outlinks??[]){let d=(m.rel??"").split(/\s+/).includes("nofollow"),l=oe(m.href),u=T.get(l)??null;if(await le(JSON.stringify({from:e.url,to:m.href,anchor:m.anchor,rel:m.rel,internal:m.internal,nofollow:d,targetStatus:u})+`
+`),k++,q>=Ce){he=!0;continue}if(q++,m.internal){if(x.int++,i++,D.has(l)||D.set(l,new Set),D.get(l).add(o),m.anchor){N.has(l)||N.set(l,new Map);let h=N.get(l);h.set(m.anchor,(h.get(m.anchor)??0)+1)}!d&&(T.get(l)===200||!T.has(l))&&(z.has(o)||z.set(o,new Set),z.get(o).add(l)),u!=null&&u>=400&&(y++,ue.add(e.url))}else{x.ext++;let h="";try{h=st(new URL(m.href).hostname)}catch{continue}if(!h||h===fe)continue;p++;let S=H.get(h)??{links:0,nofollow:0,pages:new Set};S.links++,d&&S.nofollow++,S.pages.add(e.url),H.set(h,S)}}F.set(e.url,x)}let ot=Promise.all([ae(X),ae(L),ae(v)]);j.end(),L.end(),v.end(),await ot,await Promise.all(Re);let W=new Map,we=oe(r.startUrl);W.set(we,0);let Se=[we];for(;Se.length>0;){let t=Se.shift(),e=W.get(t);for(let o of z.get(t)??[])W.has(o)||(W.set(o,e+1),Se.push(o))}let K=new Map;for(let t of f){let e=oe(t.url),o=D.get(e)??new Set,x=N.get(e),m=x?[...x.entries()].sort((l,u)=>u[1]-l[1]).slice(0,3).map(l=>l[0]):[],d=F.get(t.url)??{int:0,ext:0};K.set(t.url,{url:t.url,inlinks:o.size,uniqueInlinks:o.size,outlinksInternal:d.int,outlinksExternal:d.ext,crawlDepth:W.has(e)?W.get(e):null,orphan:o.size===0&&e!==we,topAnchors:m})}let je=He(f,K,ue),Q=We(r.startUrl,f,je,K),V=Ze(r),J={jobId:r.id,startUrl:r.startUrl,status:r.successfulUrls===0?"failed":r.failedUrls>0||r.remainingUrls>0||V.creditTruncated?"partial":"complete",...V,discovered:r.totalUrls,attempted:r.attemptedUrls,successful:r.successfulUrls,failed:r.failedUrls,remaining:r.remainingUrls,linkAnalysis:{truncated:he,analyzedEdges:q,streamedEdges:k,edgeCap:Ce},generatedAt:new Date().toISOString()};Q=["## Crawl delivery",`- Discovered: ${J.discovered}`,`- Attempted: ${J.attempted}`,`- Successful: ${J.successful}`,`- Failed: ${J.failed}`,`- Remaining: ${J.remaining}`,`- Requested page cap: ${V.requestedMaxPages}`,`- Funded page cap: ${V.effectiveMaxPages}`,`- Credit-truncated: ${V.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${he?` (capped at ${Ce})`:""}`,"",Q].join(`
+`),s.imageAudit&&(Q+=`
+
+${ze(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},$e=0,Ue=0;for(let t of me)$e+=t.inlinks,Ue+=t.outlinksInternal+t.outlinksExternal,t.inlinks===0?ee.zero++:t.inlinks<=2?ee.oneToTwo++:t.inlinks<=10?ee.threeToTen++:ee.elevenPlus++;let be=[...H.entries()].map(([t,e])=>({domain:t,links:e.links,nofollow:e.nofollow,pages:e.pages.size})).sort((t,e)=>e.links-t.links),xe={summary:{internal:{totalLinks:i,pages:f.length,orphans:me.filter(t=>t.orphan).length,brokenInternal:y,avgInlinks:f.length?Ee($e/f.length):0,avgOutlinks:f.length?Ee(Ue/f.length):0,distribution:ee,topByInlinks:[...me].sort((t,e)=>e.inlinks-t.inlinks).slice(0,20).map(t=>({url:t.url,inlinks:t.inlinks,outlinksInternal:t.outlinksInternal,outlinksExternal:t.outlinksExternal}))},external:{totalLinks:p,uniqueDomains:be.length,topDomains:be.slice(0,20)}},externalDomains:be},de=se(a(n,"pages.jsonl")),lt=new Map(C.map(t=>[t.pageId,t])),E=_?se(a(n,"capture-matrix.jsonl")):null,De=new Set,Be=(t,e)=>{let o=e;try{o=new URL(e).href}catch{}return`${t??""}\0${o}`};for await(let t of Ae(r.id))for(let e of t){let o=e.extractionStatus==="failed"?ke(e.failureCode,e.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:x,schema:m,outlinks:d,contentRef:l,...u}=e,h=K.get(e.url),S=e.pageId?lt.get(e.pageId):void 0,R={...u,...e.extractionStatus==="failed"?o:{},representations:S?{jsonPath:S.jsonPath,htmlPath:S.htmlPath,markdownPath:S.markdownPath,jsonSha256:S.jsonSha256,htmlSha256:S.htmlSha256,markdownSha256:S.markdownSha256,htmlBytes:S.htmlBytes,markdownBytes:S.markdownBytes}:null,inlinks:h?.inlinks??0,crawlDepth:h?.crawlDepth??null,orphan:h?.orphan??!1};if(de.write(JSON.stringify(R)+`
+`)||await Y(de,"drain"),E){let te=Be(e.archiveRequestedMonth,e.originalUrl??e.url);De.add(te);let ne={requestedMonth:e.archiveRequestedMonth??null,captureTimestamp:e.archiveTimestamp??null,originalUrl:e.originalUrl??e.url,archivedUrl:e.archivedUrl??null,status:e.extractionStatus==="successful"?"captured":"failed",httpStatus:e.status,contentHash:e.contentHash||null,wordCount:e.wordCount,failureCode:o.failureCode,failureReason:o.failureReason};E.write(JSON.stringify(ne)+`
+`)||await Y(E,"drain")}}if(E&&_?.timeline?.urls){let t=_.timeline,e=t.urls,o=(()=>{if(t.months?.length)return[...new Set(t.months)].sort();if(!t.from||!t.to)return[];let x=[],m=new Date(`${t.from}-01T00:00:00Z`),d=new Date(`${t.to}-01T00:00:00Z`);for(;m<=d&&x.length<60;)x.push(`${m.getUTCFullYear()}-${String(m.getUTCMonth()+1).padStart(2,"0")}`),m.setUTCMonth(m.getUTCMonth()+(t.intervalMonths??1));return x})();for(let x of o)for(let m of e)De.has(Be(x,m))||E.write(JSON.stringify({requestedMonth:x,captureTimestamp:null,originalUrl:m,archivedUrl:null,status:"missing",httpStatus:null,contentHash:null,wordCount:0,failureCode:"wayback_capture_missing",failureReason:"No successful HTML capture was available in the requested month."})+`
+`)||await Y(E,"drain")}let ut=ae(de),ct=E?ae(E):Promise.resolve();if(de.end(),E?.end(),await Promise.all([ut,ct]),w&&g(a(n,"rendered-dom","manifest.jsonl"),I.map(t=>JSON.stringify(t)).join(`
+`)),$){let t=await tt(M,{threshold:Number(r.options.similarityThreshold??void 0),maxPairs:Number(r.options.similarityMaxPairs??void 0),embedTexts:s.embedSimilarityTexts}),e=t.rows.map($t),o=e.length>0?Object.keys(e[0]):["source_url","source_title","target_url","target_title","similarity","similarity_percent","corpus_percentile","exact_content_duplicate","source_word_count","target_word_count","cluster_id"],x=[o.join(","),...e.map(u=>o.map(h=>Et(u[h]??null)).join(","))].join(`
+`)+`
+`,{rows:m,clusters:d,...l}=t;g(a(n,"similarity-table.csv"),x),g(a(n,"similarity.jsonl"),e.map(u=>JSON.stringify(u)).join(`
+`)),g(a(n,"similarity-table-schema.json"),JSON.stringify({tableNameSuggestion:`site_similarity_${new URL(r.startUrl).hostname.replace(/[^a-z0-9]+/gi,"_").replace(/^_+|_+$/g,"").toLowerCase()}`,defaultSort:[{column:"similarity",direction:"desc"}],columns:{source_url:"text",source_title:"text",target_url:"text",target_title:"text",similarity:"number",similarity_percent:"number",corpus_percentile:"number",exact_content_duplicate:"boolean",source_word_count:"integer",target_word_count:"integer",cluster_id:"text"}},null,2)),g(a(n,"similarity-summary.json"),JSON.stringify(l,null,2)),g(a(n,"content-clusters.json"),JSON.stringify(t.clusters,null,2)),Q+=["","## Rendered content similarity",`- Compared pages: ${t.comparedPages}`,`- Raw cosine threshold: ${t.threshold}`,`- Corpus score distribution: min ${t.scoreDistribution.min??"n/a"} / median ${t.scoreDistribution.median??"n/a"} / p90 ${t.scoreDistribution.p90??"n/a"} / max ${t.scoreDistribution.max??"n/a"}`,`- Qualifying pairs: ${t.qualifyingPairs}`,`- Returned table rows: ${t.returnedPairs}${t.pairsTruncated?" (capped)":""}`,`- Multi-page clusters: ${t.clusters.length}`,`- Embedding model: ${t.model} (${t.dimensions} dimensions)`,`- Corpus boilerplate removed: ${t.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${t.boilerplateRemoval.removedCharacters} characters`,`- Corpus SHA-256: ${t.corpusSha256}`].join(`
+`)}g(a(n,"report.md"),Q),g(a(n,"crawl-summary.json"),JSON.stringify(J,null,2)),g(a(n,"images-manifest.jsonl"),Te.map(t=>JSON.stringify(t)).join(`
+`)),g(a(n,"manifest.json"),JSON.stringify({version:"site-export.v1",jobId:r.id,startUrl:r.startUrl,generatedAt:J.generatedAt,outputProfile:{json:!0,html:!0,markdown:!0,imageManifest:!0,imageBytes:c},pages:[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId)),imagesManifestPath:"images-manifest.jsonl",download:"This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."},null,2)),_&&g(a(n,"wayback-manifest.json"),JSON.stringify({rootUrl:_.rootUrl??r.startUrl,requested:_.timeline??null,maxPagesPerSnapshot:_.maxPagesPerSnapshot??null,captureMatrix:"capture-matrix.jsonl"},null,2)),g(a(n,"issues.json"),JSON.stringify(je,null,2)),g(a(n,"link-metrics.jsonl"),me.map(t=>JSON.stringify(t)).join(`
+`)),g(a(n,"link-report.md"),Fe(xe)),g(a(n,"links-summary.json"),JSON.stringify(xe.summary,null,2)),g(a(n,"external-domains.json"),JSON.stringify(xe.externalDomains,null,2)),s.imageAudit&&(g(a(n,"images.jsonl"),s.imageAudit.rows.map(t=>JSON.stringify(t)).join(`
+`)),g(a(n,"images-summary.json"),JSON.stringify(s.imageAudit.summary,null,2))),s.seoAudit&&g(a(n,"seo-audit.json"),JSON.stringify(s.seoAudit,null,2)),s.branding&&g(a(n,"branding.json"),JSON.stringify(s.branding,null,2)),c&&g(a(n,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-ce,failed:ce,downloadedBytes:ye,perPageCap:Me,perSiteCap:Ie,perFileByteCap:nt,totalByteCap:rt},null,2));let P=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(w){P.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let t of I)P.push({name:t.path,type:"text/plain"})}$&&(P.push({name:"similarity-table.csv",type:"text/csv"}),P.push({name:"similarity.jsonl",type:"application/x-ndjson"}),P.push({name:"similarity-table-schema.json",type:"application/json"}),P.push({name:"similarity-summary.json",type:"application/json"}),P.push({name:"content-clusters.json",type:"application/json"})),_&&(P.push({name:"wayback-manifest.json",type:"application/json"}),P.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),s.imageAudit&&(P.push({name:"images.jsonl",type:"application/x-ndjson"}),P.push({name:"images-summary.json",type:"application/json"})),s.seoAudit&&P.push({name:"seo-audit.json",type:"application/json"}),s.branding&&P.push({name:"branding.json",type:"application/json"}),c&&P.push({name:"images-download-summary.json",type:"application/json"});let B=new Ct,mt=Ge({ownerId:String(r.userId),jobId:r.id,createdAt:new Date,content:B.outputStream});for(let t of P)B.addFile(a(n,t.name),t.name);let pe=[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId));for(let t of pe)t.markdownPath&&B.addFile(a(n,t.markdownPath),t.markdownPath);for(let t of pe)t.legacyMarkdownPath&&B.addFile(a(n,t.legacyMarkdownPath),t.legacyMarkdownPath);for(let t of pe)t.htmlPath&&B.addFile(a(n,t.htmlPath),t.htmlPath);for(let t of pe)B.addFile(a(n,t.jsonPath),t.jsonPath);if(c&&Z>ce){let t=a(n,"images");for(let e of xt(t,{recursive:!0})){let o=a(t,e);_t(o).isFile()&&B.addFile(o,`images/${e.split("\\").join("/")}`)}}return B.end(),[await mt,...ve]}finally{kt(n,{recursive:!0,force:!0})}}export{Ce as MAX_ANALYZED_LINK_EDGES,Rt as SITE_EXTRACT_PAGE_CHUNK,rn as assembleExtractArtifacts};

package/dist/gmail-service-BZ3H75XC.js

@@ -0,0 +1 @@
+import{A as d,B as e,C as f,D as g,E as h,x as a,y as b,z as c}from"./chunk-4AI7DOS7.js";import"./chunk-M22MM4N4.js";import"./chunk-HUV2WTRW.js";import"./chunk-MZN4U5BL.js";import"./chunk-MASR22K4.js";import"./chunk-GMWKPIYX.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};