mcp-scraper 0.90.4 → 0.90.5

package/dist/{chunk-WTOKMWDS.js → chunk-NC32JCZT.js}

@@ -1,4 +1,4 @@
-import{f as F,h as V,i as re,j as ae,l as $}from"./chunk-2GRE4BAO.js";import{_ as R,e as U,q as _,t as G,u as ne,v as oe,w as M,y as ie}from"./chunk-YEFXVTMJ.js";import{b as K,h as L}from"./chunk-DKQZNSYM.js";import{Q as te,Rb as D,Wb as x,t as C}from"./chunk-7BEWBRMT.js";import{createHash as Te,randomUUID as Ne}from"crypto";import{z as f}from"zod";async function at(e,t){!Number.isFinite(t)||t<=0||await K({op:R.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>L({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function se(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await K({op:R.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let o of n)!Number.isFinite(o.units)||o.units<=0||await L(o)})}var E="connected_usage",Re="mastra_scheduler",Ie=f.object({toolName:f.string().trim().min(1).max(200).optional(),operationKind:f.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:f.enum(["success","error","partial"]).optional(),requestId:f.string().trim().min(1).max(200).optional(),occurrenceId:f.string().trim().min(1).max(200).optional(),scheduleId:f.string().trim().min(1).max(200).optional(),nangoOperationId:f.string().trim().min(1).max(200).optional(),startedAt:f.string().datetime({offset:!0}).optional(),completedAt:f.string().datetime({offset:!0}).optional()}).strict(),Ae=f.object({identity:f.string().trim().min(1).max(320),idempotencyKey:f.string().trim().min(1).max(500),ratePolicyVersion:f.string().trim().min(1).max(100),connectionId:f.string().trim().min(1).max(500),providerConfigKey:f.string().trim().min(1).max(200),functionRuns:f.number().int().safe().nonnegative(),proxyRequests:f.number().int().safe().nonnegative(),computeMilliseconds:f.number().int().safe().nonnegative(),sourceSurface:f.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Re),metadata:Ie.optional()}).strict(),Pe=f.object({identity:f.string().trim().min(1).max(320),ratePolicyVersion:f.string().trim().min(1).max(100)}).strict(),h=class extends Error{constructor(n,o,a,i={}){super(o);this.code=n;this.httpStatus=a;this.details=i;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function z(e){return Te("sha256").update(e).digest("hex")}function xe(e){return`connected-usage:${z(e)}`}function Ue(e){return`connected-usage-debit:${z(e)}`}function Me(e){return e.trim().toLowerCase()}async function ce(e){let t=await te(Me(e));if(!t)throw new h("user_not_found","billing identity was not found",404);return t}function Oe(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return z(JSON.stringify({ratePolicyVersion:_,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function A(e){let t=JSON.stringify(e);if(t.length>4e3)throw new h("invalid_request","connected usage metadata is too large",400);return t}function W(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==_||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new h("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function H(e){let t=await C().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function de(e,t){return(await C().execute({sql:`SELECT 1
+import{f as F,h as V,i as re,j as ae,l as $}from"./chunk-Q5SUDDSQ.js";import{_ as R,e as U,q as _,t as G,u as ne,v as oe,w as M,y as ie}from"./chunk-YEFXVTMJ.js";import{b as K,k as L}from"./chunk-56UT5M5Y.js";import{Q as te,Rb as D,Wb as x,t as C}from"./chunk-KKHTMYCD.js";import{createHash as Te,randomUUID as Ne}from"crypto";import{z as f}from"zod";async function at(e,t){!Number.isFinite(t)||t<=0||await K({op:R.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>L({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function se(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await K({op:R.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let o of n)!Number.isFinite(o.units)||o.units<=0||await L(o)})}var E="connected_usage",Re="mastra_scheduler",Ie=f.object({toolName:f.string().trim().min(1).max(200).optional(),operationKind:f.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:f.enum(["success","error","partial"]).optional(),requestId:f.string().trim().min(1).max(200).optional(),occurrenceId:f.string().trim().min(1).max(200).optional(),scheduleId:f.string().trim().min(1).max(200).optional(),nangoOperationId:f.string().trim().min(1).max(200).optional(),startedAt:f.string().datetime({offset:!0}).optional(),completedAt:f.string().datetime({offset:!0}).optional()}).strict(),Ae=f.object({identity:f.string().trim().min(1).max(320),idempotencyKey:f.string().trim().min(1).max(500),ratePolicyVersion:f.string().trim().min(1).max(100),connectionId:f.string().trim().min(1).max(500),providerConfigKey:f.string().trim().min(1).max(200),functionRuns:f.number().int().safe().nonnegative(),proxyRequests:f.number().int().safe().nonnegative(),computeMilliseconds:f.number().int().safe().nonnegative(),sourceSurface:f.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Re),metadata:Ie.optional()}).strict(),Pe=f.object({identity:f.string().trim().min(1).max(320),ratePolicyVersion:f.string().trim().min(1).max(100)}).strict(),h=class extends Error{constructor(n,o,a,i={}){super(o);this.code=n;this.httpStatus=a;this.details=i;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function z(e){return Te("sha256").update(e).digest("hex")}function xe(e){return`connected-usage:${z(e)}`}function Ue(e){return`connected-usage-debit:${z(e)}`}function Me(e){return e.trim().toLowerCase()}async function ce(e){let t=await te(Me(e));if(!t)throw new h("user_not_found","billing identity was not found",404);return t}function Oe(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return z(JSON.stringify({ratePolicyVersion:_,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function A(e){let t=JSON.stringify(e);if(t.length>4e3)throw new h("invalid_request","connected usage metadata is too large",400);return t}function W(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==_||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new h("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function H(e){let t=await C().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function de(e,t){return(await C().execute({sql:`SELECT 1
           FROM billing_events
           WHERE user_id = ?
             AND billing_class = ?

package/dist/{chunk-HR2ALIEH.js → chunk-PNDXAOZD.js}

@@ -1,4 +1,4 @@
-import{A as f,Ab as _,Bb as y,Cb as C,Ya as g,_a as S,lb as R,mb as E,ob as v,wb as T,yb as h,z as u,zb as w}from"./chunk-LTT2B7NY.js";import{a as P}from"./chunk-7QHYBK6U.js";import{a as m}from"./chunk-KJA62B7P.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
+import{A as f,Ab as _,Bb as y,Cb as C,Ya as g,_a as S,lb as R,mb as E,ob as v,wb as T,yb as h,z as u,zb as w}from"./chunk-5JCPVCBT.js";import{a as P}from"./chunk-W4QBJ7WW.js";import{a as m}from"./chunk-ARKQYLKU.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
 `).filter(o=>o.startsWith("data:")).map(o=>o.slice(5).trim()).filter(Boolean);if(!s.length)return JSON.parse(r);for(let o of s){let e=JSON.parse(o);if(e.result||e.error)return e}throw new Error("hosted MCP returned no JSON-RPC result")}function a(r){return{content:[{type:"text",text:JSON.stringify({ok:!1,error:r})}],isError:!0}}var c=class{baseUrl;apiKey;constructor(s,o){this.baseUrl=s.replace(/\/$/,""),this.apiKey=o}async callMemoryTool(s,o){let e=x.get(s);if(!e)return a(`unknown memory tool: ${s}`);try{let n=await fetch(`${this.baseUrl}/mcp`,{method:"POST",headers:{"content-type":"application/json",accept:"application/json, text/event-stream","x-api-key":this.apiKey},body:JSON.stringify({jsonrpc:"2.0",id:`memory:${e}`,method:"tools/call",params:{name:e,arguments:o}})}),l=await n.text();if(!n.ok)return a(`hosted memory ${e} failed (HTTP ${n.status})`);let t=O(l);return t.error?a(t.error.message??`hosted memory ${e} failed`):t.result??a(`hosted memory ${e} returned no result`)}catch(n){return a(n instanceof Error?n.message:`hosted memory ${e} call failed`)}}};function M(r,s){if(s.length===0)throw new Error("Restricted MCP tool allowlist must contain at least one tool name");let o=new Set;for(let t of s){if(typeof t!="string"||t.trim()!==t||t.length===0)throw new Error("Restricted MCP tool allowlist contains a malformed tool name");if(o.has(t))throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${t}`);o.add(t)}let e=new Set,n=r,l=n.registerTool.bind(r);return n.registerTool=(t,i,p)=>{if(o.has(t))return e.add(t),l(t,i,p)},{assertComplete(){let t=[...o].filter(i=>!e.has(i));if(t.length>0)throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${t.join(", ")}`)},registeredToolNames(){return[...e]}}}var L=new Set(["paa","serp","browser-agent","scheduled-results","memory"]);function $(){let s=[process.env.MCP_SCRAPER_KEY_PATH?.trim(),A(I(),".mcp-scraper-key")].filter(Boolean);for(let o of s)try{let e=N(o,"utf8").trim();if(e)return e}catch{}}function Y(r,s={}){let o=s.toolsets??L,e=process.env.MCP_SCRAPER_BASE_URL?.trim()||process.env.MCP_BASE_URL?.trim()||"https://mcpscraper.dev",n=g(),l=S({deploymentProfile:n,transportProfile:"stdio",baseUrl:e,explicitlyEnabled:process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK==="1"}),t=process.env.BROWSER_AGENT_CONSOLE_URL?.trim()||e,i=new k({name:"mcp-scraper",version:m},{instructions:u,cacheHints:{"server/discover":{ttlMs:3e5,cacheScope:"private"},"tools/list":{ttlMs:3e5,cacheScope:"private"},"prompts/list":{ttlMs:3e5,cacheScope:"private"},"resources/list":{ttlMs:3e5,cacheScope:"private"},"resources/templates/list":{ttlMs:3e5,cacheScope:"private"},"resources/read":{ttlMs:6e4,cacheScope:"private"}}});f(i);let p=s.allowedToolNames?M(i,s.allowedToolNames):void 0,d=s.httpExecutor??new T(e,r,{localNetworkAccess:l});return o.has("paa")&&v(i,d,{ownerId:R(r),deploymentProfile:n,transportProfile:"stdio",baseUrl:e,localNetworkAccess:l,taskHandleSecret:r}),o.has("serp")&&E(i,d,{exposeDevelopmentDiagnostics:n==="development"||n==="test"}),o.has("browser-agent")&&h(i,{baseUrl:e,apiKey:r,consoleBaseUrl:t}),o.has("scheduled-results")&&C(i,new y(e,r)),o.has("memory")&&_(i,new c(e,r)),p?.assertComplete(),i}function se(r={}){let s=process.argv.includes("--stdio")||process.env.MCP_SCRAPER_FORCE_STDIO==="1",o=!!(process.stdin.isTTY&&process.stdout.isTTY),e=process.argv.includes("--help")||process.argv.includes("-h");if(!s&&(o||e)){let l=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY;process.stdout.write(P({version:m,color:!l,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()})),process.exit(0)}let n=(process.env.MCP_SCRAPER_API_KEY??process.env.MCP_SCRAPER_KEY??process.env.MCP_API_KEY??$())?.trim();n||(process.stderr.write(`MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required
 `),process.exit(1)),b(()=>Y(n,r),{legacy:"serve",onerror(l){process.stderr.write(`${l.message}
 `)}})}export{se as a};

package/dist/{chunk-2GRE4BAO.js → chunk-Q5SUDDSQ.js}

@@ -1,4 +1,4 @@
-import{Q as p,t as o}from"./chunk-7BEWBRMT.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
+import{Q as p,t as o}from"./chunk-KKHTMYCD.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
       CREATE TABLE IF NOT EXISTS service_connections (
         id                         TEXT PRIMARY KEY,
         user_id                    INTEGER NOT NULL REFERENCES users(id),

package/dist/{chunk-7QHYBK6U.js → chunk-W4QBJ7WW.js}

@@ -1,4 +1,4 @@
-var s={message:"Scheduled analytics retries now use valid timestamp arithmetic, preventing the production scheduler from rejecting retry updates."};var _={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${_[e]}${t}${_.reset}`:t}function o(t,e,n){let a=t.padEnd(9," ");return`  ${r(a,"muted",n)} ${e.join(r("  .  ","muted",n))}`}function p(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",a=String.raw`
+var s={message:"SERP and PAA runs now preserve provider attempts, retry causes, customer billing links, and honest estimated or pending provider costs for operational troubleshooting."};var _={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${_[e]}${t}${_.reset}`:t}function o(t,e,n){let a=t.padEnd(9," ");return`  ${r(a,"muted",n)} ${e.join(r("  .  ","muted",n))}`}function p(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",a=String.raw`
  __  __  ____ ____ 
 |  \/  |/ ___|  _ \
 | |\/| | |   | |_) |

package/dist/chunk-X4YCWF4B.js

@@ -0,0 +1,48 @@
+import{a as P}from"./chunk-XG6GCEUE.js";import{d as k,g as L}from"./chunk-56UT5M5Y.js";import{b as y,g as c,h as v,ha as O,ia as N,j as R,ka as x,m as I,o as A,t as a}from"./chunk-KKHTMYCD.js";import{createHash as j,randomUUID as b}from"crypto";var w=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function g(e){return j("sha256").update(e).digest("hex")}function S(e){return Array.isArray(e)?e.map(S):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([r],[o])=>r.localeCompare(o)).map(([r,o])=>[r,S(o)])):e}function T(e){return JSON.stringify(S(e??{}))}function l(e,r){let o=e.trim();if(!o)throw new Error(`${r} is required`);return o}function p(e,r=2e3){return e==null?null:e.slice(0,r)}async function C(e){let r=a(),o=l(e.tool,"tool"),s=l(e.ownerScope,"ownerScope"),i=e.idempotencyKey?.trim()||null,t=e.requestFingerprint?.trim()||null;if(i==null!=(t==null))throw new Error("idempotencyKey and requestFingerprint must be supplied together");let n=i?g(`${s}\0${o}\0${i}`):null,d=e.id??b(),u=[d,e.userId??null,g(s),o,n,t,e.jobId??null,e.requestId??null,e.deploymentCommit??null,T(e.normalizedFlags),e.startedAt??new Date().toISOString()];if(n){await r.execute({sql:`INSERT OR IGNORE INTO operation_runs
+        (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
+         job_id, request_id, deployment_commit, normalized_flags_json, started_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:u});let f=(await r.execute({sql:"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?",args:[n]})).rows[0];if(!f)throw new Error("Operation run insert did not produce an idempotency receipt");if(String(f.request_fingerprint)!==t)throw new w;return{id:String(f.id),replayed:String(f.id)!==d}}return await r.execute({sql:`INSERT INTO operation_runs
+      (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
+       job_id, request_id, deployment_commit, normalized_flags_json, started_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:u}),{id:d,replayed:!1}}async function U(e){let r=e.endedAt??new Date().toISOString();if((await a().execute({sql:`UPDATE operation_runs
+      SET status = ?, failure_class = ?, ended_at = ?, updated_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,p(e.failureClass,200),r,r,e.runId]})).rowsAffected===1)return;let s=p(e.failureClass,200),i=await a().execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]});if(i.rows.length===0)throw new Error(`Unknown operation run: ${e.runId}`);let t=i.rows[0]?.failure_class==null?null:String(i.rows[0]?.failure_class);if(String(i.rows[0]?.status)!==e.status||t!==s)throw new Error(`Operation run ${e.runId} is already terminal as ${String(i.rows[0]?.status)}`)}async function q(e){if(!Number.isInteger(e.attemptNumber)||e.attemptNumber<1)throw new Error("attemptNumber must be a positive integer");if(e.attemptNumber===1&&(e.parentAttemptId||e.edgeKind||e.retryKind))throw new Error("The first attempt cannot have a parent or retry edge");if(e.attemptNumber>1&&(!e.parentAttemptId||!e.edgeKind))throw new Error("Every non-first attempt requires a parentAttemptId and edgeKind");if(e.retryKind&&!["same_method_retry","provider_fallback","durable_resume","reconciliation"].includes(e.edgeKind??""))throw new Error("retryKind requires a retry or reconciliation edgeKind");let r=e.id??b();return await a().execute({sql:`INSERT INTO operation_attempts
+      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason, retry_owner,
+       method, provider, normalized_flags_json, started_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[r,e.runId,e.parentAttemptId??null,e.attemptNumber,e.edgeKind??null,e.retryKind??null,p(e.retryReason,500),p(e.retryOwner,100),l(e.method,"method"),e.provider??null,T(e.normalizedFlags),e.startedAt??new Date().toISOString()]}),r}async function D(e){if(!Number.isInteger(e.durationMs)||e.durationMs<0)throw new Error("durationMs must be a non-negative integer");let r=e.endedAt??new Date().toISOString();if((await a().execute({sql:`UPDATE operation_attempts SET
+      status = ?, error_code = ?, error_message = ?, ended_at = ?, duration_ms = ?, updated_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,p(e.errorCode,200),p(e.errorMessage),r,e.durationMs,r,e.attemptId]})).rowsAffected===1)return;let s=await a().execute({sql:"SELECT status, duration_ms FROM operation_attempts WHERE id = ?",args:[e.attemptId]});if(s.rows.length===0)throw new Error(`Unknown operation attempt: ${e.attemptId}`);if(String(s.rows[0]?.status)!==e.status||Number(s.rows[0]?.duration_ms)!==e.durationMs)throw new Error(`Operation attempt ${e.attemptId} is already terminal as ${String(s.rows[0]?.status)}`)}async function H(e){let r=l(e.provider,"provider"),o=l(e.method,"method"),s=g(l(e.accountScope,"accountScope")),i=e.providerSourceKey?.trim()||null,t=i?g(i):null,n=e.id??b(),d=await a().execute({sql:`INSERT OR IGNORE INTO provider_usage_receipts
+      (id, attempt_id, provider, account_scope_hash, method, provider_source_key,
+       provider_source_key_hash, provider_status)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,args:[n,e.attemptId,r,s,o,i,t,e.providerStatus??null]});if(!t){if(d.rowsAffected===1)return{id:n,replayed:!1};let _=(await a().execute({sql:"SELECT attempt_id, provider, method FROM provider_usage_receipts WHERE id = ?",args:[n]})).rows[0];if(!_||String(_.attempt_id)!==e.attemptId||String(_.provider)!==r||String(_.method)!==o)throw new Error("Provider receipt ID is already used by a different receipt");return{id:n,replayed:!0}}let m=(await a().execute({sql:`SELECT id, attempt_id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[r,s,o,t]})).rows[0];if(!m)throw new Error("Provider receipt insert did not produce a source receipt");if(String(m.attempt_id)!==e.attemptId)throw new Error("Provider source identity is already attached to a different attempt");return{id:String(m.id),replayed:String(m.id)!==n}}async function M(e,r){let o=l(r,"providerSourceKey"),s=g(o);if((await a().execute({sql:`UPDATE provider_usage_receipts
+      SET provider_source_key = ?, provider_source_key_hash = ?, updated_at = datetime('now')
+      WHERE id = ? AND provider_source_key IS NULL`,args:[o,s,e]})).rowsAffected===1)return;let t=await a().execute({sql:"SELECT provider_source_key_hash FROM provider_usage_receipts WHERE id = ?",args:[e]});if(!t.rows[0])throw new Error(`Unknown provider receipt: ${e}`);if(String(t.rows[0].provider_source_key_hash)!==s)throw new Error(`Provider receipt ${e} already has a different source identity`)}async function h(e){if(e.units!=null&&(!Number.isFinite(e.units)||e.units<0))throw new Error("units must be non-negative");if(e.amountUsdNanos!=null&&(!Number.isSafeInteger(e.amountUsdNanos)||e.amountUsdNanos<0))throw new Error("amountUsdNanos must be a non-negative safe integer");if(e.costState==="actual"&&e.amountUsdNanos==null)throw new Error("actual receipts require amountUsdNanos");if(e.costState==="actual"&&e.amountUsdNanos===0&&!e.zeroCostReason?.trim())throw new Error("actual zero-cost receipts require zeroCostReason");let r=e.checkedAt??new Date().toISOString(),o=e.finalizedAt??r;if((await a().execute({sql:`UPDATE provider_usage_receipts SET
+      cost_state = ?, units = ?, unit_type = ?, amount_usd_nanos = ?, rate_source = ?, zero_cost_reason = ?,
+      provider_status = ?, provider_duration_ms = ?, provider_captcha = ?, error_code = ?, error_message = ?,
+      checked_at = ?, finalized_at = ?, updated_at = ?
+      WHERE id = ? AND (cost_state = 'pending' OR (cost_state = 'estimated' AND ? = 'actual'))`,args:[e.costState,e.units??null,e.unitType??null,e.amountUsdNanos??null,e.rateSource??null,e.zeroCostReason??null,e.providerStatus??null,e.providerDurationMs??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,p(e.errorCode,200),p(e.errorMessage),r,o,o,e.receiptId,e.costState]})).rowsAffected===1)return;let i=await a().execute({sql:"SELECT cost_state, units, unit_type, amount_usd_nanos FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]});if(i.rows.length===0)throw new Error(`Unknown provider receipt: ${e.receiptId}`);let t=i.rows[0],n=t?.units==null&&e.units==null||Number(t?.units)===e.units,d=t?.amount_usd_nanos==null&&e.amountUsdNanos==null||Number(t?.amount_usd_nanos)===e.amountUsdNanos,u=t?.unit_type==null&&e.unitType==null||String(t?.unit_type)===e.unitType;if(String(t?.cost_state)!==e.costState||!n||!d||!u)throw new Error(`Provider receipt ${e.receiptId} is already finalized as ${String(i.rows[0]?.cost_state)}`)}async function ie(e){let o=(await a().execute({sql:`SELECT id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[l(e.provider,"provider"),g(l(e.accountScope,"accountScope")),l(e.method,"method"),g(l(e.providerSourceKey,"providerSourceKey"))]})).rows[0];if(!o)return!1;let{provider:s,accountScope:i,method:t,providerSourceKey:n,...d}=e;return await h({...d,receiptId:String(o.id)}),!0}async function se(e=30){if(!Number.isInteger(e)||e<1||e>365)throw new Error("retentionDays must be an integer between 1 and 365");let r=`-${e} days`,o=a(),[s,i,t]=await Promise.all([o.execute({sql:`UPDATE operation_attempts
+        SET error_message = NULL, updated_at = datetime('now')
+        WHERE ended_at < datetime('now', ?) AND error_message IS NOT NULL`,args:[r]}),o.execute({sql:`UPDATE provider_usage_receipts
+        SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
+        WHERE finalized_at < datetime('now', ?)
+          AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[r]}),o.execute({sql:`UPDATE harvest_attempts
+        SET provider_session_id = NULL, \u006b\u0065\u0072\u006e\u0065\u006c_session_id = NULL, error = NULL,
+            \u006b\u0065\u0072\u006e\u0065\u006c_delete_error = NULL, browser_close_error = NULL,
+            provider_error_message = NULL, provider_detail_json = NULL,
+            provider_disconnect_message = NULL, debug_json = NULL
+        WHERE completed_at < datetime('now', ?)
+          AND (provider_session_id IS NOT NULL OR \u006b\u0065\u0072\u006e\u0065\u006c_session_id IS NOT NULL OR error IS NOT NULL
+            OR \u006b\u0065\u0072\u006e\u0065\u006c_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
+            OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
+            OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[r]})]);return{operationAttempts:s.rowsAffected,providerReceipts:i.rowsAffected,legacyHarvestAttempts:t.rowsAffected}}async function F(e){let o={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],s=l(e.billingKey,"billingKey"),i=await a().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
+      FROM operation_runs r
+      JOIN ${e.billingTable} b ON CAST(b.${o.column} AS TEXT) = ?
+      WHERE r.id = ?`,args:[s,e.runId]});if(!i.rows[0])throw new Error("Billing link target or operation run does not exist");let t=i.rows[0].billing_user_id,n=i.rows[0].run_user_id;if(n!=null&&t!=null&&String(n)!==String(t))throw new Error("Billing link owner does not match operation run owner");await a().execute({sql:`INSERT OR IGNORE INTO operation_billing_links
+      (run_id, billing_table, billing_key, relation, amount_mc) VALUES (?, ?, ?, ?, ?)`,args:[e.runId,e.billingTable,s,e.relation,e.amountMc??null]})}async function ae(e,r=new Date().toISOString()){let o=b();return await a().execute({sql:"INSERT INTO provider_reconciliation_runs (id, provider, started_at) VALUES (?, ?, ?)",args:[o,l(e,"provider"),r]}),o}async function de(e){if([e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount].some(i=>!Number.isInteger(i)||i<0))throw new Error("Reconciliation counts must be non-negative integers");if(e.completedCount+e.pendingCount+e.unavailableCount+e.failedCount>e.checkedCount)throw new Error("Reconciliation outcome counts cannot exceed checkedCount");let o=e.completedAt??new Date().toISOString();if((await a().execute({sql:`UPDATE provider_reconciliation_runs SET
+      status = ?, checked_count = ?, completed_count = ?, pending_count = ?, unavailable_count = ?,
+      failed_count = ?, error_code = ?, completed_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount,p(e.errorCode,200),o,e.id]})).rowsAffected!==1)throw new Error(`Unknown or terminal provider reconciliation run: ${e.id}`)}async function K(e){let r=a(),[o,s,i,t]=await Promise.all([r.execute({sql:"SELECT * FROM operation_runs WHERE id = ?",args:[e]}),r.execute({sql:"SELECT * FROM operation_attempts WHERE run_id = ? ORDER BY attempt_number",args:[e]}),r.execute({sql:`SELECT r.* FROM provider_usage_receipts r
+        JOIN operation_attempts a ON a.id = r.attempt_id
+        WHERE a.run_id = ? ORDER BY a.attempt_number, r.created_at, r.id`,args:[e]}),r.execute({sql:"SELECT * FROM operation_billing_links WHERE run_id = ? ORDER BY created_at, billing_table, billing_key",args:[e]})]);return o.rows[0]?{run:o.rows[0],attempts:[...s.rows],providerReceipts:[...i.rows],billingLinks:[...t.rows]}:null}function W(e){return e instanceof Error?e.message:String(e)}function B(e,r){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(r)}function V(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function J(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function G(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function Q(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function Y(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function X(e){let r=W(e);return e instanceof I?{error_code:"request_aborted",error_type:"request_aborted",message:c("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof R||V(r)?{error_code:"captcha_exhausted",error_type:"captcha",message:c("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:P(e,r)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:c("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof A?{error_code:"location_mismatch",error_type:"location_mismatch",message:c("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:J(r)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:c("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:G(r)?{error_code:"proxy_unavailable",error_type:"connection",message:c("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Y(r)?{error_code:"browser_session_interrupted",error_type:"extraction",message:c("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:B(e,r)?{error_code:"harvest_timeout",error_type:"timeout",message:c("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:Q(r)?{error_code:"service_unavailable",error_type:"service_unavailable",message:y,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:y,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function pe(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function z(e,r={}){let o=v({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:r.retryAfterSeconds,chargeStatus:r.chargeStatus,details:r.details});return{error:o.message,...o}}function me(e,r={}){let{error:o,...s}=z(e,r);return s}function _e(e,r={}){let o=X(e);return{body:z(o,r),status:o.httpStatus}}import{createHash as Z}from"crypto";var ee=1e9;function re(e){return Z("sha256").update(JSON.stringify(e)).digest("hex")}function te(e){return`harvest:${e}`}async function we(e){let r=te(e.jobId);try{return(await C({id:r,userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,idempotencyKey:`job:${e.jobId}`,requestFingerprint:re(e.normalizedFlags),jobId:e.jobId,requestId:e.requestId??null,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id}catch(o){return console.warn(JSON.stringify({event:"operation_run_start_failed",job_id:e.jobId,tool:e.tool,message:o instanceof Error?o.message:String(o)})),null}}async function Se(e,r,o){if(e)try{await U({runId:e,status:r,failureClass:o})}catch(s){console.warn(JSON.stringify({event:"operation_run_finish_failed",operation_run_id:e,message:s instanceof Error?s.message:String(s)}))}}async function he(e,r,o,s){if(!(!e||!r))try{await F({runId:e,billingTable:"billing_debits",billingKey:r,relation:o,amountMc:s})}catch(i){console.warn(JSON.stringify({event:"operation_billing_link_failed",operation_run_id:e,relation:o,message:i instanceof Error?i.message:String(i)}))}}function E(e){return e==="timeout"?"timed_out":e==="request_aborted"?"cancelled":e==="browser_session_interrupted"?"interrupted":e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"||e==="location_mismatch"||e==="error"?"failed":"succeeded"}function $(e,r,o=0){let s=new Map,i=null;return async t=>{if(e)try{if(t.type==="started"){let d=o+t.attemptNumber;if(d>1&&!i){let _=(await K(e))?.attempts.at(-1);i=_?.id==null?null:String(_.id)}let u=await q({runId:e,attemptNumber:d,method:t.method,provider:t.provider,parentAttemptId:d===1?null:i,edgeKind:d===1?null:o>0&&t.attemptNumber===1?"durable_resume":t.retryKind==="provider_fallback"?"provider_fallback":"same_method_retry",retryKind:d===1?null:o>0&&t.attemptNumber===1?"durable_resume":t.retryKind,retryReason:d===1?null:o>0&&t.attemptNumber===1?"durable_worker_resume":t.retryReason,retryOwner:d===1?null:o>0&&t.attemptNumber===1?"durable_reconciler":"harvest",normalizedFlags:{maxAttempts:t.maxAttempts,maxQuestions:t.maxQuestions,hasLocation:!!t.location},startedAt:t.startedAt}),m=await H({attemptId:u,provider:t.provider,accountScope:r,method:t.method,providerStatus:"started"});s.set(t.attemptNumber,{attemptId:u,receiptId:m.id,provider:t.provider,method:t.method,startedAtMs:Date.parse(t.startedAt)}),i=u;return}let n=s.get(t.attemptNumber);if(!n)throw new Error(`Missing operation attempt state for attempt ${t.attemptNumber}`);if(t.type==="provider_session_started"){await M(n.receiptId,t.providerSessionId);return}if(await D({attemptId:n.attemptId,status:E(t.outcome),errorCode:E(t.outcome)==="succeeded"?null:t.outcome,errorMessage:t.error,durationMs:t.durationMs,endedAt:t.completedAt}),n.provider==="\u006b\u0065\u0072\u006e\u0065\u006c"){let d=L(t.durationMs,!0);await h({receiptId:n.receiptId,costState:"estimated",units:t.durationMs/1e3,unitType:"browser_second",amountUsdNanos:Math.round(d*ee),rateSource:`\u006b\u0065\u0072\u006e\u0065\u006c_headful_usd_per_second:${k}`,providerStatus:t.outcome,providerDurationMs:t.durationMs,errorCode:E(t.outcome)==="succeeded"?null:t.outcome,errorMessage:t.error,finalizedAt:t.completedAt})}}catch(n){console.warn(JSON.stringify({event:"operation_attempt_telemetry_failed",operation_run_id:e,attempt_number:t.attemptNumber,message:n instanceof Error?n.message:String(n)}))}}}function Ie(e,r,o=null,s="production-default",i=0){let t=$(o,s,i);return async n=>{if(await t(n),n.type==="started"){await O({jobId:e,userId:r,attemptNumber:n.attemptNumber,maxAttempts:n.maxAttempts,query:n.query,location:n.location,maxQuestions:n.maxQuestions,startedAt:n.startedAt});return}if(n.type==="provider_session_started"){await N({jobId:e,attemptNumber:n.attemptNumber,browserProvider:n.provider,providerSessionId:n.providerSessionId,observedAt:n.observedAt});return}await x({jobId:e,attemptNumber:n.attemptNumber,outcome:n.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:n.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:n.questionCount,durationMs:n.durationMs,error:n.error,willRetry:n.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:n.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:n.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:n.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:n.cleanup.browserCloseSucceeded,browserCloseError:n.cleanup.browserCloseError,browserProvider:n.cleanup.provider??null,providerSessionId:n.cleanup.providerSessionId??null,providerLookupStatus:n.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:n.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:n.cleanup.providerDisconnectMessage??null,providerReasonSource:n.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:n.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:n.cleanup.providerDisconnectMessage??null,debug:n.debug,completedAt:n.completedAt})}}async function Oe(e,r,o=3){for(let s=1;s<=o;s++){try{let i=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(r),signal:AbortSignal.timeout(1e4)});if(i.ok)return;console.warn(`[webhook] attempt ${s} \u2192 ${i.status} from ${e}`)}catch(i){console.warn(`[webhook] attempt ${s} failed:`,i instanceof Error?i.message:i)}s<o&&await new Promise(i=>setTimeout(i,1e3*s*2))}console.error(`[webhook] gave up after ${o} attempts for ${e}`)}export{q as a,D as b,H as c,M as d,h as e,ie as f,se as g,ae as h,de as i,X as j,pe as k,z as l,me as m,_e as n,te as o,we as p,Se as q,he as r,Ie as s,Oe as t};

package/dist/{chunk-ZMLCK45S.js → chunk-XSYLDTMI.js}

@@ -1,4 +1,4 @@
-import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{_ as w}from"./chunk-YEFXVTMJ.js";import{Jb as N,Sb as D,d as m,e as I,i as u,t as s}from"./chunk-7BEWBRMT.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{_ as w}from"./chunk-YEFXVTMJ.js";import{Jb as N,Sb as D,d as m,e as I,i as u,t as s}from"./chunk-KKHTMYCD.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
             (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs

package/dist/{chunk-JVU2THSY.js → chunk-XXK5PLRB.js}

@@ -1,4 +1,4 @@
-import{a as N}from"./chunk-XG6GCEUE.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,t as x}from"./chunk-7BEWBRMT.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
+import{a as N}from"./chunk-XG6GCEUE.js";import{b as L,c as P}from"./chunk-HE45FFBU.js";import{h as $,t as x}from"./chunk-KKHTMYCD.js";import{createWriteStream as Z,mkdirSync as ee,rmSync as te}from"fs";import{homedir as re}from"os";import{join as I,extname as k,basename as B}from"path";import{pipeline as ne}from"stream/promises";import{Readable as oe,Transform as ie}from"stream";var se=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],ae=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),de=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),ce=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function le(e){let t=e.toLowerCase();return se.some(r=>t.includes(r))}function ue(e){return e.startsWith("data:")}function F(e){try{let t=k(new URL(e).pathname).toLowerCase();if(ae.has(t))return"image";if(de.has(t))return"video";if(ce.has(t))return"audio"}catch{}return null}function q(e){let t=e.toLowerCase();return t.startsWith("image/")?"image":t.startsWith("video/")?"video":t.startsWith("audio/")?"audio":null}function me(e,t){if(!e||ue(e))return null;try{return new URL(e,t).href}catch{return null}}function U(e,t){try{let r=new URL(e);return B(r.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${t}`}catch{return`asset-${t}`}}function O(e,t){let r=e?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return r?r.slice(0,t):null}function ge(e){let t=e.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return t?{width:Number(t[1]),height:Number(t[2])}:null}function pe(e){try{let t=new URL(e);t.hash="",t.pathname=t.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let r of["w","width","h","height","resize"])t.searchParams.delete(r);return t.searchParams.sort(),t.href}catch{return e}}function S(e){let t=(e.width??0)*(e.height??0),r=e.discoveryMethods.some(g=>g.includes("lightbox"))?1e12:0,n=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(e.url)?0:1e11;return r+n+t}function A(e,t){let r=e.get(t.url);if(!r){e.set(t.url,t);return}r.discoveryMethods=[...new Set([...r.discoveryMethods,...t.discoveryMethods])],r.altTexts=[...new Set([...r.altTexts,...t.altTexts])],r.contexts=[...new Set([...r.contexts,...t.contexts])],!r.type&&t.type&&(r.type=t.type),(t.width??0)*(t.height??0)>(r.width??0)*(r.height??0)&&(r.width=t.width,r.height=t.height)}function he(e,t){let r=new Map,n=(d,i,o=null,h,l,u)=>{let s=d?me(d.trim().replace(/&/g,"&"),t):null;if(!s||le(s))return;let m=u??ge(s);A(r,{url:s,type:o??F(s),discoveryMethods:[i],altTexts:h?[h]:[],contexts:l?[l]:[],width:m?.width??null,height:m?.height??null})},g=(d,i,o,h)=>{for(let l of(d??"").split(",")){let[u,s]=l.trim().split(/\s+/);if(!u)continue;let m=s?.match(/^(\d+)w$/)?.[1];n(u,i,o,h,null,m?{width:Number(m),height:0}:null)}},a=(d,i="")=>{let o=d.$;for(let l of d.images){let u=l.attributes,s=O(l.alt,500),m=l.width&&l.height?{width:l.width,height:l.height}:null;for(let p of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=u[p];n(y,`static-${p}${i}`,"image",s,null,m)}g(u.srcset,`static-srcset${i}`,"image",s),g(u["data-srcset"],`static-data-srcset${i}`,"image",s)}o("source").each((l,u)=>{let s=o(u),m=s.attr("type")??"",p=m.startsWith("video/")?"video":m.startsWith("audio/")?"audio":"image";n(s.attr("src"),`static-source${i}`,p),g(s.attr("srcset"),`static-source-srcset${i}`,p)}),o("video,audio").each((l,u)=>{let s=o(u),m=u.tagName.toLowerCase();n(s.attr("src"),`static-${m}${i}`,m),m==="video"&&n(s.attr("poster"),`static-video-poster${i}`,"image")}),o("meta").each((l,u)=>{let s=o(u),m=(s.attr("property")??s.attr("name")??"").toLowerCase();(m==="og:image"||m==="og:image:url"||m==="twitter:image"||m==="twitter:image:src")&&n(s.attr("content"),`static-${m}${i}`,"image")}),o("link[href]").each((l,u)=>{let s=o(u);(s.attr("rel")??"").toLowerCase().split(/\s+/).some(p=>p==="icon"||p==="apple-touch-icon")&&n(s.attr("href"),`static-icon${i}`,"image")}),o("svg image").each((l,u)=>{let s=o(u);n(s.attr("href")??s.attr("xlink:href"),`static-svg-image${i}`,"image")}),o("a[href]").has("img").each((l,u)=>{let s=o(u),m=s.attr("href");if(m&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${s.attr("class")??""} ${s.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(m))){let p=O(s.find("img").first().attr("alt"),500);n(m,`static-lightbox-target${i}`,"image",p)}});let h=o("style").toArray().map(l=>o(l).text());o("[style]").each((l,u)=>{h.push(o(u).attr("style")??"")});for(let l of h.join(`
 `).matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?([^"')&]+)(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?\s*\)/gi))n(l[1],`static-css-url${i}`,"image");for(let l of d.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi))n(l[1],`static-bare-url${i}`)},c=typeof e=="string"?P(e):e;return a(c),c.$("script").each((d,i)=>{let o=c.$(i).text(),h=o.replace(/\\u003[cC]/g,"<").replace(/\\u003[eE]/g,">").replace(/\\u0026/gi,"&").replace(/\\["']/g,l=>l.slice(1)).replace(/\\\//g,"/");h!==o&&/<(?:img|source|video|audio)\b/i.test(h)&&a(P(h),"-json-unescaped")}),[...r.values()]}function fe(e,t,r=100){let n=new Map;for(let a of e)A(n,a);for(let a of t?.assets??[])A(n,{...a,type:a.type});let g=new Map;for(let a of n.values()){let c=pe(a.url),d=g.get(c)??[];d.push(a),g.set(c,d)}return[...g.values()].map(a=>{let c=[...a].sort((i,o)=>S(o)-S(i)||i.url.localeCompare(o.url)),d={...c[0]};return d.discoveryMethods=[...new Set(a.flatMap(i=>i.discoveryMethods))],d.altTexts=[...new Set(a.flatMap(i=>i.altTexts))],d.contexts=[...new Set(a.flatMap(i=>i.contexts))],{...d,variants:c.slice(1).map(i=>i.url)}}).sort((a,c)=>S(c)-S(a)||a.url.localeCompare(c.url)).slice(0,Math.max(1,r))}async function ye(e,t,r,n={}){let g=Math.max(1,Math.min(n.maxBytes??52428800,104857600)),a=e,c=null;for(let p=0;p<=5;p++){let y=await L(a,{field:"media URL"});if(y.error||!y.parsed)throw new Error(y.error??"Media URL was rejected");if(c=await fetch(y.parsed.href,{signal:AbortSignal.timeout(15e3),redirect:"manual"}),c.status>=300&&c.status<400){let w=c.headers.get("location");if(!w)throw new Error(`HTTP ${c.status} redirect did not include Location`);a=new URL(w,y.parsed.href).href,c=null;continue}break}if(!c)throw new Error("Media download exceeded five redirects");if(!c.ok)throw new Error(`HTTP ${c.status}`);if(!c.body)throw new Error("Empty response body");let d=c.headers.get("content-type")?.split(";")[0].trim()??null;if(n.expectedType&&(!d||q(d)!==n.expectedType))throw new Error(`Expected ${n.expectedType} content but received ${d??"no content-type"}`);let i=Number(c.headers.get("content-length")??0);if(Number.isFinite(i)&&i>g)throw new Error(`Media asset exceeds ${g} byte limit`);let o=I(t,r);if(d&&!k(r)){let y={"image/jpeg":".jpg","image/png":".png","image/webp":".webp","image/gif":".gif","image/svg+xml":".svg","image/avif":".avif","video/mp4":".mp4","video/webm":".webm","audio/mpeg":".mp3","audio/ogg":".ogg","audio/wav":".wav"}[d];y&&(o=o+y)}let h=Z(o);await new Promise((p,y)=>{h.once("open",()=>p()),h.once("error",y)});let l=0,u=new ie({transform(p,y,w){let _=p.length;if(l+_>g){w(new Error(`Media asset exceeds ${g} byte limit`));return}if(n.consumeBytes&&!n.consumeBytes(_)){w(new Error("Media export total byte limit exceeded"));return}l+=_,w(null,p)}});try{await ne(oe.fromWeb(c.body),u,h)}catch(p){throw h.destroy(),te(o,{force:!0}),p}let{statSync:s}=await import("fs"),m=s(o).size;return{savedPath:o,sizeBytes:m,mimeType:d}}async function Le(e,t,r={}){let n=r.types??["image","video","audio"],g=new Set(n),a=he(e,t),c=Math.max(1,Math.min(r.maxAssets??100,250)),d=fe(a,r.rendered,2e3),i=d.slice(0,c),o=new Set([...a.map(f=>f.url),...(r.rendered?.assets??[]).map(f=>f.url)]).size,h=i.flatMap(f=>{let b=f.type??F(f.url);return b&&g.has(b)?[{...f,type:b}]:[]}),l=o-h.length,u=d.length>c,s=r.rendered?.exhausted===!0&&!u,m=u?"asset_limit":r.rendered?.stopReason??"render_unavailable",p=u?[`Media discovery found ${d.length} responsive families; only the requested ${c} records were retained.`]:r.rendered?r.rendered.exhausted?[]:[`Rendered media discovery stopped with ${r.rendered.stopReason}; the inventory may be incomplete.`]:["Rendered media discovery was unavailable; this manifest contains static-source evidence only."],y={pageUrl:t,staticFound:a.length,renderedFound:r.rendered?.assets.length??0,filteredCount:l,totalFound:o,completeness:s?"complete":"partial",exhausted:s,stopReason:m,scrollRounds:r.rendered?.scrollRounds??0,warnings:p,artifact:null};if(r.outputDir===null)return{...y,outputDir:null,assets:h.map((f,b)=>({...f,mimeType:null,filename:U(f.url,b),savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"not_attempted",downloadError:null,inlinePreview:null}))};let w=(()=>{try{return new URL(t).hostname.replace(/^www\./,"")}catch{return"unknown"}})(),_=new Date().toISOString().replace(/[:.]/g,"-").slice(0,19),C=r.outputDir??I(re(),"Downloads","mcp-scraper","media",`${_}-${w}`);ee(C,{recursive:!0});let T=[];return await Promise.allSettled(h.map(async(f,b)=>{let R=U(f.url,b);try{let{savedPath:v,sizeBytes:Q,mimeType:D}=await ye(f.url,C,R),G=D?q(D)??f.type:f.type;T.push({...f,type:G,mimeType:D,filename:B(v),savedPath:v,sizeBytes:Q,sha256:null,downloadStatus:"downloaded",downloadError:null,inlinePreview:null})}catch(v){T.push({...f,mimeType:null,filename:R,savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"failed",downloadError:v instanceof Error?v.message:"media_download_failed",inlinePreview:null})}})),T.sort((f,b)=>f.savedPath&&!b.savedPath?-1:!f.savedPath&&b.savedPath?1:f.url.localeCompare(b.url)),{...y,outputDir:C,assets:T}}var be=new Set(["browser_navigation_blocked","browser_navigation_failed","browser_timeout","browser_session_failed","browser_result_missing"]);function we(e){return e instanceof Error?e.code:void 0}function _e(e){return e instanceof Error?e.message:String(e)}function H(e){let t=e?.match(/^browser_http_(\d+)$/);return t?Number(t[1]):null}function ve(e){return e===404?"page_not_found":e===403?"page_forbidden":e===429?"page_rate_limited":e>=500&&e<=599?"page_server_error":"page_http_error"}function xe(e){let t=_e(e),r=we(e);if(N(e,t))return{errorCode:"vendor_unavailable",httpStatus:null,retryable:!0};let n=H(r)??H(t.match(/^(browser_http_\d+)/)?.[1]);return n!=null?{errorCode:ve(n),httpStatus:n,retryable:n===429||n>=500}:r==="browser_challenge_unresolved"||/^browser_challenge_unresolved/.test(t)?{errorCode:"bot_check_unresolved",httpStatus:null,retryable:!0}:r==="browser_response_too_large"||/exceeds \d+ byte limit/i.test(t)?{errorCode:"page_too_large",httpStatus:null,retryable:!1}:/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(t)?{errorCode:"browser_session_interrupted",httpStatus:null,retryable:!0}:be.has(r??"")||/timeout|navigation_failed|ERR_/i.test(t)?{errorCode:"page_unreachable",httpStatus:null,retryable:!0}:{errorCode:"extraction_failed",httpStatus:null,retryable:!1}}function j(e,t){switch(e){case"page_not_found":return`The page could not be found (HTTP ${t??404}). It may have been moved or deleted.`;case"page_forbidden":return`The site refused the request (HTTP ${t??403}).`;case"page_rate_limited":return"The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";case"page_server_error":return`The page's own server returned an error (HTTP ${t}). This is on the target site's side, not something a retry here can fix.`;case"page_http_error":return`The page returned an unexpected HTTP ${t} response.`;case"bot_check_unresolved":return"This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";case"page_too_large":return"The page is larger than we can safely process.";case"page_unreachable":return"The page did not respond in time, or the connection could not be completed.";case"browser_session_interrupted":return"The browser session closed before extraction completed. A retry uses a fresh browser session.";case"vendor_unavailable":return"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";case"extraction_failed":return"The page could not be extracted. Please retry, or contact support if this persists for the same URL."}}function Ie(e,t={}){let r=j(e.errorCode,e.httpStatus);return{...$({errorCode:e.errorCode,retryable:e.retryable,chargeStatus:t.chargeStatus,details:{...t.details??{},...e.httpStatus!=null?{http_status:e.httpStatus}:{}}}),message:r,error:r}}function ke(e){return e.httpStatus===404?404:e.httpStatus===403?403:e.httpStatus===429?429:e.httpStatus!=null&&e.httpStatus>=500&&e.httpStatus<=599?500:e.httpStatus!=null?502:e.errorCode==="vendor_unavailable"?503:502}var Ee="wayback_capture_missing";function Be(e,t){if(e==null)return{failureCode:null,failureReason:null};if(e===Ee)return{failureCode:e,failureReason:t??null};let r=new Error(t??e);r.code=e;let n=xe(r);return{failureCode:n.errorCode,failureReason:j(n.errorCode,n.httpStatus)}}import{neon as Te}from"@neondatabase/serverless";var Se="jina",M=null,z=!1;function W(){return(process.env.JINA_EMBED_MODEL??"jina-embeddings-v5-omni-small").trim()}function Y(){return Number((process.env.JINA_EMBED_DIM??"1024").trim())}function J(){return!!(process.env.JINA_API_KEY?.trim()&&process.env.MEMORY_DATABASE_URL?.trim())}function E(){if(M)return M;let e=process.env.MEMORY_DATABASE_URL?.trim();if(!e)throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");return M=Te(e),M}async function K(){if(z)return;let e=Y();await E().query("CREATE EXTENSION IF NOT EXISTS vector"),await E().query(`
     CREATE TABLE IF NOT EXISTS commons_index_vectors (
       document_id   TEXT PRIMARY KEY,