mcp-scraper 0.97.0 → 0.98.0

package/dist/chunk-HCK6PLM3.js

@@ -1,3 +0,0 @@
-import{a as w}from"./chunk-JMZIDHYQ.js";import{e as v,j as T}from"./chunk-SEZETSB3.js";import{i as I}from"./chunk-SDQW7IJN.js";import{b as y}from"./chunk-B3JRL45T.js";import{c as E}from"./chunk-OILOHQNL.js";import{createHash as L,randomBytes as N}from"crypto";import{basename as D,extname as O,join as j}from"path";import k from"p-limit";import{ZipFile as $}from"yazl";var P="page-media/",z=10080*60*1e3,W=900*1e3,_=10*1024*1024,G=45*1024*1024,X=15e5,K=25e5,V=6,B=5;function S(){return process.env.PAGE_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function C(){return{prefix:P,artifactTtlMs:z,downloadTtlMs:W,token:S()}}function Y(e,r){if(e.length>=3&&e[0]===255&&e[1]===216&&e[2]===255)return{mimeType:"image/jpeg",extension:"jpg"};if(e.length>=8&&e[0]===137&&e[1]===80&&e[2]===78&&e[3]===71)return{mimeType:"image/png",extension:"png"};if(e.length>=12&&String.fromCharCode(...e.slice(0,4))==="RIFF"&&String.fromCharCode(...e.slice(8,12))==="WEBP")return{mimeType:"image/webp",extension:"webp"};if(e.length>=6&&["GIF87a","GIF89a"].includes(String.fromCharCode(...e.slice(0,6))))return{mimeType:"image/gif",extension:"gif"};if(e.length>=12&&String.fromCharCode(...e.slice(4,12)).match(/^ftyp(?:avif|avis)$/))return{mimeType:"image/avif",extension:"avif"};let t=Buffer.from(e.slice(0,Math.min(e.length,1024))).toString("utf8").replace(/^\uFEFF/,"").trimStart();return(r==="image/svg+xml"||t.startsWith("<svg")||/^<\?xml[\s\S]*?<svg/i.test(t))&&/<svg[\s>]/i.test(t)?{mimeType:"image/svg+xml",extension:"svg"}:null}function H(e,r,t){if(t==="image")return Y(e,r);if(e.length>=12&&String.fromCharCode(...e.slice(4,8))==="ftyp")return{mimeType:"video/mp4",extension:"mp4"};if(e.length>=4&&e[0]===26&&e[1]===69&&e[2]===223&&e[3]===163)return{mimeType:"video/webm",extension:"webm"};if(e.length>=4&&String.fromCharCode(...e.slice(0,4))==="OggS")return t==="audio"?{mimeType:"audio/ogg",extension:"ogg"}:{mimeType:"video/ogg",extension:"ogv"};if(t==="audio"&&e.length>=3&&String.fromCharCode(...e.slice(0,3))==="ID3")return{mimeType:"audio/mpeg",extension:"mp3"};if(r&&w(r)===t){let i=r.split("/")[1]?.replace("jpeg","jpg").replace("mpeg","mp3").replace("svg+xml","svg");if(i&&/^[a-z0-9]+$/i.test(i))return{mimeType:r,extension:i}}return null}function J(e,r){let t=Buffer.from(e);if(r==="image/png"&&t.length>=24)return{width:t.readUInt32BE(16),height:t.readUInt32BE(20)};if(r==="image/gif"&&t.length>=10)return{width:t.readUInt16LE(6),height:t.readUInt16LE(8)};if(r==="image/webp"&&t.length>=30){let i=t.toString("ascii",12,16);if(i==="VP8X")return{width:1+t.readUIntLE(24,3),height:1+t.readUIntLE(27,3)};if(i==="VP8L"&&t[20]===47){let a=t.readUInt32LE(21);return{width:(a&16383)+1,height:(a>>14&16383)+1}}}if(r==="image/jpeg"){let i=2;for(;i+9<t.length;){if(t[i]!==255){i+=1;continue}let a=t[i+1];if(a===216||a===217){i+=2;continue}let o=t.readUInt16BE(i+2);if(o<2||i+o+2>t.length)break;if([192,193,194,195,197,198,199,201,202,203,205,206,207].includes(a))return{height:t.readUInt16BE(i+5),width:t.readUInt16BE(i+7)};i+=o+2}}if(r==="image/svg+xml"){let a=t.toString("utf8",0,Math.min(t.length,65536)).match(/<svg\b[^>]*>/i)?.[0]??"",o=Number(a.match(/\bwidth=["']?([\d.]+)/i)?.[1]??0),d=Number(a.match(/\bheight=["']?([\d.]+)/i)?.[1]??0);if(o>0&&d>0)return{width:Math.round(o),height:Math.round(d)};let c=a.match(/\bviewBox=["']([^"']+)["']/i)?.[1]?.trim().split(/[\s,]+/).map(Number);if(c?.length===4&&c[2]>0&&c[3]>0)return{width:Math.round(c[2]),height:Math.round(c[3])}}return null}async function Z(e){let r=Number(e.headers.get("content-length")??"");if(Number.isFinite(r)&&r>_)throw new Error("media_too_large");if(!e.body){let o=Buffer.from(await e.arrayBuffer());if(o.length>_)throw new Error("media_too_large");return o}let t=e.body.getReader(),i=[],a=0;try{for(;;){let{done:o,value:d}=await t.read();if(o)break;if(d){if(a+=d.byteLength,a>_)throw new Error("media_too_large");i.push(Buffer.from(d))}}}finally{t.releaseLock()}return Buffer.concat(i)}async function q(e,r){let t=e;for(let i=0;i<=B;i+=1){let a=await y(t,{field:"website media URL"});if(a.error||!a.parsed)throw new Error("media_url_rejected");let o=await fetch(a.parsed.href,{redirect:"manual",headers:{Accept:r==="image"?"image/*":`${r}/*`},signal:AbortSignal.timeout(15e3)});if([301,302,303,307,308].includes(o.status)){let s=o.headers.get("location");if(!s||i===B)throw new Error("media_redirect_rejected");t=new URL(s,a.parsed).href;continue}if(!o.ok)throw new Error(`media_fetch_${o.status}`);let d=await Z(o),c=o.headers.get("content-type")?.split(";")[0].trim().toLowerCase()??null,m=H(d,c,r);if(!m||w(m.mimeType)!==r)throw new Error("media_content_invalid");let f=r==="image"?J(d,m.mimeType):null;return{bytes:d,...m,finalUrl:a.parsed.href,width:f?.width??null,height:f?.height??null,sha256:L("sha256").update(d).digest("hex")}}throw new Error("media_redirect_rejected")}function Q(e){return new Promise((r,t)=>{let i=new $,a=[];i.outputStream.on("data",o=>a.push(Buffer.from(o))),i.outputStream.once("error",t),i.outputStream.once("end",()=>r(Buffer.concat(a)));for(let o of e)i.addBuffer(o.content,o.path);i.end()})}function M(e){return e.toLowerCase().replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,70)||"website"}function ee(e){try{return D(new URL(e).pathname).replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)}catch{return""}}function te(e){return I(e,P)}async function me(e){return te(e.artifactId)!==e.ownerId?null:T({policy:C(),artifactId:e.artifactId,maxBytes:60*1024*1024})}async function fe(e){let r=[],t=k(V),i=new Map,a=0,o=0,d=0;await Promise.all(e.media.assets.map((n,h)=>t(async()=>{try{let l=await q(n.url,n.type);n.finalUrl=l.finalUrl,n.mimeType=l.mimeType,n.sizeBytes=l.bytes.length,n.sha256=l.sha256,n.width=l.width??n.width,n.height=l.height??n.height;let g=i.get(l.sha256);if(g){n.filename=g,n.duplicateOf=g,n.downloadStatus="downloaded",n.downloadError=null;return}if(a+l.bytes.length>G)throw new Error("archive_byte_limit_reached");a+=l.bytes.length;let u=ee(l.finalUrl),A=O(u),F=u?A?u.slice(0,-A.length):u:n.type,p=`media/${String(h+1).padStart(4,"0")}-${M(F)}.${l.extension}`;i.set(l.sha256,p),r.push({path:p,content:l.bytes}),n.filename=p,n.duplicateOf=null,n.downloadStatus="downloaded",n.downloadError=null,n.type==="image"&&d<e.maxInlineImages&&l.bytes.length<=X&&o+l.bytes.length<=K&&(n.inlinePreview={data:l.bytes.toString("base64"),mimeType:l.mimeType},o+=l.bytes.length,d+=1)}catch(l){n.finalUrl=null,n.mimeType=null,n.sizeBytes=null,n.sha256=null,n.savedPath=null,n.downloadStatus="failed",n.downloadError=l instanceof Error?l.message:"media_download_failed",n.inlinePreview=null}})));let c=e.media.assets.filter(n=>n.downloadStatus==="downloaded").length,m=e.media.assets.length-c;if(m>0&&e.media.warnings.push(`${m} media asset(s) could not be downloaded; media.jsonl contains bounded error codes.`),c===0)return{media:e.media,artifact:null};let f=e.media.assets.map(({inlinePreview:n,...h})=>h),s={pageUrl:e.pageUrl,title:e.title,extractedAt:e.extractedAt,completeness:e.media.completeness,exhausted:e.media.exhausted,stopReason:e.media.stopReason,scrollRounds:e.media.scrollRounds,staticFound:e.media.staticFound,renderedFound:e.media.renderedFound,totalFound:e.media.totalFound,retainedAfterFilteringAndVariantCollapse:e.media.assets.length,downloaded:c,failed:m,notes:["The inventory unions static-source and rendered-page discovery.","complete means rendered scrolling reached three stable document-height checks; it does not claim every interactive control was opened.","Media records preserve discovery evidence but do not assert ownership, rights, identity, or publishability."]};r.push({path:"summary.json",content:Buffer.from(JSON.stringify(s,null,2))}),r.push({path:"media.jsonl",content:Buffer.from(f.map(n=>JSON.stringify(n)).join(`
-`)+`
-`)});let R=await Q(r),U=N(6).toString("hex"),x=await v({policy:C(),ownerId:e.ownerId,artifactKey:`${U}.zip`,createdAt:new Date(e.extractedAt),filename:`${M(e.title||new URL(e.pageUrl).hostname)}-website-media.zip`,contentType:"application/zip",content:R}),b=S()?null:j(E(),"blobs",x.artifactId);return{media:e.media,artifact:{...x,localPath:b}}}export{P as a,z as b,C as c,te as d,me as e,fe as f};

package/dist/chunk-HNNYXMSI.js

@@ -1 +0,0 @@
-import{createHash as s}from"crypto";var e=class extends Error{constructor(r){super(r),this.name="OperationInputError"}},o=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function u(t){return s("sha256").update(t).digest("hex")}function i(t){return Array.isArray(t)?t.map(i):t&&typeof t=="object"?Object.fromEntries(Object.entries(t).sort(([r],[n])=>r.localeCompare(n)).map(([r,n])=>[r,i(n)])):t}function l(t){return JSON.stringify(i(t??{}))}function d(t,r){let n=t.trim();if(!n)throw new Error(`${r} is required`);return n}function p(t,r){if(!Number.isSafeInteger(r.userId)||r.userId<=0||r.userId!==t.userId||!r.tool||r.tool!==t.tool||!r.jobId||r.jobId!==t.jobId||typeof r.sql!="string"||!r.sql.trim()||r.sql.length>64*1024||!Array.isArray(r.args)||r.args.length>512)throw new e("Store gate must match the operation principal, tool and job")}export{e as a,o as b,u as c,l as d,d as e,p as f};

package/dist/chunk-IFPION5D.js

@@ -1,121 +0,0 @@
-import{a as I}from"./chunk-7POHNTVW.js";import{b as S}from"./chunk-4ZVO6GX4.js";import{a as R}from"./chunk-WNUXMPVX.js";import{randomUUID as X}from"crypto";import{createMiddleware as b}from"hono/factory";async function j(e){await e.execute(`CREATE TABLE IF NOT EXISTS mcp_analytics_requests (
-    id TEXT PRIMARY KEY, parent_request_id TEXT, user_id INTEGER NOT NULL REFERENCES users(id),
-    service TEXT NOT NULL, method TEXT NOT NULL, http_status INTEGER NOT NULL,
-    duration_ms INTEGER NOT NULL, client_surface TEXT NOT NULL,
-    install_channel TEXT NOT NULL, ai_client TEXT, ai_model TEXT,
-    attribution_basis TEXT NOT NULL, started_at TEXT NOT NULL, deployment_commit TEXT
-  )`),(await e.execute("PRAGMA table_info(mcp_analytics_requests)")).rows.some(n=>n.name==="parent_request_id")||await e.execute("ALTER TABLE mcp_analytics_requests ADD COLUMN parent_request_id TEXT"),await e.execute("CREATE INDEX IF NOT EXISTS mcp_analytics_requests_user_started ON mcp_analytics_requests(user_id, started_at)"),await e.execute("CREATE INDEX IF NOT EXISTS mcp_analytics_requests_started ON mcp_analytics_requests(started_at)")}var k=new Set(["npx","npm","sdk","frontend","mcp-stdio","mcp-http","rest"]),D=new Set(["npx","npm","mcpb","source"]),v=new Set(["claude","claude-desktop","claude-code","chatgpt","codex","cursor","gemini","other"]);function y(e,t){return e&&t.has(e.toLowerCase())?e.toLowerCase():null}function A(e,t=!1){let n=y(e.get("x-mcp-client-surface")??void 0,k),s=e.get("x-mcp-ai-model")??"";return{surface:t?"frontend":n??"unknown",installChannel:y(e.get("x-mcp-install-channel")??void 0,D)??"unknown",aiClient:y(e.get("x-mcp-ai-client")??void 0,v),aiModel:/^[a-zA-Z0-9][a-zA-Z0-9._:/-]{0,79}$/.test(s)?s:null,basis:t?"session_authenticated":n?"caller_declared":"unknown"}}async function q(e,t=50){let n;try{await Promise.race([e,new Promise((s,a)=>{n=setTimeout(()=>a(new Error("analytics_write_budget_exceeded")),t)})])}finally{n&&clearTimeout(n)}}function F(e=2){let t=0;return async n=>{if(t>=e)return!1;t++;let s=Promise.resolve().then(n).finally(()=>{t--});return await q(s),!0}}var H=F();function z(e){return b(async(t,n)=>{let s=X(),a=new Date().toISOString(),o=performance.now(),r=new URL(t.req.url).pathname,d=e(r),i=d??(r==="/mcp"?"mcp_http":"other_authenticated_http"),f={op:i,probeRunId:t.req.header("x-cost-probe-id")??null,requestId:s},_=A(t.req.raw.headers),c={};_.surface!=="unknown"&&(c["x-mcp-client-surface"]=_.surface),new URL(t.req.url).pathname==="/mcp"&&!c["x-mcp-client-surface"]&&(c["x-mcp-client-surface"]="mcp-http"),_.installChannel!=="unknown"&&(c["x-mcp-install-channel"]=_.installChannel),_.aiClient&&(c["x-mcp-ai-client"]=_.aiClient),_.aiModel&&(c["x-mcp-ai-model"]=_.aiModel);let l=t.req.header("x-mcp-parent-request-id"),N=l&&/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(l)?l:null;return I({requestId:s,headers:c},()=>S(f,async()=>{t.header("x-mcp-request-id",s);try{await n()}finally{t.header("x-mcp-request-id",s);let T=t.get("user"),x=t.get("sessionUser"),O=Number(T?.id??x?.id),w=/\/(?:jobs|status|progress|events|history|balance|usage)(?:\/|$)/.test(r);if(process.env.MCP_ANALYTICS_CAPTURE==="1"&&!w&&t.req.method!=="HEAD"&&t.req.method!=="OPTIONS"&&(d!==null||r==="/mcp"&&t.req.method==="POST")&&Number.isSafeInteger(O)&&O>0){let E=A(t.req.raw.headers,!!x?.id);new URL(t.req.url).pathname==="/mcp"&&E.surface==="unknown"&&(E.surface="mcp-http",E.basis="hosted_mcp_route");try{await H(()=>R().execute({sql:`INSERT INTO mcp_analytics_requests
-                (id,parent_request_id,user_id,service,method,http_status,duration_ms,client_surface,install_channel,
-                 ai_client,ai_model,attribution_basis,started_at,deployment_commit)
-                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[s,N,O,i,t.req.method,t.error?500:t.res.status,Math.max(0,Math.round(performance.now()-o)),E.surface,E.installChannel,E.aiClient,E.aiModel,E.basis,a,process.env.VERCEL_GIT_COMMIT_SHA??null]}))}catch{console.warn("[mcp-analytics] request observation unavailable")}}}}))})}var m=["oauth_store_metadata","oauth_clients","oauth_client_redirect_uris","oauth_codes","oauth_code_exchanges","oauth_refresh_families","oauth_refresh_tokens","oauth_refresh_rotations","oauth_security_events","oauth_rate_limit_buckets"],h=["oauth_clients_inactivity_idx","oauth_clients_cleanup_idx","oauth_codes_expiry_idx","oauth_codes_client_idx","oauth_code_exchanges_replay_expiry_idx","oauth_refresh_families_lifecycle_idx","oauth_refresh_families_user_status_idx","oauth_refresh_families_client_idx","oauth_refresh_families_cleanup_idx","oauth_refresh_tokens_expiry_idx","oauth_refresh_rotations_replay_expiry_idx","oauth_security_events_retention_idx","oauth_rate_limit_buckets_expiry_idx"],C={oauth_clients:"oauth_clients_legacy_raw",oauth_codes:"oauth_codes_legacy_raw",oauth_tokens:"oauth_tokens_legacy_raw"},u="strftime('%Y-%m-%dT%H:%M:%fZ','now')";async function p(e,t,n=!1){try{await e.execute(t)}catch(s){let a=s.code;if(!n||a!=="SQL_PARSE_ERROR")throw s}}var M=[`CREATE TABLE IF NOT EXISTS oauth_store_metadata (
-    id INTEGER PRIMARY KEY CHECK (id = 1),
-    store_id TEXT NOT NULL UNIQUE,
-    schema_version INTEGER NOT NULL CHECK (schema_version > 0),
-    authority_epoch INTEGER NOT NULL CHECK (authority_epoch > 0),
-    authority_state TEXT NOT NULL CHECK (authority_state IN ('ready','authoritative','retired')),
-    updated_at TEXT NOT NULL DEFAULT (${u})
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_clients (
-    client_id TEXT PRIMARY KEY,
-    client_name TEXT NOT NULL,
-    client_type TEXT NOT NULL CHECK (client_type = 'public'),
-    token_endpoint_auth_method TEXT NOT NULL CHECK (token_endpoint_auth_method = 'none'),
-    status TEXT NOT NULL CHECK (status IN ('active','disabled','expired')),
-    created_at TEXT NOT NULL DEFAULT (${u}),
-    last_used_at TEXT,
-    expires_at TEXT
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_client_redirect_uris (
-    client_id TEXT NOT NULL REFERENCES oauth_clients(client_id) ON DELETE CASCADE,
-    redirect_uri TEXT NOT NULL,
-    redirect_kind TEXT NOT NULL CHECK (redirect_kind IN ('https','loopback','private_scheme')),
-    PRIMARY KEY (client_id, redirect_uri)
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_codes (
-    code_digest BLOB PRIMARY KEY,
-    lookup_key_id TEXT NOT NULL,
-    client_id TEXT NOT NULL REFERENCES oauth_clients(client_id) ON DELETE RESTRICT,
-    redirect_uri TEXT NOT NULL,
-    code_challenge TEXT NOT NULL,
-    challenge_method TEXT NOT NULL CHECK (challenge_method = 'S256'),
-    user_id TEXT NOT NULL,
-    scope TEXT NOT NULL,
-    resource TEXT,
-    issued_at TEXT NOT NULL DEFAULT (${u}),
-    expires_at TEXT NOT NULL
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_code_exchanges (
-    code_digest BLOB PRIMARY KEY,
-    request_fingerprint BLOB NOT NULL UNIQUE,
-    client_id TEXT NOT NULL,
-    family_id TEXT NOT NULL UNIQUE,
-    grant_scope TEXT NOT NULL,
-    grant_resource TEXT,
-    scope TEXT NOT NULL,
-    resource TEXT,
-    sealed_refresh BLOB NOT NULL,
-    sealed_nonce BLOB NOT NULL,
-    sealed_tag BLOB NOT NULL,
-    replay_key_id TEXT NOT NULL,
-    completed_at TEXT NOT NULL DEFAULT (${u}),
-    replay_expires_at TEXT NOT NULL
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_refresh_families (
-    family_id TEXT PRIMARY KEY,
-    user_id TEXT NOT NULL,
-    client_id TEXT NOT NULL REFERENCES oauth_clients(client_id) ON DELETE RESTRICT,
-    issuer TEXT NOT NULL,
-    scope TEXT NOT NULL,
-    resource TEXT,
-    status TEXT NOT NULL CHECK (status IN ('active','revoked','compromised','expired')),
-    current_generation INTEGER NOT NULL CHECK (current_generation > 0),
-    created_at TEXT NOT NULL DEFAULT (${u}),
-    expires_at TEXT NOT NULL,
-    revoked_at TEXT,
-    compromised_at TEXT,
-    UNIQUE (family_id, client_id),
-    FOREIGN KEY (family_id, current_generation)
-      REFERENCES oauth_refresh_tokens(family_id, generation)
-      DEFERRABLE INITIALLY DEFERRED
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_refresh_tokens (
-    family_id TEXT NOT NULL REFERENCES oauth_refresh_families(family_id) ON DELETE CASCADE,
-    generation INTEGER NOT NULL CHECK (generation > 0),
-    token_digest BLOB NOT NULL UNIQUE,
-    lookup_key_id TEXT NOT NULL,
-    token_prefix TEXT NOT NULL,
-    predecessor_generation INTEGER,
-    issued_at TEXT NOT NULL DEFAULT (${u}),
-    expires_at TEXT NOT NULL,
-    used_at TEXT,
-    revoked_at TEXT,
-    PRIMARY KEY (family_id, generation),
-    CHECK (predecessor_generation IS NULL OR (predecessor_generation > 0 AND predecessor_generation < generation))
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_refresh_rotations (
-    family_id TEXT NOT NULL REFERENCES oauth_refresh_families(family_id) ON DELETE CASCADE,
-    predecessor_generation INTEGER NOT NULL,
-    request_fingerprint BLOB NOT NULL,
-    successor_generation INTEGER NOT NULL,
-    sealed_successor BLOB NOT NULL,
-    sealed_nonce BLOB NOT NULL,
-    sealed_tag BLOB NOT NULL,
-    replay_key_id TEXT NOT NULL,
-    rotated_at TEXT NOT NULL DEFAULT (${u}),
-    replay_expires_at TEXT NOT NULL,
-    PRIMARY KEY (family_id, predecessor_generation),
-    UNIQUE (family_id, successor_generation),
-    CHECK (successor_generation = predecessor_generation + 1)
-  ) STRICT`,`CREATE TABLE IF NOT EXISTS oauth_security_events (
-    event_id TEXT PRIMARY KEY,
-    family_id TEXT NOT NULL,
-    event_type TEXT NOT NULL CHECK (event_type IN ('reuse','binding_mismatch','manual_revoke','expiry')),
-    reason_code TEXT NOT NULL,
-    occurred_at TEXT NOT NULL DEFAULT (${u}),
-    retain_until TEXT NOT NULL
-  ) STRICT`,"CREATE INDEX IF NOT EXISTS oauth_clients_inactivity_idx ON oauth_clients(status, last_used_at, client_id)","CREATE INDEX IF NOT EXISTS oauth_codes_expiry_idx ON oauth_codes(expires_at, code_digest)","CREATE INDEX IF NOT EXISTS oauth_code_exchanges_replay_expiry_idx ON oauth_code_exchanges(replay_expires_at, code_digest)","CREATE INDEX IF NOT EXISTS oauth_refresh_families_lifecycle_idx ON oauth_refresh_families(status, expires_at, family_id)","CREATE INDEX IF NOT EXISTS oauth_refresh_families_user_status_idx ON oauth_refresh_families(user_id, status)","CREATE INDEX IF NOT EXISTS oauth_refresh_tokens_expiry_idx ON oauth_refresh_tokens(expires_at, family_id, generation)","CREATE INDEX IF NOT EXISTS oauth_refresh_rotations_replay_expiry_idx ON oauth_refresh_rotations(replay_expires_at, family_id, predecessor_generation)","CREATE INDEX IF NOT EXISTS oauth_security_events_retention_idx ON oauth_security_events(retain_until, event_id)"],U=["CREATE INDEX IF NOT EXISTS oauth_clients_cleanup_idx ON oauth_clients(COALESCE(last_used_at,created_at), client_id)","CREATE INDEX IF NOT EXISTS oauth_codes_client_idx ON oauth_codes(client_id, code_digest)","CREATE INDEX IF NOT EXISTS oauth_refresh_families_client_idx ON oauth_refresh_families(client_id, family_id)",`CREATE INDEX IF NOT EXISTS oauth_refresh_families_cleanup_idx
-    ON oauth_refresh_families(COALESCE(compromised_at,revoked_at,expires_at), family_id)
-    WHERE status!='active'`],g=[`CREATE TABLE IF NOT EXISTS oauth_rate_limit_buckets (
-    scope TEXT NOT NULL CHECK (length(scope) BETWEEN 1 AND 80),
-    key_digest BLOB NOT NULL CHECK (typeof(key_digest) = 'blob' AND length(key_digest) = 32),
-    window_start INTEGER NOT NULL CHECK (window_start >= 0),
-    window_seconds INTEGER NOT NULL CHECK (window_seconds BETWEEN 1 AND 604800),
-    count INTEGER NOT NULL CHECK (count >= 1),
-    expires_at INTEGER NOT NULL CHECK (expires_at = window_start + window_seconds),
-    updated_at TEXT NOT NULL DEFAULT (${u}),
-    PRIMARY KEY (scope, key_digest, window_start, window_seconds)
-  ) STRICT`,`CREATE INDEX IF NOT EXISTS oauth_rate_limit_buckets_expiry_idx
-    ON oauth_rate_limit_buckets(expires_at, scope, key_digest, window_start, window_seconds)`],L=[...M,...U,...g];async function B(e){if(await p(e,"PRAGMA foreign_keys = ON"),await p(e,"PRAGMA busy_timeout = 5000",!0),(await e.execute("SELECT name FROM pragma_table_info('oauth_codes')")).rows.some(r=>String(r.name)==="code"))throw new Error("Legacy raw OAuth tables require the explicit maintenance-mode coexistence migration");let s=(await e.execute("SELECT 1 present FROM sqlite_master WHERE type='table' AND name='oauth_store_metadata'")).rows.length>0?await e.execute("SELECT schema_version FROM oauth_store_metadata WHERE id=1"):void 0,a=s?.rows[0]?.schema_version==null?void 0:Number(s.rows[0].schema_version);if(a!==void 0&&(!Number.isSafeInteger(a)||a<1||a>3))throw new Error("OAuth store schema version cannot be migrated by this runtime");let o=a===void 0?L:a===1?[...U,...g]:a===2?g:[];for(let r of o)await e.execute(r);if(a!==void 0&&a<3&&(await e.execute({sql:`UPDATE oauth_store_metadata SET schema_version=?,updated_at=${u} WHERE id=1 AND schema_version=?`,args:[3,a]})).rowsAffected!==1)throw new Error("OAuth store schema version changed during migration");await p(e,"PRAGMA optimize",!0)}async function Z(e){await p(e,"PRAGMA foreign_keys = ON"),await p(e,"PRAGMA busy_timeout = 5000",!0);let t=await e.execute("SELECT name FROM pragma_table_info('oauth_codes')");if(!t.rows.some(o=>String(o.name)==="code")){let o=t.rows.some(r=>String(r.name)==="code_digest");return await B(e),o?"current":"created"}if((await e.execute({sql:"SELECT name FROM sqlite_master WHERE type='table' AND name IN (?,?,?)",args:Object.values(C)})).rows.length>0)throw new Error("Legacy OAuth archive names already exist; refusing ambiguous migration");let a=await e.transaction("write");try{for(let[o,r]of Object.entries(C))(await a.execute({sql:"SELECT 1 present FROM sqlite_master WHERE type='table' AND name=?",args:[o]})).rows.length>0&&await a.execute({sql:`ALTER TABLE ${o} RENAME TO ${r}`,args:[]});for(let o of L)await a.execute({sql:o,args:[]});await a.commit()}finally{a.close()}return"archived_legacy"}var P={oauth_store_metadata:["id","store_id","schema_version","authority_epoch","authority_state","updated_at"],oauth_clients:["client_id","client_name","client_type","token_endpoint_auth_method","status","created_at","last_used_at","expires_at"],oauth_client_redirect_uris:["client_id","redirect_uri","redirect_kind"],oauth_codes:["code_digest","lookup_key_id","client_id","redirect_uri","code_challenge","challenge_method","user_id","scope","resource","issued_at","expires_at"],oauth_code_exchanges:["code_digest","request_fingerprint","client_id","family_id","grant_scope","grant_resource","scope","resource","sealed_refresh","sealed_nonce","sealed_tag","replay_key_id","completed_at","replay_expires_at"],oauth_refresh_families:["family_id","user_id","client_id","issuer","scope","resource","status","current_generation","created_at","expires_at","revoked_at","compromised_at"],oauth_refresh_tokens:["family_id","generation","token_digest","lookup_key_id","token_prefix","predecessor_generation","issued_at","expires_at","used_at","revoked_at"],oauth_refresh_rotations:["family_id","predecessor_generation","request_fingerprint","successor_generation","sealed_successor","sealed_nonce","sealed_tag","replay_key_id","rotated_at","replay_expires_at"],oauth_security_events:["event_id","family_id","event_type","reason_code","occurred_at","retain_until"],oauth_rate_limit_buckets:["scope","key_digest","window_start","window_seconds","count","expires_at","updated_at"]},K={oauth_clients_inactivity_idx:["status","last_used_at","client_id"],oauth_clients_cleanup_idx:[null,"client_id"],oauth_codes_expiry_idx:["expires_at","code_digest"],oauth_codes_client_idx:["client_id","code_digest"],oauth_code_exchanges_replay_expiry_idx:["replay_expires_at","code_digest"],oauth_refresh_families_lifecycle_idx:["status","expires_at","family_id"],oauth_refresh_families_user_status_idx:["user_id","status"],oauth_refresh_families_client_idx:["client_id","family_id"],oauth_refresh_families_cleanup_idx:[null,"family_id"],oauth_refresh_tokens_expiry_idx:["expires_at","family_id","generation"],oauth_refresh_rotations_replay_expiry_idx:["replay_expires_at","family_id","predecessor_generation"],oauth_security_events_retention_idx:["retain_until","event_id"],oauth_rate_limit_buckets_expiry_idx:["expires_at","scope","key_digest","window_start","window_seconds"]};async function ee(e){let t=await e.execute({sql:`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${m.map(()=>"?").join(",")})`,args:[...m]}),n=await e.execute({sql:`SELECT name FROM sqlite_master WHERE type = 'index' AND name IN (${h.map(()=>"?").join(",")})`,args:[...h]}),s=new Set(t.rows.map(i=>String(i.name))),a=new Set(n.rows.map(i=>String(i.name))),o=await e.execute("PRAGMA foreign_keys"),r=[],d=i=>i.replace(/\bIF\s+NOT\s+EXISTS\b/gi,"").replace(/\s+/g," ").trim().toLowerCase();for(let i of m){if(!s.has(i))continue;let _=(await e.execute(`PRAGMA table_info('${i}')`)).rows.map(T=>String(T.name));JSON.stringify(_)!==JSON.stringify(P[i])&&r.push(`${i}:columns`);let c=await e.execute({sql:"SELECT sql FROM sqlite_master WHERE type='table' AND name=?",args:[i]}),l=String(c.rows[0]?.sql??""),N=L.find(T=>d(T).startsWith(`create table ${i} `));(!N||d(l)!==d(N))&&r.push(`${i}:definition`)}for(let i of h){if(!a.has(i))continue;let _=(await e.execute(`PRAGMA index_info('${i}')`)).rows.map(T=>T.name==null?null:String(T.name));JSON.stringify(_)!==JSON.stringify(K[i])&&r.push(`${i}:columns`);let c=await e.execute({sql:"SELECT sql FROM sqlite_master WHERE type='index' AND name=?",args:[i]}),l=String(c.rows[0]?.sql??""),N=L.find(T=>d(T).startsWith(`create index ${i} `));(!N||d(l)!==d(N))&&r.push(`${i}:definition`)}return{ok:m.every(i=>s.has(i))&&h.every(i=>a.has(i))&&Number(o.rows[0]?.foreign_keys??0)===1&&r.length===0,missingTables:m.filter(i=>!s.has(i)),missingIndexes:h.filter(i=>!a.has(i)),foreignKeysEnabled:Number(o.rows[0]?.foreign_keys??0)===1,problems:r}}export{j as a,z as b,Z as c,ee as d};

package/dist/chunk-JMZIDHYQ.js

@@ -1,2 +0,0 @@
-import{a as D}from"./chunk-OLYPLX5C.js";import{b as R}from"./chunk-B3JRL45T.js";import{c as P}from"./chunk-OILOHQNL.js";import{createWriteStream as H,mkdirSync as W,rmSync as N}from"fs";import{join as A,extname as B,basename as z}from"path";import{pipeline as O}from"stream/promises";import{Readable as I,Transform as q}from"stream";var X=["doubleclick.net","googlesyndication.com","googletagmanager.com","google-analytics.com","googletagservices.com","adservice.google","googletag.","pagead2.googlesyndication","facebook.net/tr","connect.facebook.net","fbcdn.net/rsrc","analytics.twitter.com","static.ads-twitter.com","ads.twitter.com","t.co/i/adsct","pixel.advertising.com","hotjar.com","clarity.ms","quantserve.com","scorecardresearch.com","newrelic.com","nr-data.net","segment.io","segment.com","amplitude.com","mixpanel.com","heap.io","fullstory.com","moatads.com","criteo.com","adsrvr.org","rubiconproject.com","pubmatic.com","openx.net","appnexus.com","amazon-adsystem.com","media.net","yieldmo.com","triplelift.com","sharethrough.com","prebid.","smaato.net","indexworm.com","casalemedia.com","outbrain.com","taboola.com","revcontent.com","mgid.com","tawk.to","intercom.io","drift.com","hs-scripts.com","zopim.com","livechatinc.com","userlike.com","onetrust.com","cookielaw.org","cookieinformation.com","trustarc.com","/ads/","/ad/","/banner/","/banners/","/pixel/","/beacon/","/tracking/","/tracker/","/remarketing/","/conversion/","1x1.gif","spacer.gif","blank.gif","transparent.gif"],G=new Set([".jpg",".jpeg",".png",".webp",".gif",".avif",".svg",".tiff"]),K=new Set([".mp4",".webm",".mov",".avi",".m4v",".ogv",".mkv"]),V=new Set([".mp3",".wav",".ogg",".aac",".m4a",".flac",".opus"]);function Z(r){let e=r.toLowerCase();return X.some(t=>e.includes(t))}function J(r){return r.startsWith("data:")}function U(r){try{let e=B(new URL(r).pathname).toLowerCase();if(G.has(e))return"image";if(K.has(e))return"video";if(V.has(e))return"audio"}catch{}return null}function L(r){let e=r.toLowerCase();return e.startsWith("image/")?"image":e.startsWith("video/")?"video":e.startsWith("audio/")?"audio":null}function Q(r,e){if(!r||J(r))return null;try{return new URL(r,e).href}catch{return null}}function C(r,e){try{let t=new URL(r);return z(t.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`asset-${e}`}catch{return`asset-${e}`}}function k(r,e){let t=r?.replace(/&(?:amp|#0?38|#x26);/gi,"&").replace(/\s+/g," ").trim();return t?t.slice(0,e):null}function Y(r){let e=r.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);return e?{width:Number(e[1]),height:Number(e[2])}:null}function ee(r){try{let e=new URL(r);e.hash="",e.pathname=e.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i,"");for(let t of["w","width","h","height","resize"])e.searchParams.delete(t);return e.searchParams.sort(),e.href}catch{return r}}function T(r){let e=(r.width??0)*(r.height??0),t=r.discoveryMethods.some(f=>f.includes("lightbox"))?1e12:0,u=/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(r.url)?0:1e11;return t+u+e}function S(r,e){let t=r.get(e.url);if(!t){r.set(e.url,e);return}t.discoveryMethods=[...new Set([...t.discoveryMethods,...e.discoveryMethods])],t.altTexts=[...new Set([...t.altTexts,...e.altTexts])],t.contexts=[...new Set([...t.contexts,...e.contexts])],!t.type&&e.type&&(t.type=e.type),(e.width??0)*(e.height??0)>(t.width??0)*(t.height??0)&&(t.width=e.width,t.height=e.height)}function te(r,e){let t=new Map,u=(l,s,a=null,h,i,c)=>{let n=l?Q(l.trim().replace(/&/g,"&"),e):null;if(!n||Z(n))return;let d=c??Y(n);S(t,{url:n,type:a??U(n),discoveryMethods:[s],altTexts:h?[h]:[],contexts:i?[i]:[],width:d?.width??null,height:d?.height??null})},f=(l,s,a,h)=>{for(let i of(l??"").split(",")){let[c,n]=i.trim().split(/\s+/);if(!c)continue;let d=n?.match(/^(\d+)w$/)?.[1];u(c,s,a,h,null,d?{width:Number(d),height:0}:null)}},m=(l,s="")=>{let a=l.$;for(let i of l.images){let c=i.attributes,n=k(i.alt,500),d=i.width&&i.height?{width:i.width,height:i.height}:null;for(let g of["src","data-src","data-lazy-src","data-original","data-bg","data-background","data-bg-url","data-lazy-bg","data-echo"]){let y=c[g];u(y,`static-${g}${s}`,"image",n,null,d)}f(c.srcset,`static-srcset${s}`,"image",n),f(c["data-srcset"],`static-data-srcset${s}`,"image",n)}a("source").each((i,c)=>{let n=a(c),d=n.attr("type")??"",g=d.startsWith("video/")?"video":d.startsWith("audio/")?"audio":"image";u(n.attr("src"),`static-source${s}`,g),f(n.attr("srcset"),`static-source-srcset${s}`,g)}),a("video,audio").each((i,c)=>{let n=a(c),d=c.tagName.toLowerCase();u(n.attr("src"),`static-${d}${s}`,d),d==="video"&&u(n.attr("poster"),`static-video-poster${s}`,"image")}),a("meta").each((i,c)=>{let n=a(c),d=(n.attr("property")??n.attr("name")??"").toLowerCase();(d==="og:image"||d==="og:image:url"||d==="twitter:image"||d==="twitter:image:src")&&u(n.attr("content"),`static-${d}${s}`,"image")}),a("link[href]").each((i,c)=>{let n=a(c);(n.attr("rel")??"").toLowerCase().split(/\s+/).some(g=>g==="icon"||g==="apple-touch-icon")&&u(n.attr("href"),`static-icon${s}`,"image")}),a("svg image").each((i,c)=>{let n=a(c);u(n.attr("href")??n.attr("xlink:href"),`static-svg-image${s}`,"image")}),a("a[href]").has("img").each((i,c)=>{let n=a(c),d=n.attr("href");if(d&&(/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${n.attr("class")??""} ${n.attr("data-elementor-open-lightbox")??""}`)||/\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(d))){let g=k(n.find("img").first().attr("alt"),500);u(d,`static-lightbox-target${s}`,"image",g)}});let h=a("style").toArray().map(i=>a(i).text());a("[style]").each((i,c)=>{h.push(a(c).attr("style")??"")});for(let i of h.join(`
-`).matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?([^"')&]+)(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?\s*\)/gi))u(i[1],`static-css-url${s}`,"image");for(let i of l.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi))u(i[1],`static-bare-url${s}`)},o=typeof r=="string"?D(r):r;return m(o),o.$("script").each((l,s)=>{let a=o.$(s).text(),h=a.replace(/\\u003[cC]/g,"<").replace(/\\u003[eE]/g,">").replace(/\\u0026/gi,"&").replace(/\\["']/g,i=>i.slice(1)).replace(/\\\//g,"/");h!==a&&/<(?:img|source|video|audio)\b/i.test(h)&&m(D(h),"-json-unescaped")}),[...t.values()]}function re(r,e,t=100){let u=new Map;for(let m of r)S(u,m);for(let m of e?.assets??[])S(u,{...m,type:m.type});let f=new Map;for(let m of u.values()){let o=ee(m.url),l=f.get(o)??[];l.push(m),f.set(o,l)}return[...f.values()].map(m=>{let o=[...m].sort((s,a)=>T(a)-T(s)||s.url.localeCompare(a.url)),l={...o[0]};return l.discoveryMethods=[...new Set(m.flatMap(s=>s.discoveryMethods))],l.altTexts=[...new Set(m.flatMap(s=>s.altTexts))],l.contexts=[...new Set(m.flatMap(s=>s.contexts))],{...l,variants:o.slice(1).map(s=>s.url)}}).sort((m,o)=>T(o)-T(m)||m.url.localeCompare(o.url)).slice(0,Math.max(1,t))}async function ne(r,e,t,u={}){let f=Math.max(1,Math.min(u.maxBytes??52428800,104857600)),m=r,o=null;for(let g=0;g<=5;g++){let y=await R(m,{field:"media URL"});if(y.error||!y.parsed)throw new Error(y.error??"Media URL was rejected");if(o=await fetch(y.parsed.href,{signal:AbortSignal.timeout(15e3),redirect:"manual"}),o.status>=300&&o.status<400){let v=o.headers.get("location");if(!v)throw new Error(`HTTP ${o.status} redirect did not include Location`);m=new URL(v,y.parsed.href).href,o=null;continue}break}if(!o)throw new Error("Media download exceeded five redirects");if(!o.ok)throw new Error(`HTTP ${o.status}`);if(!o.body)throw new Error("Empty response body");let l=o.headers.get("content-type")?.split(";")[0].trim()??null;if(u.expectedType&&(!l||L(l)!==u.expectedType))throw new Error(`Expected ${u.expectedType} content but received ${l??"no content-type"}`);let s=Number(o.headers.get("content-length")??0);if(Number.isFinite(s)&&s>f)throw new Error(`Media asset exceeds ${f} byte limit`);let a=A(e,t);if(l&&!B(t)){let y={"image/jpeg":".jpg","image/png":".png","image/webp":".webp","image/gif":".gif","image/svg+xml":".svg","image/avif":".avif","video/mp4":".mp4","video/webm":".webm","audio/mpeg":".mp3","audio/ogg":".ogg","audio/wav":".wav"}[l];y&&(a=a+y)}let h=H(a);await new Promise((g,y)=>{h.once("open",()=>g()),h.once("error",y)});let i=0,c=new q({transform(g,y,v){let b=g.length;if(i+b>f){v(new Error(`Media asset exceeds ${f} byte limit`));return}if(u.consumeBytes&&!u.consumeBytes(b)){v(new Error("Media export total byte limit exceeded"));return}i+=b,v(null,g)}});try{await O(I.fromWeb(o.body),c,h)}catch(g){throw h.destroy(),N(a,{force:!0}),g}let{statSync:n}=await import("fs"),d=n(a).size;return{savedPath:a,sizeBytes:d,mimeType:l}}async function ue(r,e,t={}){let u=t.types??["image","video","audio"],f=new Set(u),m=te(r,e),o=Math.max(1,Math.min(t.maxAssets??100,250)),l=re(m,t.rendered,2e3),s=l.slice(0,o),a=new Set([...m.map(p=>p.url),...(t.rendered?.assets??[]).map(p=>p.url)]).size,h=s.flatMap(p=>{let w=p.type??U(p.url);return w&&f.has(w)?[{...p,type:w}]:[]}),i=a-h.length,c=l.length>o,n=t.rendered?.exhausted===!0&&!c,d=c?"asset_limit":t.rendered?.stopReason??"render_unavailable",g=c?[`Media discovery found ${l.length} responsive families; only the requested ${o} records were retained.`]:t.rendered?t.rendered.exhausted?[]:[`Rendered media discovery stopped with ${t.rendered.stopReason}; the inventory may be incomplete.`]:["Rendered media discovery was unavailable; this manifest contains static-source evidence only."],y={pageUrl:e,staticFound:m.length,renderedFound:t.rendered?.assets.length??0,filteredCount:i,totalFound:a,completeness:n?"complete":"partial",exhausted:n,stopReason:d,scrollRounds:t.rendered?.scrollRounds??0,warnings:g,artifact:null};if(t.outputDir===null)return{...y,outputDir:null,assets:h.map((p,w)=>({...p,mimeType:null,filename:C(p.url,w),savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"not_attempted",downloadError:null,inlinePreview:null}))};let v=(()=>{try{return new URL(e).hostname.replace(/^www\./,"")}catch{return"unknown"}})(),b=new Date().toISOString().replace(/[:.]/g,"-").slice(0,19),$=t.outputDir??A(P(),"media",`${b}-${v}`);W($,{recursive:!0});let M=[];return await Promise.allSettled(h.map(async(p,w)=>{let E=C(p.url,w);try{let{savedPath:x,sizeBytes:j,mimeType:_}=await ne(p.url,$,E),F=_?L(_)??p.type:p.type;M.push({...p,type:F,mimeType:_,filename:z(x),savedPath:x,sizeBytes:j,sha256:null,downloadStatus:"downloaded",downloadError:null,inlinePreview:null})}catch(x){M.push({...p,mimeType:null,filename:E,savedPath:null,sizeBytes:null,sha256:null,downloadStatus:"failed",downloadError:x instanceof Error?x.message:"media_download_failed",inlinePreview:null})}})),M.sort((p,w)=>p.savedPath&&!w.savedPath?-1:!p.savedPath&&w.savedPath?1:p.url.localeCompare(w.url)),{...y,outputDir:$,assets:M}}export{L as a,te as b,ne as c,ue as d};

package/dist/chunk-JNPM3U3L.js

@@ -1,189 +0,0 @@
-import{i as T,m as j}from"./chunk-KXJCGV3H.js";import{n as f,r as w}from"./chunk-ANQ4JJVI.js";import{d as E,e as C,i as p}from"./chunk-C7MGJUTU.js";import{a as o}from"./chunk-WNUXMPVX.js";import{createHash as U}from"crypto";import{gunzipSync as k,gzipSync as $}from"zlib";var H=10,m=20*1024*1024;function g(t){return U("sha256").update(t).digest("hex")}function W(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??g(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??g(t.acquiredHtml),markdownSha256:t.markdownSha256??g(t.bodyMarkdown)}}function A(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function I(t){let e=t.pages.flatMap(i=>{let c=W(i);return c?[c]:[]}),n=new Map,r=[],a=0,s=async()=>{if(!r.length)return;let i=A(r);if(i.length>m)throw new Error(`site export content chunk exceeds ${m} bytes`);let c=g(r.map(l=>l.pageId).join("\0")).slice(0,16),u=await T({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(a).padStart(4,"0")}-${c}`,content:$(i,{level:6})});for(let l of r)n.set(l.url,{artifactId:u.artifactId,artifactSha256:u.sha256,pageId:l.pageId,uncompressedBytes:i.length});a++,r=[]};for(let i of e){let c=[...r,i];if(r.length>0&&(c.length>H||A(c).length>m)&&await s(),r.push(i),A(r).length>m)throw new Error(`page ${i.pageId} exceeds the durable content chunk limit`)}return await s(),n}function Y(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(r&&r.digest!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let a=r?.chunk;if(!a){let i=await j({artifactId:n.artifactId,ownerId:t});if(!i)throw new Error("site export content chunk is missing or unauthorized");if(g(i)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let c=k(i);if(c.length>m)throw new Error("site export content chunk exceeds its read limit");if(a=JSON.parse(c.toString("utf8")),a.version!=="site-extract-content.v1"||!Array.isArray(a.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,{digest:n.artifactSha256,chunk:a});e.size>2;)e.delete(e.keys().next().value)}let s=a.pages.find(i=>i.pageId===n.pageId);if(!s)throw new Error("site export page is missing from its content chunk");if(g(s.html)!==s.htmlSha256||g(s.markdown)!==s.markdownSha256)throw new Error("site export page checksum mismatch");return s}}import{randomUUID as J}from"crypto";function x(t,e,n=!1){if(!e)return{sql:`NOT EXISTS (SELECT 1 FROM operation_executions x JOIN operation_runs r ON r.id=x.run_id
-    WHERE r.id='extract-site:' || site_extract_jobs.id AND r.tool='extract_site')`,args:[]};if(e.workflow.runId!==`extract-site:${t}`||n&&(!e.phase||e.recovery))return{sql:"0",args:[]};let r=f(e.workflow,{tool:"extract_site",jobId:t,recovery:e.recovery}),a="site_extract_jobs.user_id = ?";if(!e.phase)return{sql:`(${r.sql} AND ${a})`,args:[...r.args,e.workflow.userId]};if(e.phase.userId!==e.workflow.userId)return{sql:"0",args:[]};let s=f(e.phase,{tool:"extract_site_phase",jobId:t});return{sql:`(${r.sql} AND ${a} AND ${s.sql})`,args:[...r.args,e.workflow.userId,...s.args]}}function h(t="site_extract_jobs."){return`(${w(`${t}id`,`${t}user_id`,"extract_site")}
-    AND ${w(`${t}id`,`${t}user_id`,"extract_site_phase")})`}function y(){return w("site_extract_jobs.id","site_extract_jobs.user_id","extract_site_phase")}function D(t="site_extract_jobs."){return`(${t}status='running' OR (${t}status='pending' AND ${t}execution_generation=0 AND (
-    EXISTS (SELECT 1 FROM operation_runs r WHERE r.id='extract-site:' || ${t}id AND r.user_id=${t}user_id AND r.tool='extract_site')
-    OR COALESCE(json_extract(${t}options,'$.__websiteExecutionProtocol'),0) <> 1)))`}var q="xray_entitlement";function st(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function it(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),a=Number(n?t.done_urls??0:t.successful_urls??0),s=Number(t.failed_urls??Math.max(0,r-a)),i=t.options?JSON.parse(String(t.options)):{},c=i.__websiteExecutionProtocol;delete i.__websiteExecutionProtocol;let u={id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",executionGeneration:Number(t.execution_generation??0),refinalizeUntil:t.refinalize_until==null?null:String(t.refinalize_until),settlementPlan:t.settlement_plan_json?JSON.parse(String(t.settlement_plan_json)):null,settlementReceipt:t.settlement_receipt_json?JSON.parse(String(t.settlement_receipt_json)):null,dispatchClaimToken:t.dispatch_claim_token==null?null:String(t.dispatch_claim_token),dispatchClaimUntil:t.dispatch_claim_until==null?null:String(t.dispatch_claim_until),dispatchedAt:t.dispatched_at==null?null:String(t.dispatched_at),dispatchAttempts:Number(t.dispatch_attempts??0),dispatchError:t.dispatch_error==null?null:String(t.dispatch_error),startUrl:String(t.start_url??""),options:i,totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:a,failedUrls:s,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:C(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")};return c===1&&Object.defineProperty(u,"executionProtocol",{value:1,enumerable:!1}),u}async function v(t,e){let r=await o().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function ot(t){let n=await o().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
-            (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
-          VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify({...t.options,__websiteExecutionProtocol:1}),t.idempotencyKey,t.requestFingerprint]}),r=await v(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function P(t){let n=await o().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function ct(t=25){return(await o().execute({sql:`SELECT * FROM site_extract_jobs
-          WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')
-            AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))
-          ORDER BY updated_at ASC LIMIT ?`,args:[Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(r=>_(r))}var N=`(
-  EXISTS (SELECT 1 FROM billing_debits d
-    WHERE d.idempotency_key = json_extract(site_extract_jobs.options, '$.debitKey')
-      AND d.user_id = site_extract_jobs.user_id AND d.status = 'applied'
-      AND d.operation = 'extract_site_hold'
-      AND json_type(options, '$.heldMc') = 'integer'
-      AND json_extract(options, '$.heldMc') > 0
-      AND json_extract(options, '$.heldMc') <= 9007199254740991
-      AND d.amount_mc = json_extract(options, '$.heldMc')
-      AND COALESCE(json_extract(options, '$.billingClass'), 'credits') = 'credits')
-  OR (json_extract(options, '$.billingClass') = 'xray_entitlement'
-    AND json_type(options, '$.heldMc') IN ('integer', 'real')
-    AND json_extract(options, '$.heldMc') = 0 AND json_extract(options, '$.debitKey') IS NULL
-    AND json_extract(options, '$.xraySetupReceipt.billingClass') = 'xray_entitlement'
-    AND json_type(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 'false'
-    AND json_type(options, '$.xraySetupReceipt.heldMc') IN ('integer', 'real')
-    AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
-    AND json_type(options, '$.xraySetupReceipt.billedMc') IN ('integer', 'real')
-    AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0))`;async function ut(t){let e=await o().execute({sql:`SELECT * FROM site_extract_jobs WHERE id = ? AND user_id > 0
-      AND status IN ('pending', 'running') AND billed_mc IS NULL
-      AND execution_generation = 0 AND ${N}`,args:[t]});return e.rows[0]?_(e.rows[0]):null}async function F(t=25,e=2){let n=Math.max(1,Math.min(60,Math.round(e)));return(await o().execute({sql:`SELECT * FROM site_extract_jobs WHERE status = 'pending' AND billed_mc IS NULL
-      AND execution_generation = 0 AND dispatched_at IS NULL AND ${N}
-      AND (dispatch_claim_token IS NULL OR dispatch_claim_until <= datetime('now'))
-      AND (next_dispatch_at IS NULL OR next_dispatch_at <= datetime('now'))
-      AND updated_at <= datetime('now', ?) ORDER BY updated_at, id LIMIT ?`,args:[`-${n} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function lt(t=25){return(await o().execute({sql:`SELECT * FROM site_extract_jobs WHERE status IN ('pending', 'running') AND billed_mc IS NULL
-      AND execution_generation = 0 AND ${N}
-      AND (
-        (dispatched_at <= datetime('now', '-30 seconds') AND NOT EXISTS (SELECT 1 FROM operation_executions e WHERE e.run_id = 'extract-site:' || site_extract_jobs.id))
-        OR EXISTS (SELECT 1 FROM operation_executions e JOIN operation_runs r ON r.id = e.run_id
-          WHERE e.run_id = 'extract-site:' || site_extract_jobs.id AND r.status = 'running'
-            AND e.phase = 'claimed' AND e.expires_at_ms <= CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER))
-      ) ORDER BY updated_at, id LIMIT ?`,args:[Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(n=>_(n))}async function dt(t={}){let e=t.jobId?[t.jobId]:(await F(t.limit)).map(r=>r.id),n=[];for(let r of e){let a=J(),i=(await o().execute({sql:`UPDATE site_extract_jobs SET dispatch_claim_token = ?, dispatch_claim_until = datetime('now', ?),
-        dispatch_attempts = dispatch_attempts + 1, dispatch_error = NULL, next_dispatch_at = NULL,
-        updated_at = datetime('now')
-        WHERE id = ? AND status = 'pending' AND billed_mc IS NULL AND execution_generation = 0
-          AND dispatched_at IS NULL AND ${N}
-          AND (dispatch_claim_token IS NULL OR dispatch_claim_until <= datetime('now'))
-          AND (next_dispatch_at IS NULL OR next_dispatch_at <= datetime('now'))
-        RETURNING id, dispatch_claim_token, dispatch_attempts`,args:[a,`+${Math.max(15,Math.min(3600,Math.round(t.leaseSeconds??300)))} seconds`,r]})).rows[0];i&&n.push({id:String(i.id),claimToken:String(i.dispatch_claim_token),attempts:Number(i.dispatch_attempts)})}return n}async function _t(t,e){return(await o().execute({sql:`SELECT id FROM site_extract_jobs WHERE id = ? AND status = 'pending' AND billed_mc IS NULL
-      AND execution_generation = 0 AND dispatched_at IS NULL AND dispatch_claim_token = ?
-      AND dispatch_claim_until > datetime('now') AND ${N}`,args:[t,e]})).rows.length===1}async function gt(t,e){return(await o().execute({sql:`UPDATE site_extract_jobs SET dispatched_at = datetime('now'), dispatch_claim_token = NULL,
-      dispatch_claim_until = NULL, next_dispatch_at = NULL, dispatch_error = NULL, updated_at = datetime('now')
-      WHERE id = ? AND execution_generation = 0 AND dispatched_at IS NULL
-        AND dispatch_claim_token = ? AND dispatch_claim_until > datetime('now')`,args:[t,e]})).rowsAffected===1}async function pt(t,e,n,r){return(await o().execute({sql:`UPDATE site_extract_jobs SET dispatch_error = ?, next_dispatch_at = datetime('now', ?),
-      dispatch_claim_token = NULL, dispatch_claim_until = NULL, updated_at = datetime('now')
-      WHERE id = ? AND status = 'pending' AND billed_mc IS NULL AND execution_generation = 0
-        AND dispatched_at IS NULL AND dispatch_claim_token = ? AND dispatch_claim_until > datetime('now')`,args:[n.slice(0,2e3),`+${Math.max(15,Math.min(3600,Math.round(r)))} seconds`,t,e]})).rowsAffected===1}async function Et(t,e,n=0){await o().execute({sql:`UPDATE site_extract_jobs SET
-            settlement_attempts = settlement_attempts + 1,
-            settlement_error = ?,
-            next_settlement_at = datetime('now', '+15 minutes'),
-            updated_at = datetime('now')
-          WHERE id = ? AND billed_mc IS NULL AND execution_generation = ? AND status IN ('complete', 'partial', 'failed')`,args:[p(e).slice(0,1e3),t,n]})}function O(t=""){return`((${t}execution_generation = 0 AND ${t}updated_at <= datetime('now', ?))
-    OR (${t}execution_generation > 0 AND ${t}refinalize_until <= datetime('now')))`}async function mt(t=25,e=60){let n=o(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
-          LEFT JOIN billing_debits d
-            ON d.idempotency_key = json_extract(j.options, '$.debitKey')
-           AND d.user_id = j.user_id
-           AND d.status = 'applied'
-          WHERE ${D("j.")}
-            AND j.billed_mc IS NULL
-            AND (
-              json_extract(j.options, '$.debitKey') IS NULL
-              OR d.idempotency_key IS NOT NULL
-            )
-            AND ${O("j.")} AND ${h("j.")}
-          ORDER BY j.updated_at ASC LIMIT ?`,args:[`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(s=>_(s))}async function xt(t,e=60){let n=o(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`UPDATE site_extract_jobs
-          SET status = 'failed',
-              error = 'Background extraction attempt expired; completed pages were preserved and settlement is pending.',
-              public_error_json = ?,
-              updated_at = datetime('now')
-          WHERE id = ?
-            AND ${D()}
-            AND billed_mc IS NULL
-            AND ${O()} AND ${h()}`,args:[E({error_code:"extraction_failed",error_type:"extraction",message:"The background extraction attempt expired. Completed pages were preserved; retry after settlement completes.",retryable:!0,charge_status:"refund_pending"}),t,`-${r} minutes`]})).rowsAffected===1}async function Nt(t){let e=await o().execute({sql:`UPDATE site_extract_jobs
-          SET status = 'running', execution_generation = execution_generation + 1,
-              refinalize_until = datetime('now', '+60 minutes'), updated_at = datetime('now')
-          WHERE id = ? AND (settlement_plan_json IS NULL OR billed_mc IS NOT NULL) AND (status = 'failed' OR (
-            status = 'running' AND execution_generation > 0 AND refinalize_until <= datetime('now')
-          )) RETURNING *`,args:[t]});return e.rows[0]?_(e.rows[0]):null}async function z(t,e=0){return(await o().execute({sql:`SELECT 1 FROM site_extract_jobs WHERE id = ? AND status IN ('pending', 'running')
-          AND execution_generation = ? AND (execution_generation = 0 OR refinalize_until > datetime('now'))`,args:[t,e]})).rows.length===1}async function bt(t,e,n=0){await o().execute({sql:`UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')
-          WHERE id = ? AND billed_mc IS NULL AND execution_generation = ? AND status IN ('complete', 'partial', 'failed')`,args:[p(e).slice(0,1e3),t,n]})}async function wt(t,e,n){let r=o(),a=x(t,n);await r.execute({sql:`UPDATE site_extract_jobs
-            SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running') AND execution_generation = 0 AND ${a.sql}`,args:[e,t,...a.args]})}async function St(t,e,n){if(e.length===0)return;let r=o(),a=await P(t);if(!a||!await z(t)){if(n)throw Error("site_crawl_page_write_not_current");return}let s=x(t,n,!0);if(!(await r.execute({sql:`SELECT 1 FROM site_extract_jobs WHERE id=? AND ${s.sql}`,args:[t,...s.args]})).rows.length){if(n)throw Error("site_crawl_page_write_not_current");return}let c=a?.userId!=null?await I({ownerId:String(a.userId),jobId:t,createdAt:a.createdAt,pages:e}):new Map,u=e.map(d=>{let{discoveryLinks:M,acquiredHtml:b,mainHtml:R,...L}=d;return L.contentRef=c.get(d.archivedUrl??d.url)??d.contentRef,{sql:`INSERT INTO site_extract_pages (job_id, url, page)
-            SELECT ?, ?, ?
-            WHERE EXISTS (
-              SELECT 1 FROM site_extract_jobs
-              WHERE id = ? AND status IN ('pending', 'running') AND execution_generation = 0 AND ${s.sql}
-            )
-            ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page
-            WHERE NOT CASE
-                    WHEN json_valid(site_extract_pages.page) = 0 THEN 0
-                    WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL
-                      THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'
-                    ELSE COALESCE(
-                      json_extract(site_extract_pages.page, '$.status') >= 200
-                      AND json_extract(site_extract_pages.page, '$.status') < 300
-                      AND json_extract(site_extract_pages.page, '$.wordCount') > 0,
-                      0
-                    )
-                  END
-               OR CASE
-                    WHEN json_valid(excluded.page) = 0 THEN 0
-                    WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL
-                      THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'
-                    ELSE COALESCE(
-                      json_extract(excluded.page, '$.status') >= 200
-                      AND json_extract(excluded.page, '$.status') < 300
-                      AND json_extract(excluded.page, '$.wordCount') > 0,
-                      0
-                    )
-                  END`,args:[t,d.archivedUrl??d.url,JSON.stringify(L),t,...s.args]}}),l,S;try{S=await r.batch([...u,{sql:`UPDATE site_extract_jobs SET
-            done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
-            attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
-            successful_urls = (
-              SELECT COUNT(*) FROM site_extract_pages
-              WHERE job_id = ? AND CASE
-                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                  THEN json_extract(page, '$.extractionStatus') = 'successful'
-                ELSE json_extract(page, '$.status') >= 200
-                  AND json_extract(page, '$.status') < 300
-                  AND json_extract(page, '$.wordCount') > 0
-              END
-            ),
-            failed_urls = (
-              SELECT COUNT(*) FROM site_extract_pages
-              WHERE job_id = ? AND NOT CASE
-                WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                  THEN json_extract(page, '$.extractionStatus') = 'successful'
-                ELSE COALESCE(
-                  json_extract(page, '$.status') >= 200
-                  AND json_extract(page, '$.status') < 300
-                  AND json_extract(page, '$.wordCount') > 0,
-                  0
-                )
-              END
-            ),
-            updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running') AND execution_generation = 0 AND ${s.sql}`,args:[t,t,t,t,t,...s.args]}],"write")}catch(d){l=d}if(l){let d=await Promise.all(u.map(b=>r.execute({sql:"SELECT page FROM site_extract_pages WHERE job_id=? AND url=?",args:b.args.slice(0,2)})));if(!(await r.execute({sql:`SELECT 1 FROM site_extract_jobs WHERE id=? AND ${s.sql}
-      AND attempted_urls=(SELECT COUNT(*) FROM site_extract_pages WHERE job_id=?)
-      AND successful_urls=(SELECT COUNT(*) FROM site_extract_pages WHERE job_id=site_extract_jobs.id AND CASE
-        WHEN json_extract(page,'$.extractionStatus') IS NOT NULL THEN json_extract(page,'$.extractionStatus')='successful'
-        ELSE COALESCE(json_extract(page,'$.status')>=200 AND json_extract(page,'$.status')<300 AND json_extract(page,'$.wordCount')>0,0) END)
-      AND failed_urls=attempted_urls-successful_urls`,args:[t,...s.args,t]})).rows.length||d.some((b,R)=>b.rows[0]?.page!==u[R].args[2]))throw l}else if(n&&S?.at(-1)?.rowsAffected!==1)throw Error("site_crawl_page_write_not_current")}async function ft(t){return(await o().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function At(t,e=181){let n=o(),r=Math.max(1,Math.min(5001,Math.round(e)));return(await n.execute({sql:`SELECT DISTINCT CAST(images.value AS TEXT) AS url
-          FROM site_extract_pages p,
-               json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images
-          WHERE p.job_id = ?
-            AND images.type = 'text'
-            AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096
-          LIMIT ?`,args:[t,r]})).rows.map(s=>String(s.url))}async function ht(t){let n=await o().execute({sql:`SELECT COUNT(*) AS n FROM site_extract_pages
-            WHERE job_id = ? AND CASE
-              WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
-                THEN json_extract(page, '$.extractionStatus') = 'successful'
-              ELSE COALESCE(
-                json_extract(page, '$.status') >= 200
-                AND json_extract(page, '$.status') < 300
-                AND json_extract(page, '$.wordCount') > 0,
-                0
-              )
-            END`,args:[t]});return Number(n.rows[0]?.n??0)}async function Dt(t,e,n,r=null,a=!1,s=null,i=0,c){let u=o(),l=x(t,c,!0);return(await u.execute({sql:`UPDATE site_extract_jobs
-            SET status = ?, artifacts = ?, error = ?, public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND execution_generation = ? AND (settlement_plan_json IS NULL OR billed_mc IS NOT NULL)
-            AND (
-              (execution_generation = 0 AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed')))
-              OR (execution_generation > 0 AND status = 'running' AND refinalize_until > datetime('now'))
-            ) AND (execution_generation > 0 OR ${l.sql})`,args:[n,JSON.stringify(e??[]),r?p(r).slice(0,2e3):null,E(s),t,i,a?1:0,...l.args]})).rowsAffected===1}async function Rt(t,e,n,r=0,a){let s=x(t,a);return(await o().execute({sql:`UPDATE site_extract_jobs
-            SET status = 'failed', error = ?, public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND status IN ('pending', 'running') AND execution_generation = ?
-            AND (execution_generation = 0 OR refinalize_until > datetime('now'))
-            AND (execution_generation > 0 OR (${s.sql} AND ${y()}))`,args:[p(e).slice(0,2e3),E(n),t,r,...s.args]})).rowsAffected===1}async function Lt(t,e,n){await o().execute({sql:`UPDATE site_extract_jobs
-            SET status = 'failed', billed_mc = 0, error = ?, public_error_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,args:[p(e).slice(0,2e3),E(n),t]})}async function Ct(t){return await o().execute({sql:`UPDATE site_extract_jobs SET settlement_plan_json = ?, updated_at = datetime('now')
-          WHERE id = ? AND user_id = ? AND execution_generation = ? AND status = ?
-            AND billed_mc IS NULL AND settlement_plan_json IS NULL`,args:[JSON.stringify(t),t.jobId,t.userId,t.generation,t.status]}),P(t.jobId)}async function Tt(t,e){let n=t.billingClass===q?"not_charged":e.netChargeMc>0?"charged":e.refundMc>0?"refunded":"not_charged";await o().execute({sql:`UPDATE site_extract_jobs SET billed_mc = ?, settlement_receipt_json = ?,
-          public_error_json = CASE WHEN json_valid(public_error_json) THEN json_set(public_error_json, '$.charge_status', ?) ELSE public_error_json END,
-          updated_at = datetime('now')
-          WHERE id = ? AND user_id = ? AND execution_generation = ? AND status = ?
-            AND billed_mc IS NULL AND settlement_plan_json = ?`,args:[e.netChargeMc,JSON.stringify(e),n,t.jobId,t.userId,t.generation,t.status,JSON.stringify(t)]})}async function jt(t,e,n,r,a,s=0){let{settleExtractJobCharge:i}=await import("./settlement-3PS32LQO.js");return i(t,e,n,r,a,s)}var B=25;async function*It(t){let e=o(),n=0;for(;;){let r=await e.execute({sql:`SELECT rowid AS cursor_id, page
-            FROM site_extract_pages
-            WHERE job_id = ? AND rowid > ?
-            ORDER BY rowid
-            LIMIT ${B}`,args:[t,n]});if(!r.rows.length)return;if(n=Number(r.rows.at(-1)?.cursor_id??n),!Number.isSafeInteger(n)||n<=0)throw new Error("site extract page cursor is invalid");yield r.rows.map(a=>JSON.parse(String(a.page)))}}export{Y as a,q as b,st as c,it as d,v as e,ot as f,P as g,ct as h,ut as i,lt as j,dt as k,_t as l,gt as m,pt as n,Et as o,mt as p,xt as q,Nt as r,z as s,bt as t,wt as u,St as v,ft as w,At as x,ht as y,Dt as z,Rt as A,Lt as B,Ct as C,Tt as D,jt as E,B as F,It as G};

package/dist/chunk-K3VDGJNK.js

@@ -1 +0,0 @@
-import{a as w,b as u,c as _,d as l,e as h,m as g,p}from"./chunk-7SR7MEU5.js";import{k as c,m}from"./chunk-ANQ4JJVI.js";import{b as I,c as s}from"./chunk-4ZVO6GX4.js";async function R(e){let r=s(),n=!r?.operationRunId,o=r?.operationRunId??(await c({userId:r?.userId??null,ownerScope:r?.userId!=null?`user:${r.userId}`:"system:unscoped",tool:e.tool,normalizedFlags:{subOperation:r?.subOp??null},deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,a=n?null:await p(o),i=a?.attempts.at(-1),t=e.parentAttemptId??(e.retryKind&&i?String(i.id):r?.operationRootAttemptId),b=e.edgeKind??(e.retryKind==="same_method"?"same_method_retry":e.retryKind==="provider_fallback"?"provider_fallback":void 0),d=t?await _({runId:o,parentAttemptId:t,edgeKind:b??"parallel_child",method:e.method,provider:e.provider,retryKind:e.retryKind,retryReason:e.retryReason}):await u({runId:o,attemptNumber:(a?.attempts.length??0)+1,method:e.method,provider:e.provider,...i?{parentAttemptId:String(i.id),edgeKind:b??"next_phase",retryKind:e.retryKind??null,retryReason:e.retryReason??null}:{}}),y;try{y=(await h({attemptId:d,provider:e.provider,providerProduct:e.providerProduct,billingTier:e.billingTier,accountScope:r?.operationAccountScope??"production-default",method:e.method,providerStatus:"started"})).id}catch(f){throw await w({tool:e.tool,method:e.method,phase:"pending_receipt_create",errorCode:f instanceof Error?f.name:"UnknownError",runId:o,attemptId:d}).catch(()=>{}),await l({attemptId:d,status:"failed",errorCode:"pending_receipt_create_failed",durationMs:0}).catch(()=>{}),f}return{runId:o,attemptId:d,receiptId:y,ownsRun:n,startedAtMs:Date.now()}}async function S(e,r,n){await l({attemptId:e.attemptId,status:r,errorCode:n??null,durationMs:Date.now()-e.startedAtMs}),e.ownsRun&&await m({runId:e.runId,status:r,failureClass:n??null})}async function T(e,r){let n=s(),o=null,a=Date.now();if(n?.operationRunId&&n.operationRootAttemptId)try{o=await _({runId:n.operationRunId,parentAttemptId:n.operationRootAttemptId,edgeKind:"parallel_child",method:e})}catch(t){console.warn(JSON.stringify({event:"local_cost_phase_start_failed",method:e,message:t instanceof Error?t.message:String(t)}))}let i="failed";try{let t=await r();return i="succeeded",t}finally{o&&await l({attemptId:o,status:i,durationMs:Date.now()-a,errorCode:i==="failed"?"local_phase_failed":null}).catch(t=>console.warn(JSON.stringify({event:"local_cost_phase_finish_failed",method:e,message:t instanceof Error?t.message:String(t)})))}}async function M(e,r){let n=s()?.operationRunId;n&&await g({runId:n,billingTable:"billing_debits",billingKey:e,relation:"debit",amountMc:r}).catch(o=>console.warn(JSON.stringify({event:"cost_billing_link_failed",message:o instanceof Error?o.message:String(o)})))}async function K(e,r){let n=s();if(n?.operationRunId)return I({...n,op:e.op,userId:e.userId},r);let o=null,a=null;try{o=(await c({userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,e.billingDebitKey&&await g({runId:o,billingTable:"billing_debits",billingKey:e.billingDebitKey,relation:"debit"}).catch(t=>console.warn(JSON.stringify({event:"cost_billing_link_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))),a=await u({runId:o,attemptNumber:1,method:`${e.tool}_root`,normalizedFlags:e.normalizedFlags})}catch(t){console.warn(JSON.stringify({event:"logical_cost_run_start_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))}let i="failed";try{let t=await I({...n,op:e.op,userId:e.userId,operationRunId:o,operationRootAttemptId:a},r);return i=t instanceof Response&&t.status>=400?"failed":"succeeded",t}finally{o&&a?await v({runId:o,rootAttemptId:a,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(t=>console.warn(JSON.stringify({event:"logical_cost_run_finish_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))):o&&await m({runId:o,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(()=>{})}}async function O(e){let r=await c({...e.jobId?{id:`extract-url:${e.jobId}`,idempotencyKey:`job:${e.jobId}`,requestFingerprint:e.jobId,jobId:e.jobId}:{},userId:e.userId,ownerScope:`user:${e.userId}`,tool:"extract_url",normalizedFlags:e.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null}),n=r.replayed?await p(r.id):null,o=n?.attempts[0]?.id==null?await u({runId:r.id,attemptNumber:1,method:"extract_url_root",normalizedFlags:e.normalizedFlags}):String(n.attempts[0].id);return e.billingDebitKey&&await g({runId:r.id,billingTable:"billing_debits",billingKey:e.billingDebitKey,relation:"debit"}),{runId:r.id,rootAttemptId:o}}async function v(e){let r=await p(e.runId),n=r?.attempts.find(o=>String(o.id)===e.rootAttemptId);n?.status==="running"&&await l({attemptId:e.rootAttemptId,status:e.status,errorCode:e.failureClass??null,durationMs:Math.max(0,Date.now()-Date.parse(String(n.started_at)))}),r?.run.status==="running"&&await m({runId:e.runId,status:e.status,failureClass:e.failureClass??null})}export{R as a,S as b,T as c,M as d,K as e,O as f,v as g};

package/dist/chunk-KJQXUZ4Y.js

@@ -1,4 +0,0 @@
-function l(r){let t=[],e=[],n="",i=!1;for(let s=0;s<r.length;s+=1){let o=r[s],c=r[s+1];if(i){o==='"'&&c==='"'?(n+='"',s+=1):o==='"'?i=!1:n+=o;continue}o==='"'?i=!0:o===","?(e.push(n),n=""):o===`
-`?(e.push(n),t.push(e),e=[],n=""):o!=="\r"&&(n+=o)}return(n.length>0||e.length>0)&&(e.push(n),t.push(e)),t}function u(r){let t=l(r).filter(n=>n.some(i=>i.trim()!=="")),e=t[0]?.map(n=>n.trim())??[];return t.slice(1).map(n=>{let i={};for(let s=0;s<e.length;s+=1)i[e[s]]=n[s]??"";return i})}function f(r){if(r==null)return"";let t=String(r);return/[",\n\r]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function g(r,t){return[r.join(","),...t.map(e=>r.map(n=>f(e[n])).join(","))].join(`
-`)+`
-`}export{l as a,u as b,g as c};

package/dist/chunk-KXJCGV3H.js

@@ -1 +0,0 @@
-import{e as i,f as I,h as m,j as o}from"./chunk-SEZETSB3.js";import{i as A}from"./chunk-SDQW7IJN.js";var s="site-extracts/",y=10080*60*1e3,b=900*1e3;function x(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function _(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function r(){return{prefix:s,artifactTtlMs:y,downloadTtlMs:b,token:x()}}function l(t){return A(t,s)}async function v(t){let e=await i({policy:r(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function R(t){let e=await I({policy:r(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function D(t){let e=await i({policy:r(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function S(t){return i({policy:r(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function C(t){return m({policy:r(),artifactId:t.artifactId,ownerId:t.ownerId})}async function P(t){return o({policy:r(),artifactId:t,maxBytes:50*1024*1024})}async function B(t){return l(t.artifactId)!==t.ownerId?null:P(t.artifactId)}async function k(t){return l(t.artifactId)!==t.ownerId?null:o({policy:r(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function h(t){return l(t.artifactId)!==t.ownerId?null:o({policy:r(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function U(t={}){let e=t.now??new Date,a=x();if(!a)return{deleted:0,store:_()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let E=e.getTime()-y,{list:T,del:w}=await import("@vercel/blob"),f,p=0;for(let u=0;u<20;u+=1){let n=await T({prefix:s,token:a,limit:1e3,cursor:f}),c=n.blobs.filter(d=>new Date(d.uploadedAt).getTime()<=E);if(c.length>0&&(await w(c.map(d=>d.pathname),{token:a}),p+=c.length),!n.hasMore||!n.cursor)break;f=n.cursor}return{deleted:p,store:"private-vercel-blob"}}export{s as a,y as b,b as c,r as d,l as e,v as f,R as g,D as h,S as i,C as j,P as k,B as l,k as m,h as n,U as o};