mcp-scraper 0.93.3 → 0.94.0

package/dist/{chunk-5JKBFNYF.js → chunk-ZTEDLK3Y.js}

@@ -1,4 +1,4 @@
-import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{ba as w}from"./chunk-UN6FVDJQ.js";import{Kb as N,Tb as D,d as m,e as I,i as u,t as s}from"./chunk-WJ4XFLS4.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{ba as w}from"./chunk-ZGITQEFC.js";import{Kb as N,Tb as D,d as m,e as I,i as u,t as s}from"./chunk-ZPK5LYEN.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
             (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs

package/dist/{chunk-ORB4RHCK.js → chunk-ZUGTYD2I.js}

@@ -1,4 +1,4 @@
-import{t as s}from"./chunk-WJ4XFLS4.js";import{createHash as v,randomUUID as m}from"crypto";async function O(e){let r=m();return await s().execute({sql:`INSERT INTO operation_coverage_defects (id, tool, method, phase, error_code, run_id, attempt_id)
+import{t as s}from"./chunk-ZPK5LYEN.js";import{createHash as v,randomUUID as m}from"crypto";async function O(e){let r=m();return await s().execute({sql:`INSERT INTO operation_coverage_defects (id, tool, method, phase, error_code, run_id, attempt_id)
       VALUES (?, ?, ?, ?, ?, ?, ?)`,args:[r,a(e.tool,"tool"),a(e.method,"method"),a(e.phase,"phase"),a(e.errorCode,"errorCode").slice(0,200),e.runId??null,e.attemptId??null]}),r}var E=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function u(e){return v("sha256").update(e).digest("hex")}function f(e){return Array.isArray(e)?e.map(f):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([r],[t])=>r.localeCompare(t)).map(([r,t])=>[r,f(t)])):e}function S(e){return JSON.stringify(f(e??{}))}function a(e,r){let t=e.trim();if(!t)throw new Error(`${r} is required`);return t}function l(e,r=2e3){return e==null?null:e.slice(0,r)}async function I(e){let r=s(),t=a(e.tool,"tool"),i=a(e.ownerScope,"ownerScope"),n=e.idempotencyKey?.trim()||null,o=e.requestFingerprint?.trim()||null;if(n==null!=(o==null))throw new Error("idempotencyKey and requestFingerprint must be supplied together");let d=n?u(`${i}\0${t}\0${n}`):null,c=e.id??m(),g=[c,e.userId??null,u(i),t,d,o,e.jobId??null,e.requestId??null,e.deploymentCommit??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()];if(d){await r.execute({sql:`INSERT OR IGNORE INTO operation_runs
         (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
          job_id, request_id, deployment_commit, normalized_flags_json, started_at)
@@ -43,13 +43,13 @@ import{t as s}from"./chunk-WJ4XFLS4.js";import{createHash as v,randomUUID as m}f
         SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
         WHERE finalized_at < datetime('now', ?)
           AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[r]}),t.execute({sql:`UPDATE harvest_attempts
-        SET provider_session_id = NULL, kernel_session_id = NULL, error = NULL,
-            kernel_delete_error = NULL, browser_close_error = NULL,
+        SET provider_session_id = NULL, \u006b\u0065\u0072\u006e\u0065\u006c_session_id = NULL, error = NULL,
+            \u006b\u0065\u0072\u006e\u0065\u006c_delete_error = NULL, browser_close_error = NULL,
             provider_error_message = NULL, provider_detail_json = NULL,
             provider_disconnect_message = NULL, debug_json = NULL
         WHERE completed_at < datetime('now', ?)
-          AND (provider_session_id IS NOT NULL OR kernel_session_id IS NOT NULL OR error IS NOT NULL
-            OR kernel_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
+          AND (provider_session_id IS NOT NULL OR \u006b\u0065\u0072\u006e\u0065\u006c_session_id IS NOT NULL OR error IS NOT NULL
+            OR \u006b\u0065\u0072\u006e\u0065\u006c_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
             OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
             OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[r]})]);return{operationAttempts:i.rowsAffected,providerReceipts:n.rowsAffected,legacyHarvestAttempts:o.rowsAffected}}async function D(e){let t={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],i=a(e.billingKey,"billingKey"),n=await s().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
       FROM operation_runs r

package/dist/{db-B5XJTOGN.js → db-B6RNXKEI.js}

@@ -1 +1 @@
-import{$ as M,$a as Ma,$b as Mb,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Yb as Jb,Z as K,Za as Ka,Zb as Kb,_ as L,_a as La,_b as Lb,aa as N,ab as Na,ac as Nb,ba as O,bb as Oa,bc as Ob,ca as P,cb as Pa,cc as Pb,da as Q,db as Qa,dc as Qb,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-WJ4XFLS4.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,vb as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Jb as SiteAuditJobRowSchema,Kb as SiteAuditPhaseLogRowSchema,Aa as advanceWorkflowRunStep,ya as appendWorkflowArtifacts,mb as applySubscriptionInvoiceGrant,W as attachHarvestAttemptProviderSession,Ya as cancelJob,K as checkRateLimit,Ja as checkpointDurablePaaJob,Ia as checkpointJob,t as claimDueStripeEvents,Ha as claimDurablePaaJob,Bb as claimMonthlyFreeRefresh,Ga as claimPendingJob,_a as claimPendingKpoJob,La as completeDurablePaaJob,Oa as completeExpiredDurablePaaJob,Ua as completeJob,bb as completeKpoJob,Va as completeRunningJob,u as completeStripeEvent,va as completeWorkflowRunRecord,n as consumeCode,M as consumePasswordResetToken,Fa as countActiveJobsForUser,Ea as countActiveUsers,R as createJob,S as createJobWithId,Za as createKpoJob,L as createPasswordResetToken,T as createRunningJob,J as createUser,qa as createWorkflowRun,ja as createWorkflowSchedule,yb as creditMc,zb as creditMcIdempotent,Q as deactivateUser,w as deadLetterStripeEvent,Cb as debitMc,Db as debitMcIdempotent,na as deleteWorkflowSchedule,s as enqueueStripeEvent,wb as expireOldLots,Ma as failDurablePaaJob,Pa as failExpiredDurablePaaJob,Wa as failJob,cb as failKpoJob,Xa as failRunningJob,wa as failWorkflowRunRecord,X as finalizeExpiredHarvestAttempts,Y as finishHarvestAttempt,z as generateApiKey,Gb as getBillingLedgerSummary,l as getClient,nb as getConnectedAccountBillingState,e as getDb,Mb as getInboxMessage,aa as getJob,$a as getKpoJob,eb as getKpoPhaseLog,Fb as getLedger,rb as getMemoryPlan,sb as getMemoryProvisioned,U as getOrCreateRunningJob,fa as getPageSnapshot,p as getRefresh,x as getStripeEventQueueHealth,D as getUserByApiKey,C as getUserByEmail,E as getUserById,gb as getUserByStripeCustomerId,F as getUserMemoryCreds,I as getUserStats,Ca as getWorkflowArtifact,ra as getWorkflowRun,sa as getWorkflowRunByIdempotencyKey,za as getWorkflowRunStepState,ka as getWorkflowSchedule,A as hashPassword,Pb as insertInboxMessage,d as inspectCurrentCoreSchema,Ab as ledgerExistsForOperation,Hb as ledgerExistsForStripePI,Qb as listActiveUserIds,oa as listDueWorkflowSchedules,$ as listHarvestAttempts,_ as listHarvestAttemptsPendingProviderTelemetry,Lb as listInboxMessages,ea as listJobs,fb as listKpoJobs,ca as listPendingExtractUrlDispatches,Qa as listRecentTerminalDurablePaaJobs,Sa as listRecoverableDurablePaaJobs,ia as listRequestEvents,Ra as listStaleRunningDurablePaaJobs,P as listUsers,pb as listUsersDueForConnectedAccountBillingReconciliation,Ba as listWorkflowArtifacts,ta as listWorkflowRuns,la as listWorkflowSchedules,db as logKpoPhaseComplete,ha as logRequestEvent,ba as markDispatchedJobRunning,Nb as markInboxMessageOpened,Ta as markJobBillingSettled,ua as markWorkflowRunRunning,pa as markWorkflowScheduleRan,i as migrate,j as migrateCoreSchemaForRelease,xb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,ma as patchWorkflowSchedule,m as putCode,o as putRefresh,Ib as reconcileBalanceMc,da as recordExtractUrlDispatchAttempt,Da as recordWorkflowWebhookDelivery,k as registerClient,Ka as renewDurablePaaLease,xa as replaceWorkflowArtifacts,Na as requeueExpiredDurablePaaJob,ub as resolveUserByMemorySubscription,v as retryStripeEvent,O as revokeApiKey,q as revokeRefresh,N as rotateApiKey,r as rotateRefresh,jb as setConcurrencyPackQuantity,kb as setConcurrencySubId,ob as setConnectedAccountBillingState,f as setDbForTesting,ib as setExtraConcurrencySlots,qb as setMemoryPlan,tb as setMemoryProvisioned,H as setPassword,hb as setStripeCustomerId,lb as setSubscriptionTier,G as setUserMemoryCreds,Eb as settleDebitMcIdempotent,Ob as softDeleteInboxMessage,V as startHarvestAttempt,y as stripeEventAlreadyProcessed,Z as updateHarvestAttemptProviderTelemetry,ab as updateKpoJobState,ga as upsertPageSnapshot,B as verifyPassword,h as verifySecuredMainOAuthSchema};
+import{$ as M,$a as Ma,$b as Mb,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Yb as Jb,Z as K,Za as Ka,Zb as Kb,_ as L,_a as La,_b as Lb,aa as N,ab as Na,ac as Nb,ba as O,bb as Oa,bc as Ob,ca as P,cb as Pa,cc as Pb,da as Q,db as Qa,dc as Qb,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-ZPK5LYEN.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,vb as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Jb as SiteAuditJobRowSchema,Kb as SiteAuditPhaseLogRowSchema,Aa as advanceWorkflowRunStep,ya as appendWorkflowArtifacts,mb as applySubscriptionInvoiceGrant,W as attachHarvestAttemptProviderSession,Ya as cancelJob,K as checkRateLimit,Ja as checkpointDurablePaaJob,Ia as checkpointJob,t as claimDueStripeEvents,Ha as claimDurablePaaJob,Bb as claimMonthlyFreeRefresh,Ga as claimPendingJob,_a as claimPendingKpoJob,La as completeDurablePaaJob,Oa as completeExpiredDurablePaaJob,Ua as completeJob,bb as completeKpoJob,Va as completeRunningJob,u as completeStripeEvent,va as completeWorkflowRunRecord,n as consumeCode,M as consumePasswordResetToken,Fa as countActiveJobsForUser,Ea as countActiveUsers,R as createJob,S as createJobWithId,Za as createKpoJob,L as createPasswordResetToken,T as createRunningJob,J as createUser,qa as createWorkflowRun,ja as createWorkflowSchedule,yb as creditMc,zb as creditMcIdempotent,Q as deactivateUser,w as deadLetterStripeEvent,Cb as debitMc,Db as debitMcIdempotent,na as deleteWorkflowSchedule,s as enqueueStripeEvent,wb as expireOldLots,Ma as failDurablePaaJob,Pa as failExpiredDurablePaaJob,Wa as failJob,cb as failKpoJob,Xa as failRunningJob,wa as failWorkflowRunRecord,X as finalizeExpiredHarvestAttempts,Y as finishHarvestAttempt,z as generateApiKey,Gb as getBillingLedgerSummary,l as getClient,nb as getConnectedAccountBillingState,e as getDb,Mb as getInboxMessage,aa as getJob,$a as getKpoJob,eb as getKpoPhaseLog,Fb as getLedger,rb as getMemoryPlan,sb as getMemoryProvisioned,U as getOrCreateRunningJob,fa as getPageSnapshot,p as getRefresh,x as getStripeEventQueueHealth,D as getUserByApiKey,C as getUserByEmail,E as getUserById,gb as getUserByStripeCustomerId,F as getUserMemoryCreds,I as getUserStats,Ca as getWorkflowArtifact,ra as getWorkflowRun,sa as getWorkflowRunByIdempotencyKey,za as getWorkflowRunStepState,ka as getWorkflowSchedule,A as hashPassword,Pb as insertInboxMessage,d as inspectCurrentCoreSchema,Ab as ledgerExistsForOperation,Hb as ledgerExistsForStripePI,Qb as listActiveUserIds,oa as listDueWorkflowSchedules,$ as listHarvestAttempts,_ as listHarvestAttemptsPendingProviderTelemetry,Lb as listInboxMessages,ea as listJobs,fb as listKpoJobs,ca as listPendingExtractUrlDispatches,Qa as listRecentTerminalDurablePaaJobs,Sa as listRecoverableDurablePaaJobs,ia as listRequestEvents,Ra as listStaleRunningDurablePaaJobs,P as listUsers,pb as listUsersDueForConnectedAccountBillingReconciliation,Ba as listWorkflowArtifacts,ta as listWorkflowRuns,la as listWorkflowSchedules,db as logKpoPhaseComplete,ha as logRequestEvent,ba as markDispatchedJobRunning,Nb as markInboxMessageOpened,Ta as markJobBillingSettled,ua as markWorkflowRunRunning,pa as markWorkflowScheduleRan,i as migrate,j as migrateCoreSchemaForRelease,xb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,ma as patchWorkflowSchedule,m as putCode,o as putRefresh,Ib as reconcileBalanceMc,da as recordExtractUrlDispatchAttempt,Da as recordWorkflowWebhookDelivery,k as registerClient,Ka as renewDurablePaaLease,xa as replaceWorkflowArtifacts,Na as requeueExpiredDurablePaaJob,ub as resolveUserByMemorySubscription,v as retryStripeEvent,O as revokeApiKey,q as revokeRefresh,N as rotateApiKey,r as rotateRefresh,jb as setConcurrencyPackQuantity,kb as setConcurrencySubId,ob as setConnectedAccountBillingState,f as setDbForTesting,ib as setExtraConcurrencySlots,qb as setMemoryPlan,tb as setMemoryProvisioned,H as setPassword,hb as setStripeCustomerId,lb as setSubscriptionTier,G as setUserMemoryCreds,Eb as settleDebitMcIdempotent,Ob as softDeleteInboxMessage,V as startHarvestAttempt,y as stripeEventAlreadyProcessed,Z as updateHarvestAttemptProviderTelemetry,ab as updateKpoJobState,ga as upsertPageSnapshot,B as verifyPassword,h as verifySecuredMainOAuthSchema};

package/dist/{extract-bundle-C3N6E6V7.js → extract-bundle-GMRZ2CW3.js}

@@ -1,4 +1,4 @@
-import{c as Ge,h as ke,i as Qe,j as Ve,l as et}from"./chunk-IK5BG7MO.js";import"./chunk-XG6GCEUE.js";import{b as Ye,c as Xe,h as Ze,j as Ke}from"./chunk-5JKBFNYF.js";import"./chunk-M5QHXNFZ.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import{c as Je}from"./chunk-RK2VCTZI.js";import"./chunk-ORB4RHCK.js";import"./chunk-TMB56NCA.js";import"./chunk-UN6FVDJQ.js";import"./chunk-6DTXIZY2.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-WJ4XFLS4.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as xt,readdirSync as kt,rmSync as _t,statSync as Pt,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Mt,join as a}from"path";import{tmpdir as It}from"os";import{createGzip as Ct}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as At}from"yazl";import Rt from"p-limit";import{createHash as _e}from"crypto";var gt=500,ht=.9,ft=1e4,yt=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function wt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function St(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function tt(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function bt(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(tt).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=tt(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
+import{c as Ge,h as ke,i as Qe,j as Ve,l as et}from"./chunk-RJPQCPDX.js";import"./chunk-XG6GCEUE.js";import{b as Ye,c as Xe,h as Ze,j as Ke}from"./chunk-ZTEDLK3Y.js";import"./chunk-M5QHXNFZ.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import{c as Je}from"./chunk-DOJMTFLB.js";import"./chunk-ZUGTYD2I.js";import"./chunk-TMB56NCA.js";import"./chunk-ZGITQEFC.js";import"./chunk-6DTXIZY2.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-ZPK5LYEN.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as xt,readdirSync as kt,rmSync as _t,statSync as Pt,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Mt,join as a}from"path";import{tmpdir as It}from"os";import{createGzip as Ct}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as At}from"yazl";import Rt from"p-limit";import{createHash as _e}from"crypto";var gt=500,ht=.9,ft=1e4,yt=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function wt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function St(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function tt(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function bt(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(tt).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=tt(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
 
 `).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
 

package/dist/{gmail-service-6V5MFBYH.js → gmail-service-U7NP5IMY.js}

@@ -1 +1 @@
-import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-NLKA6SHC.js";import"./chunk-IJF4VXJ7.js";import"./chunk-QTDLZTQ7.js";import"./chunk-HUV2WTRW.js";import"./chunk-UN6FVDJQ.js";import"./chunk-2SP57VCG.js";import"./chunk-6DTXIZY2.js";import"./chunk-WJ4XFLS4.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};
+import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-72MWPIMR.js";import"./chunk-BRNTVOCW.js";import"./chunk-L552AQQY.js";import"./chunk-HUV2WTRW.js";import"./chunk-ZGITQEFC.js";import"./chunk-NA76FFGD.js";import"./chunk-6DTXIZY2.js";import"./chunk-ZPK5LYEN.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};