mcp-scraper 0.90.4 → 0.90.6

package/dist/{chunk-HJ7OGU2J.js → chunk-MADTJXNI.js}

@@ -1,4 +1,4 @@
-import{a as Zt,s as tn}from"./chunk-WTOKMWDS.js";import{T as cn,a as de,b as Ne,c as yt,d as an,e as on,q as f,r as v,sa as G,ta as Ce,v as U,w as J}from"./chunk-HR2A2WTL.js";import{a as Fe,e as en}from"./chunk-2GRE4BAO.js";import{d as nn,g as je,i as qe,k as _t,m as mt,o as rn,r as He,t as sn,u as pt}from"./chunk-D6BTAT5Y.js";import{B as dt,F as D,G as Yt,P as Xt,Q as Gt,R as he,S as lt,T as Qt,U as ut,W as X,Y as ce,_ as Ue,r as Me,s as Jt,x as zt}from"./chunk-YEFXVTMJ.js";import{Ab as Wt,Eb as oe,G as qt,Hb as Bt,Ib as ct,Nb as Vt,Q as ke,Rb as Kt,S as Ht,t as ae,ub as T,vb as Pe,xb as st,yb as at,zb as ot}from"./chunk-7BEWBRMT.js";import{createCipheriv as Di,createDecipheriv as Ti,randomBytes as xi,scryptSync as Li}from"crypto";import{setScheduleEntitlementTool as $i}from"mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";import{createHash as Si,randomUUID as hi}from"crypto";import{exportJWK as Ni,importPKCS8 as Ci,SignJWT as Ri}from"jose";function wi(e=process.env.OAUTH_ISSUER){let t=e?.trim()||"https://mcpscraper.dev",n=new URL(t);if(n.protocol!=="https:"||n.username||n.password||n.search||n.hash||n.pathname!=="/"&&n.pathname!=="")throw new Error("OAUTH_ISSUER must be an HTTPS origin without credentials, path, query, or fragment");return n.origin}var Oi=wi(),We=null;async function dn(){if(We)return We;let e=process.env.OAUTH_SIGNING_KEY;if(!e?.trim())throw new Error("OAUTH_SIGNING_KEY is not set");let t=e.includes("BEGIN")?e:Buffer.from(e,"base64").toString("utf8"),n=await Ci(t,"RS256",{extractable:!0}),i=await Ni(n),r={kty:i.kty,n:i.n,e:i.e},s=Si("sha256").update(JSON.stringify({e:r.e,kty:r.kty,n:r.n})).digest("base64url").slice(0,16);return r.kid=s,r.alg="RS256",r.use="sig",We={privateKey:n,publicJwk:r,kid:s},We}async function Ts(){return(await dn()).publicJwk}async function ln(e){let{privateKey:t,kid:n}=await dn();return new Ri({scope:e.scope,plan:e.plan}).setProtectedHeader({alg:"RS256",kid:n}).setIssuer(Oi).setSubject(e.identity).setAudience(e.audience).setIssuedAt().setJti(hi()).setExpirationTime(`${e.ttlSeconds}s`).sign(t)}var ki=()=>(process.env.MCP_MEMORY_BASE_URL??"https://memory.mcpscraper.dev").replace(/\/$/,""),Pi=()=>(process.env.MCP_MEMORY_RESOURCE_URL??"https://memory.mcpscraper.dev/mcp").replace(/\/$/,""),Mi=()=>process.env.MCP_MEMORY_ADMIN_KEY?.trim()??"",Ui=300;function Fi(e){let t=e.split(`
+import{a as Zt,s as tn}from"./chunk-WDB5CTRY.js";import{T as cn,a as de,b as Ne,c as yt,d as an,e as on,q as f,r as v,sa as G,ta as Ce,v as U,w as J}from"./chunk-4OTWTGKC.js";import{a as Fe,e as en}from"./chunk-Q5SUDDSQ.js";import{d as nn,g as je,i as qe,k as _t,m as mt,o as rn,r as He,t as sn,u as pt}from"./chunk-D6BTAT5Y.js";import{B as dt,F as D,G as Yt,P as Xt,Q as Gt,R as he,S as lt,T as Qt,U as ut,W as X,Y as ce,_ as Ue,r as Me,s as Jt,x as zt}from"./chunk-CTX3NMZQ.js";import{Ab as Wt,Eb as oe,G as qt,Hb as Bt,Ib as ct,Nb as Vt,Q as ke,Rb as Kt,S as Ht,t as ae,ub as T,vb as Pe,xb as st,yb as at,zb as ot}from"./chunk-KKHTMYCD.js";import{createCipheriv as Di,createDecipheriv as Ti,randomBytes as xi,scryptSync as Li}from"crypto";import{setScheduleEntitlementTool as $i}from"mcpscraper-memory-tools/tools/schedule/set-schedule-entitlement";import{createHash as Si,randomUUID as hi}from"crypto";import{exportJWK as Ni,importPKCS8 as Ci,SignJWT as Ri}from"jose";function wi(e=process.env.OAUTH_ISSUER){let t=e?.trim()||"https://mcpscraper.dev",n=new URL(t);if(n.protocol!=="https:"||n.username||n.password||n.search||n.hash||n.pathname!=="/"&&n.pathname!=="")throw new Error("OAUTH_ISSUER must be an HTTPS origin without credentials, path, query, or fragment");return n.origin}var Oi=wi(),We=null;async function dn(){if(We)return We;let e=process.env.OAUTH_SIGNING_KEY;if(!e?.trim())throw new Error("OAUTH_SIGNING_KEY is not set");let t=e.includes("BEGIN")?e:Buffer.from(e,"base64").toString("utf8"),n=await Ci(t,"RS256",{extractable:!0}),i=await Ni(n),r={kty:i.kty,n:i.n,e:i.e},s=Si("sha256").update(JSON.stringify({e:r.e,kty:r.kty,n:r.n})).digest("base64url").slice(0,16);return r.kid=s,r.alg="RS256",r.use="sig",We={privateKey:n,publicJwk:r,kid:s},We}async function Ts(){return(await dn()).publicJwk}async function ln(e){let{privateKey:t,kid:n}=await dn();return new Ri({scope:e.scope,plan:e.plan}).setProtectedHeader({alg:"RS256",kid:n}).setIssuer(Oi).setSubject(e.identity).setAudience(e.audience).setIssuedAt().setJti(hi()).setExpirationTime(`${e.ttlSeconds}s`).sign(t)}var ki=()=>(process.env.MCP_MEMORY_BASE_URL??"https://memory.mcpscraper.dev").replace(/\/$/,""),Pi=()=>(process.env.MCP_MEMORY_RESOURCE_URL??"https://memory.mcpscraper.dev/mcp").replace(/\/$/,""),Mi=()=>process.env.MCP_MEMORY_ADMIN_KEY?.trim()??"",Ui=300;function Fi(e){let t=e.split(`
 `).find(i=>i.startsWith("data:")),n=t?t.slice(5).trim():e;return JSON.parse(n)}function gt(e){return e?(process.env.MEMORY_OPERATOR_IDENTITIES??"").split(",").map(n=>n.trim().toLowerCase()).filter(Boolean).includes(e.trim().toLowerCase()):!1}function un(){return Li(Fe(),"mcp-memory-key-v1",32)}function Fs(e){let t=xi(12),n=Di("aes-256-gcm",un(),t),i=Buffer.concat([n.update(e,"utf8"),n.final()]),r=n.getAuthTag();return`${t.toString("base64")}:${r.toString("base64")}:${i.toString("base64")}`}function js(e){try{let[t,n,i]=e.split(":"),r=Ti("aes-256-gcm",un(),Buffer.from(t,"base64"));return r.setAuthTag(Buffer.from(n,"base64")),Buffer.concat([r.update(Buffer.from(i,"base64")),r.final()]).toString("utf8")}catch{return null}}async function _n(e,t,n){try{let i=await fetch(`${ki()}/mcp`,{method:"POST",headers:{"content-type":"application/json",accept:"application/json, text/event-stream",authorization:`Bearer ${n}`},body:JSON.stringify({jsonrpc:"2.0",id:1,method:"tools/call",params:{name:e,arguments:t}})}),r=await i.text();if(!i.ok)return{ok:!1,error:`memory ${e} failed (${i.status})`};let s=Fi(r);if(s.error)return{ok:!1,error:s.error.message??`memory ${e} failed`};if(s.result?.structuredContent)return s.result.structuredContent;let a=s.result?.content?.find(o=>o.type==="text")?.text;if(a)try{return JSON.parse(a)}catch{return{ok:!1,error:a}}return{ok:!1,error:`memory ${e} returned no result`}}catch(i){return{ok:!1,error:i?.message??"memory call failed"}}}function mn(e){return e.email}function ji(e){return mn(e)}function qs(e){return gt(e.email)?"unlimited":e.memory_plan==="pro"||e.memory_plan==="team"?e.memory_plan:"free"}async function Hs(e){try{let{key:t}=await pn(e);if(!t)return;(await _n("listVaultsTool",{},t)).ok&&await Bt(e.id,!0)}catch{}}function qi(e){return e?!!Yt[e]?.includes_memory:!1}function F(e){return e.memory_plan==="team"?"team":qi(e.subscription_tier)||e.memory_plan==="pro"?"pro":"free"}async function pn(e,t){try{let n=gt(e.email),i=n?"unlimited":t??F(e),r=n?"memory.read memory.write memory.admin":"memory.read memory.write";return{key:await ln({identity:mn(e),scope:r,plan:i,audience:Pi(),ttlSeconds:Ui})}}catch(n){return{key:null,error:n instanceof Error?n.message:"memory service authorization unavailable"}}}async function z(e,t){let n=gt(e.email)?"unlimited":t,{key:i,error:r}=await pn(e,n);if(!i)return{ok:!1,error:r??"memory unavailable"};let s=await _n("listVaultsTool",{},i);return{ok:s.ok,error:s.error}}async function Be(e){let t=Mi();if(!t)return{ok:!1,error:"memory not configured (MCP_MEMORY_ADMIN_KEY missing)"};let n=ji(e),i=await $i.execute({apiKey:t,granteeIdentity:n,mcpScraperApiKey:e.api_key},{});return{ok:i.ok,error:i.error}}var yn=["active","trialing","past_due"];function Re(e){return yn.includes(e)}function gn(e,t){for(let n of yn){let i=e.find(r=>r.status===n&&t(r));if(i)return i}}var Hi="price_1TtYPAS8aAcsk3TGHOgrDZiQ";function Wi(e=new Date){return`${e.getUTCFullYear()}-${String(e.getUTCMonth()+1).padStart(2,"0")}`}var Bi=new Set(["connected","needs_reauth"]);function Vi(e){return Bi.has(String(e.lifecycleStatus??""))}async function Ki(e,t){let n=await ae().execute({sql:"SELECT connection_id FROM connected_account_charges WHERE user_id = ? AND period_start = ?",args:[Number(e),t]});return new Set(n.rows.map(i=>String(i.connection_id)))}async function Ji(e,t,n){await en();let i=Number(e),r=ae();return await r.batch([{sql:`
         INSERT OR IGNORE INTO connected_account_charge_transfers (
           user_id, period_start, from_connection_id, to_connection_id, provider_config_key

package/dist/chunk-NOUHD54C.js

@@ -0,0 +1,48 @@
+import{a as C}from"./chunk-XG6GCEUE.js";import{d as k,g as T}from"./chunk-56UT5M5Y.js";import{b as y,g as p,h as R,ha as N,ia as x,j as I,ka as P,m as A,o as O,t as d}from"./chunk-KKHTMYCD.js";import{createHash as j,randomUUID as f}from"crypto";var w=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function g(e){return j("sha256").update(e).digest("hex")}function S(e){return Array.isArray(e)?e.map(S):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>[t,S(n)])):e}function L(e){return JSON.stringify(S(e??{}))}function l(e,t){let n=e.trim();if(!n)throw new Error(`${t} is required`);return n}function m(e,t=2e3){return e==null?null:e.slice(0,t)}async function U(e){let t=d(),n=l(e.tool,"tool"),s=l(e.ownerScope,"ownerScope"),i=e.idempotencyKey?.trim()||null,r=e.requestFingerprint?.trim()||null;if(i==null!=(r==null))throw new Error("idempotencyKey and requestFingerprint must be supplied together");let o=i?g(`${s}\0${n}\0${i}`):null,a=e.id??f(),c=[a,e.userId??null,g(s),n,o,r,e.jobId??null,e.requestId??null,e.deploymentCommit??null,L(e.normalizedFlags),e.startedAt??new Date().toISOString()];if(o){await t.execute({sql:`INSERT OR IGNORE INTO operation_runs
+        (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
+         job_id, request_id, deployment_commit, normalized_flags_json, started_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:c});let b=(await t.execute({sql:"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?",args:[o]})).rows[0];if(!b)throw new Error("Operation run insert did not produce an idempotency receipt");if(String(b.request_fingerprint)!==r)throw new w;return{id:String(b.id),replayed:String(b.id)!==a}}return await t.execute({sql:`INSERT INTO operation_runs
+      (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
+       job_id, request_id, deployment_commit, normalized_flags_json, started_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:c}),{id:a,replayed:!1}}async function M(e){let t=e.endedAt??new Date().toISOString();if((await d().execute({sql:`UPDATE operation_runs
+      SET status = ?, failure_class = ?, ended_at = ?, updated_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,m(e.failureClass,200),t,t,e.runId]})).rowsAffected===1)return;let s=m(e.failureClass,200),i=await d().execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]});if(i.rows.length===0)throw new Error(`Unknown operation run: ${e.runId}`);let r=i.rows[0]?.failure_class==null?null:String(i.rows[0]?.failure_class);if(String(i.rows[0]?.status)!==e.status||r!==s)throw new Error(`Operation run ${e.runId} is already terminal as ${String(i.rows[0]?.status)}`)}async function q(e){if(!Number.isInteger(e.attemptNumber)||e.attemptNumber<1)throw new Error("attemptNumber must be a positive integer");if(e.attemptNumber===1&&(e.parentAttemptId||e.edgeKind||e.retryKind))throw new Error("The first attempt cannot have a parent or retry edge");if(e.attemptNumber>1&&(!e.parentAttemptId||!e.edgeKind))throw new Error("Every non-first attempt requires a parentAttemptId and edgeKind");if(e.retryKind&&!["same_method_retry","provider_fallback","durable_resume","reconciliation"].includes(e.edgeKind??""))throw new Error("retryKind requires a retry or reconciliation edgeKind");let t=e.id??f();return await d().execute({sql:`INSERT INTO operation_attempts
+      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason, retry_owner,
+       method, provider, normalized_flags_json, started_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[t,e.runId,e.parentAttemptId??null,e.attemptNumber,e.edgeKind??null,e.retryKind??null,m(e.retryReason,500),m(e.retryOwner,100),l(e.method,"method"),e.provider??null,L(e.normalizedFlags),e.startedAt??new Date().toISOString()]}),t}async function D(e){if(!Number.isInteger(e.durationMs)||e.durationMs<0)throw new Error("durationMs must be a non-negative integer");let t=e.endedAt??new Date().toISOString();if((await d().execute({sql:`UPDATE operation_attempts SET
+      status = ?, error_code = ?, error_message = ?, ended_at = ?, duration_ms = ?, updated_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,m(e.errorCode,200),m(e.errorMessage),t,e.durationMs,t,e.attemptId]})).rowsAffected===1)return;let s=await d().execute({sql:"SELECT status, duration_ms FROM operation_attempts WHERE id = ?",args:[e.attemptId]});if(s.rows.length===0)throw new Error(`Unknown operation attempt: ${e.attemptId}`);if(String(s.rows[0]?.status)!==e.status||Number(s.rows[0]?.duration_ms)!==e.durationMs)throw new Error(`Operation attempt ${e.attemptId} is already terminal as ${String(s.rows[0]?.status)}`)}async function v(e){let t=l(e.provider,"provider"),n=l(e.method,"method"),s=g(l(e.accountScope,"accountScope")),i=e.providerSourceKey?.trim()||null,r=i?g(i):null,o=e.id??f(),a=await d().execute({sql:`INSERT OR IGNORE INTO provider_usage_receipts
+      (id, attempt_id, provider, account_scope_hash, method, provider_source_key,
+       provider_source_key_hash, provider_status)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,args:[o,e.attemptId,t,s,n,i,r,e.providerStatus??null]});if(!r){if(a.rowsAffected===1)return{id:o,replayed:!1};let _=(await d().execute({sql:"SELECT attempt_id, provider, method FROM provider_usage_receipts WHERE id = ?",args:[o]})).rows[0];if(!_||String(_.attempt_id)!==e.attemptId||String(_.provider)!==t||String(_.method)!==n)throw new Error("Provider receipt ID is already used by a different receipt");return{id:o,replayed:!0}}let u=(await d().execute({sql:`SELECT id, attempt_id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[t,s,n,r]})).rows[0];if(!u)throw new Error("Provider receipt insert did not produce a source receipt");if(String(u.attempt_id)!==e.attemptId)throw new Error("Provider source identity is already attached to a different attempt");return{id:String(u.id),replayed:String(u.id)!==o}}async function H(e,t){let n=l(t,"providerSourceKey"),s=g(n);if((await d().execute({sql:`UPDATE provider_usage_receipts
+      SET provider_source_key = ?, provider_source_key_hash = ?, updated_at = datetime('now')
+      WHERE id = ? AND provider_source_key IS NULL`,args:[n,s,e]})).rowsAffected===1)return;let r=await d().execute({sql:"SELECT provider_source_key_hash FROM provider_usage_receipts WHERE id = ?",args:[e]});if(!r.rows[0])throw new Error(`Unknown provider receipt: ${e}`);if(String(r.rows[0].provider_source_key_hash)!==s)throw new Error(`Provider receipt ${e} already has a different source identity`)}async function h(e){if(e.units!=null&&(!Number.isFinite(e.units)||e.units<0))throw new Error("units must be non-negative");if(e.amountUsdNanos!=null&&(!Number.isSafeInteger(e.amountUsdNanos)||e.amountUsdNanos<0))throw new Error("amountUsdNanos must be a non-negative safe integer");if(e.costState==="actual"&&e.amountUsdNanos==null)throw new Error("actual receipts require amountUsdNanos");if(e.costState==="actual"&&e.amountUsdNanos===0&&!e.zeroCostReason?.trim())throw new Error("actual zero-cost receipts require zeroCostReason");let t=e.checkedAt??new Date().toISOString(),n=e.finalizedAt??t;if((await d().execute({sql:`UPDATE provider_usage_receipts SET
+      cost_state = ?, units = ?, unit_type = ?, amount_usd_nanos = ?, rate_source = ?, zero_cost_reason = ?,
+      provider_status = ?, provider_duration_ms = ?, provider_captcha = ?, error_code = ?, error_message = ?,
+      checked_at = ?, finalized_at = ?, updated_at = ?
+      WHERE id = ? AND (cost_state = 'pending' OR (cost_state = 'estimated' AND ? = 'actual'))`,args:[e.costState,e.units??null,e.unitType??null,e.amountUsdNanos??null,e.rateSource??null,e.zeroCostReason??null,e.providerStatus??null,e.providerDurationMs??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,m(e.errorCode,200),m(e.errorMessage),t,n,n,e.receiptId,e.costState]})).rowsAffected===1)return;let i=await d().execute({sql:"SELECT cost_state, units, unit_type, amount_usd_nanos FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]});if(i.rows.length===0)throw new Error(`Unknown provider receipt: ${e.receiptId}`);let r=i.rows[0],o=r?.units==null&&e.units==null||Number(r?.units)===e.units,a=r?.amount_usd_nanos==null&&e.amountUsdNanos==null||Number(r?.amount_usd_nanos)===e.amountUsdNanos,c=r?.unit_type==null&&e.unitType==null||String(r?.unit_type)===e.unitType;if(String(r?.cost_state)!==e.costState||!o||!a||!c)throw new Error(`Provider receipt ${e.receiptId} is already finalized as ${String(i.rows[0]?.cost_state)}`)}async function se(e){let n=(await d().execute({sql:`SELECT id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[l(e.provider,"provider"),g(l(e.accountScope,"accountScope")),l(e.method,"method"),g(l(e.providerSourceKey,"providerSourceKey"))]})).rows[0];if(!n)return!1;let{provider:s,accountScope:i,method:r,providerSourceKey:o,...a}=e;return await h({...a,receiptId:String(n.id)}),!0}async function ae(e=30){if(!Number.isInteger(e)||e<1||e>365)throw new Error("retentionDays must be an integer between 1 and 365");let t=`-${e} days`,n=d(),[s,i,r]=await Promise.all([n.execute({sql:`UPDATE operation_attempts
+        SET error_message = NULL, updated_at = datetime('now')
+        WHERE ended_at < datetime('now', ?) AND error_message IS NOT NULL`,args:[t]}),n.execute({sql:`UPDATE provider_usage_receipts
+        SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
+        WHERE finalized_at < datetime('now', ?)
+          AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[t]}),n.execute({sql:`UPDATE harvest_attempts
+        SET provider_session_id = NULL, \u006b\u0065\u0072\u006e\u0065\u006c_session_id = NULL, error = NULL,
+            \u006b\u0065\u0072\u006e\u0065\u006c_delete_error = NULL, browser_close_error = NULL,
+            provider_error_message = NULL, provider_detail_json = NULL,
+            provider_disconnect_message = NULL, debug_json = NULL
+        WHERE completed_at < datetime('now', ?)
+          AND (provider_session_id IS NOT NULL OR \u006b\u0065\u0072\u006e\u0065\u006c_session_id IS NOT NULL OR error IS NOT NULL
+            OR \u006b\u0065\u0072\u006e\u0065\u006c_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
+            OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
+            OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[t]})]);return{operationAttempts:s.rowsAffected,providerReceipts:i.rowsAffected,legacyHarvestAttempts:r.rowsAffected}}async function F(e){let n={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],s=l(e.billingKey,"billingKey"),i=await d().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
+      FROM operation_runs r
+      JOIN ${e.billingTable} b ON CAST(b.${n.column} AS TEXT) = ?
+      WHERE r.id = ?`,args:[s,e.runId]});if(!i.rows[0])throw new Error("Billing link target or operation run does not exist");let r=i.rows[0].billing_user_id,o=i.rows[0].run_user_id;if(o!=null&&r!=null&&String(o)!==String(r))throw new Error("Billing link owner does not match operation run owner");await d().execute({sql:`INSERT OR IGNORE INTO operation_billing_links
+      (run_id, billing_table, billing_key, relation, amount_mc) VALUES (?, ?, ?, ?, ?)`,args:[e.runId,e.billingTable,s,e.relation,e.amountMc??null]})}async function de(e,t=new Date().toISOString()){let n=f();return await d().execute({sql:"INSERT INTO provider_reconciliation_runs (id, provider, started_at) VALUES (?, ?, ?)",args:[n,l(e,"provider"),t]}),n}async function le(e){if([e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount].some(i=>!Number.isInteger(i)||i<0))throw new Error("Reconciliation counts must be non-negative integers");if(e.completedCount+e.pendingCount+e.unavailableCount+e.failedCount>e.checkedCount)throw new Error("Reconciliation outcome counts cannot exceed checkedCount");let n=e.completedAt??new Date().toISOString();if((await d().execute({sql:`UPDATE provider_reconciliation_runs SET
+      status = ?, checked_count = ?, completed_count = ?, pending_count = ?, unavailable_count = ?,
+      failed_count = ?, error_code = ?, completed_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount,m(e.errorCode,200),n,e.id]})).rowsAffected!==1)throw new Error(`Unknown or terminal provider reconciliation run: ${e.id}`)}async function K(e){let t=d(),[n,s,i,r]=await Promise.all([t.execute({sql:"SELECT * FROM operation_runs WHERE id = ?",args:[e]}),t.execute({sql:"SELECT * FROM operation_attempts WHERE run_id = ? ORDER BY attempt_number",args:[e]}),t.execute({sql:`SELECT r.* FROM provider_usage_receipts r
+        JOIN operation_attempts a ON a.id = r.attempt_id
+        WHERE a.run_id = ? ORDER BY a.attempt_number, r.rowid`,args:[e]}),t.execute({sql:"SELECT * FROM operation_billing_links WHERE run_id = ? ORDER BY created_at, billing_table, billing_key",args:[e]})]);return n.rows[0]?{run:n.rows[0],attempts:[...s.rows],providerReceipts:[...i.rows],billingLinks:[...r.rows]}:null}function W(e){return e instanceof Error?e.message:String(e)}function B(e,t){return e instanceof DOMException&&(e.name==="TimeoutError"||e.name==="AbortError")?!0:/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(t)}function V(e){return/captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(e)}function J(e){return/ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(e)}function G(e){return/proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(e)}function Q(e){return/(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(e)}function Y(e){return/browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(e)}function X(e){let t=W(e);return e instanceof A?{error_code:"request_aborted",error_type:"request_aborted",message:p("request_aborted"),retryable:!0,httpStatus:408,terminalStatus:"cancelled"}:e instanceof I||V(t)?{error_code:"captcha_exhausted",error_type:"captcha",message:p("captcha_exhausted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:C(e,t)?{error_code:"vendor_unavailable",error_type:"service_unavailable",message:p("vendor_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:e instanceof O?{error_code:"location_mismatch",error_type:"location_mismatch",message:p("location_mismatch"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:J(t)?{error_code:"proxy_tunnel_failed",error_type:"connection",message:p("proxy_tunnel_failed"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:G(t)?{error_code:"proxy_unavailable",error_type:"connection",message:p("proxy_unavailable"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:Y(t)?{error_code:"browser_session_interrupted",error_type:"extraction",message:p("browser_session_interrupted"),retryable:!0,httpStatus:503,terminalStatus:"failed"}:B(e,t)?{error_code:"harvest_timeout",error_type:"timeout",message:p("harvest_timeout"),retryable:!0,httpStatus:504,terminalStatus:"failed"}:Q(t)?{error_code:"service_unavailable",error_type:"service_unavailable",message:y,retryable:!0,httpStatus:503,terminalStatus:"failed"}:{error_code:"extraction_failed",error_type:"extraction",message:y,retryable:!1,httpStatus:500,terminalStatus:"failed"}}function me(e){return JSON.stringify({error_code:e.error_code,error_type:e.error_type,message:e.message,retryable:e.retryable})}function z(e,t={}){let n=R({errorCode:e.error_code,errorType:e.error_type,retryable:e.retryable,retryAfterSeconds:t.retryAfterSeconds,chargeStatus:t.chargeStatus,details:t.details});return{error:n.message,...n}}function _e(e,t={}){let{error:n,...s}=z(e,t);return s}function ge(e,t={}){let n=X(e);return{body:z(n,t),status:n.httpStatus}}import{createHash as Z}from"crypto";var ee=1e9;function re(e){return Z("sha256").update(JSON.stringify(e)).digest("hex")}function te(e){return`harvest:${e}`}async function we(e){let t=te(e.jobId);try{return(await U({id:t,userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,idempotencyKey:`job:${e.jobId}`,requestFingerprint:re(e.normalizedFlags),jobId:e.jobId,requestId:e.requestId??null,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id}catch(n){return console.warn(JSON.stringify({event:"operation_run_start_failed",job_id:e.jobId,tool:e.tool,message:n instanceof Error?n.message:String(n)})),null}}async function Se(e,t,n){if(e)try{await M({runId:e,status:t,failureClass:n})}catch(s){console.warn(JSON.stringify({event:"operation_run_finish_failed",operation_run_id:e,message:s instanceof Error?s.message:String(s)}))}}async function ve(e,t,n,s){if(!(!e||!t))try{await F({runId:e,billingTable:"billing_debits",billingKey:t,relation:n,amountMc:s})}catch(i){console.warn(JSON.stringify({event:"operation_billing_link_failed",operation_run_id:e,relation:n,message:i instanceof Error?i.message:String(i)}))}}function E(e){return e==="timeout"?"timed_out":e==="request_aborted"?"cancelled":e==="browser_session_interrupted"?"interrupted":e==="captcha"||e==="proxy_tunnel_failed"||e==="proxy_unavailable"||e==="location_mismatch"||e==="error"?"failed":"succeeded"}function oe(e){return e==="\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061":"\u006b\u0065\u0072\u006e\u0065\u006c"}function $(e,t,n=0){let s=new Map,i=null;return async r=>{if(e)try{if(r.type==="started"){let a=n+r.attemptNumber;if(a>1&&!i){let _=(await K(e))?.attempts.at(-1);i=_?.id==null?null:String(_.id)}let c=await q({runId:e,attemptNumber:a,method:r.method,provider:r.provider,parentAttemptId:a===1?null:i,edgeKind:a===1?null:n>0&&r.attemptNumber===1?"durable_resume":r.retryKind==="provider_fallback"?"provider_fallback":"same_method_retry",retryKind:a===1?null:n>0&&r.attemptNumber===1?"durable_resume":r.retryKind,retryReason:a===1?null:n>0&&r.attemptNumber===1?"durable_worker_resume":r.retryReason,retryOwner:a===1?null:n>0&&r.attemptNumber===1?"durable_reconciler":"harvest",normalizedFlags:{maxAttempts:r.maxAttempts,maxQuestions:r.maxQuestions,hasLocation:!!r.location,plannedProvider:r.provider,planReason:r.planReason??null,planTimeoutMs:r.planTimeoutMs??null},startedAt:r.startedAt}),u=await v({attemptId:c,provider:r.provider,accountScope:t,method:r.method,providerStatus:"started"});s.set(r.attemptNumber,{attemptId:c,receiptId:u.id,plannedProvider:r.provider,observedProvider:null,providerMismatch:!1,method:r.method,startedAtMs:Date.parse(r.startedAt)}),i=c;return}let o=s.get(r.attemptNumber);if(!o)throw new Error(`Missing operation attempt state for attempt ${r.attemptNumber}`);if(r.type==="provider_session_started"){let a=oe(r.provider);if(o.observedProvider=a,a!==o.plannedProvider){o.providerMismatch=!0,await h({receiptId:o.receiptId,costState:"unavailable",providerStatus:"provider_mismatch",errorCode:"provider_mismatch",errorMessage:`Planned ${o.plannedProvider}; observed ${a}`,finalizedAt:r.observedAt});let c=a==="\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061"?"\u0062\u0072\u0069\u0067\u0068\u0074\u0064\u0061\u0074\u0061_browser":"\u006b\u0065\u0072\u006e\u0065\u006c_browser",u=await v({attemptId:o.attemptId,provider:a,accountScope:t,method:c,providerStatus:"observed_provider_mismatch"});o.receiptId=u.id,o.method=c}await H(o.receiptId,r.providerSessionId);return}if(await D({attemptId:o.attemptId,status:o.providerMismatch?"failed":E(r.outcome),errorCode:o.providerMismatch?"provider_mismatch":E(r.outcome)==="succeeded"?null:r.outcome,errorMessage:r.error,durationMs:r.durationMs,endedAt:r.completedAt}),(o.observedProvider??o.plannedProvider)==="\u006b\u0065\u0072\u006e\u0065\u006c"){let a=T(r.durationMs,!0);await h({receiptId:o.receiptId,costState:"estimated",units:r.durationMs/1e3,unitType:"browser_second",amountUsdNanos:Math.round(a*ee),rateSource:`\u006b\u0065\u0072\u006e\u0065\u006c_headful_usd_per_second:${k}`,providerStatus:o.providerMismatch?"provider_mismatch":r.outcome,providerDurationMs:r.durationMs,errorCode:o.providerMismatch?"provider_mismatch":E(r.outcome)==="succeeded"?null:r.outcome,errorMessage:r.error,finalizedAt:r.completedAt})}}catch(o){console.warn(JSON.stringify({event:"operation_attempt_telemetry_failed",operation_run_id:e,attempt_number:r.attemptNumber,message:o instanceof Error?o.message:String(o)}))}}}function Ae(e,t,n=null,s="production-default",i=0){let r=$(n,s,i);return async o=>{if(await r(o),o.type==="started"){await N({jobId:e,userId:t,attemptNumber:o.attemptNumber,maxAttempts:o.maxAttempts,query:o.query,location:o.location,maxQuestions:o.maxQuestions,startedAt:o.startedAt});return}if(o.type==="provider_session_started"){await x({jobId:e,attemptNumber:o.attemptNumber,browserProvider:o.provider,providerSessionId:o.providerSessionId,observedAt:o.observedAt});return}await P({jobId:e,attemptNumber:o.attemptNumber,outcome:o.outcome,\u006b\u0065\u0072\u006e\u0065\u006cSessionId:o.\u006b\u0065\u0072\u006e\u0065\u006cSessionId,questionCount:o.questionCount,durationMs:o.durationMs,error:o.error,willRetry:o.willRetry,\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted:o.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted,\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded:o.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded,\u006b\u0065\u0072\u006e\u0065\u006cDeleteError:o.cleanup.\u006b\u0065\u0072\u006e\u0065\u006cDeleteError,browserCloseSucceeded:o.cleanup.browserCloseSucceeded,browserCloseError:o.cleanup.browserCloseError,browserProvider:o.cleanup.provider??null,providerSessionId:o.cleanup.providerSessionId??null,providerLookupStatus:o.cleanup.providerSessionId?"pending":"not_applicable",providerErrorCode:o.cleanup.providerDisconnectObserved?"browser_session_interrupted":null,providerErrorMessage:o.cleanup.providerDisconnectMessage??null,providerReasonSource:o.cleanup.providerDisconnectObserved?"client_runtime":null,providerDisconnectObserved:o.cleanup.providerDisconnectObserved??!1,providerDisconnectMessage:o.cleanup.providerDisconnectMessage??null,debug:o.debug,completedAt:o.completedAt})}}async function Ne(e,t,n=3){for(let s=1;s<=n;s++){try{let i=await fetch(e,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(t),signal:AbortSignal.timeout(1e4)});if(i.ok)return;console.warn(`[webhook] attempt ${s} \u2192 ${i.status} from ${e}`)}catch(i){console.warn(`[webhook] attempt ${s} failed:`,i instanceof Error?i.message:i)}s<n&&await new Promise(i=>setTimeout(i,1e3*s*2))}console.error(`[webhook] gave up after ${n} attempts for ${e}`)}export{q as a,D as b,v as c,H as d,h as e,se as f,ae as g,de as h,le as i,X as j,me as k,z as l,_e as m,ge as n,te as o,we as p,Se as q,ve as r,Ae as s,Ne as t};

package/dist/{chunk-ZMLCK45S.js → chunk-NWRRVZKW.js}

@@ -1,4 +1,4 @@
-import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{_ as w}from"./chunk-YEFXVTMJ.js";import{Jb as N,Sb as D,d as m,e as I,i as u,t as s}from"./chunk-7BEWBRMT.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{_ as w}from"./chunk-CTX3NMZQ.js";import{Jb as N,Sb as D,d as m,e as I,i as u,t as s}from"./chunk-KKHTMYCD.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
             (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs

package/dist/{chunk-2GRE4BAO.js → chunk-Q5SUDDSQ.js}

@@ -1,4 +1,4 @@
-import{Q as p,t as o}from"./chunk-7BEWBRMT.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
+import{Q as p,t as o}from"./chunk-KKHTMYCD.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
       CREATE TABLE IF NOT EXISTS service_connections (
         id                         TEXT PRIMARY KEY,
         user_id                    INTEGER NOT NULL REFERENCES users(id),

package/dist/chunk-UM4WSLKQ.js

@@ -0,0 +1 @@
+var o="0.90.6";export{o as a};

package/dist/{chunk-WTOKMWDS.js → chunk-WDB5CTRY.js}

@@ -1,4 +1,4 @@
-import{f as F,h as V,i as re,j as ae,l as $}from"./chunk-2GRE4BAO.js";import{_ as R,e as U,q as _,t as G,u as ne,v as oe,w as M,y as ie}from"./chunk-YEFXVTMJ.js";import{b as K,h as L}from"./chunk-DKQZNSYM.js";import{Q as te,Rb as D,Wb as x,t as C}from"./chunk-7BEWBRMT.js";import{createHash as Te,randomUUID as Ne}from"crypto";import{z as f}from"zod";async function at(e,t){!Number.isFinite(t)||t<=0||await K({op:R.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>L({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function se(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await K({op:R.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let o of n)!Number.isFinite(o.units)||o.units<=0||await L(o)})}var E="connected_usage",Re="mastra_scheduler",Ie=f.object({toolName:f.string().trim().min(1).max(200).optional(),operationKind:f.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:f.enum(["success","error","partial"]).optional(),requestId:f.string().trim().min(1).max(200).optional(),occurrenceId:f.string().trim().min(1).max(200).optional(),scheduleId:f.string().trim().min(1).max(200).optional(),nangoOperationId:f.string().trim().min(1).max(200).optional(),startedAt:f.string().datetime({offset:!0}).optional(),completedAt:f.string().datetime({offset:!0}).optional()}).strict(),Ae=f.object({identity:f.string().trim().min(1).max(320),idempotencyKey:f.string().trim().min(1).max(500),ratePolicyVersion:f.string().trim().min(1).max(100),connectionId:f.string().trim().min(1).max(500),providerConfigKey:f.string().trim().min(1).max(200),functionRuns:f.number().int().safe().nonnegative(),proxyRequests:f.number().int().safe().nonnegative(),computeMilliseconds:f.number().int().safe().nonnegative(),sourceSurface:f.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Re),metadata:Ie.optional()}).strict(),Pe=f.object({identity:f.string().trim().min(1).max(320),ratePolicyVersion:f.string().trim().min(1).max(100)}).strict(),h=class extends Error{constructor(n,o,a,i={}){super(o);this.code=n;this.httpStatus=a;this.details=i;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function z(e){return Te("sha256").update(e).digest("hex")}function xe(e){return`connected-usage:${z(e)}`}function Ue(e){return`connected-usage-debit:${z(e)}`}function Me(e){return e.trim().toLowerCase()}async function ce(e){let t=await te(Me(e));if(!t)throw new h("user_not_found","billing identity was not found",404);return t}function Oe(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return z(JSON.stringify({ratePolicyVersion:_,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function A(e){let t=JSON.stringify(e);if(t.length>4e3)throw new h("invalid_request","connected usage metadata is too large",400);return t}function W(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==_||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new h("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function H(e){let t=await C().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function de(e,t){return(await C().execute({sql:`SELECT 1
+import{f as F,h as V,i as re,j as ae,l as $}from"./chunk-Q5SUDDSQ.js";import{_ as R,e as U,q as _,t as G,u as ne,v as oe,w as M,y as ie}from"./chunk-CTX3NMZQ.js";import{b as K,k as L}from"./chunk-56UT5M5Y.js";import{Q as te,Rb as D,Wb as x,t as C}from"./chunk-KKHTMYCD.js";import{createHash as Te,randomUUID as Ne}from"crypto";import{z as f}from"zod";async function at(e,t){!Number.isFinite(t)||t<=0||await K({op:R.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>L({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function se(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await K({op:R.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let o of n)!Number.isFinite(o.units)||o.units<=0||await L(o)})}var E="connected_usage",Re="mastra_scheduler",Ie=f.object({toolName:f.string().trim().min(1).max(200).optional(),operationKind:f.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:f.enum(["success","error","partial"]).optional(),requestId:f.string().trim().min(1).max(200).optional(),occurrenceId:f.string().trim().min(1).max(200).optional(),scheduleId:f.string().trim().min(1).max(200).optional(),nangoOperationId:f.string().trim().min(1).max(200).optional(),startedAt:f.string().datetime({offset:!0}).optional(),completedAt:f.string().datetime({offset:!0}).optional()}).strict(),Ae=f.object({identity:f.string().trim().min(1).max(320),idempotencyKey:f.string().trim().min(1).max(500),ratePolicyVersion:f.string().trim().min(1).max(100),connectionId:f.string().trim().min(1).max(500),providerConfigKey:f.string().trim().min(1).max(200),functionRuns:f.number().int().safe().nonnegative(),proxyRequests:f.number().int().safe().nonnegative(),computeMilliseconds:f.number().int().safe().nonnegative(),sourceSurface:f.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Re),metadata:Ie.optional()}).strict(),Pe=f.object({identity:f.string().trim().min(1).max(320),ratePolicyVersion:f.string().trim().min(1).max(100)}).strict(),h=class extends Error{constructor(n,o,a,i={}){super(o);this.code=n;this.httpStatus=a;this.details=i;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function z(e){return Te("sha256").update(e).digest("hex")}function xe(e){return`connected-usage:${z(e)}`}function Ue(e){return`connected-usage-debit:${z(e)}`}function Me(e){return e.trim().toLowerCase()}async function ce(e){let t=await te(Me(e));if(!t)throw new h("user_not_found","billing identity was not found",404);return t}function Oe(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return z(JSON.stringify({ratePolicyVersion:_,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function A(e){let t=JSON.stringify(e);if(t.length>4e3)throw new h("invalid_request","connected usage metadata is too large",400);return t}function W(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==_||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new h("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function H(e){let t=await C().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function de(e,t){return(await C().execute({sql:`SELECT 1
           FROM billing_events
           WHERE user_id = ?
             AND billing_class = ?