mcp-scraper 0.90.7 → 0.91.0

package/dist/chunk-ORB4RHCK.js

@@ -0,0 +1,63 @@
+import{t as s}from"./chunk-WJ4XFLS4.js";import{createHash as v,randomUUID as m}from"crypto";async function O(e){let r=m();return await s().execute({sql:`INSERT INTO operation_coverage_defects (id, tool, method, phase, error_code, run_id, attempt_id)
+      VALUES (?, ?, ?, ?, ?, ?, ?)`,args:[r,a(e.tool,"tool"),a(e.method,"method"),a(e.phase,"phase"),a(e.errorCode,"errorCode").slice(0,200),e.runId??null,e.attemptId??null]}),r}var E=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function u(e){return v("sha256").update(e).digest("hex")}function f(e){return Array.isArray(e)?e.map(f):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([r],[t])=>r.localeCompare(t)).map(([r,t])=>[r,f(t)])):e}function S(e){return JSON.stringify(f(e??{}))}function a(e,r){let t=e.trim();if(!t)throw new Error(`${r} is required`);return t}function l(e,r=2e3){return e==null?null:e.slice(0,r)}async function I(e){let r=s(),t=a(e.tool,"tool"),i=a(e.ownerScope,"ownerScope"),n=e.idempotencyKey?.trim()||null,o=e.requestFingerprint?.trim()||null;if(n==null!=(o==null))throw new Error("idempotencyKey and requestFingerprint must be supplied together");let d=n?u(`${i}\0${t}\0${n}`):null,c=e.id??m(),g=[c,e.userId??null,u(i),t,d,o,e.jobId??null,e.requestId??null,e.deploymentCommit??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()];if(d){await r.execute({sql:`INSERT OR IGNORE INTO operation_runs
+        (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
+         job_id, request_id, deployment_commit, normalized_flags_json, started_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:g});let p=(await r.execute({sql:"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?",args:[d]})).rows[0];if(!p)throw new Error("Operation run insert did not produce an idempotency receipt");if(String(p.request_fingerprint)!==o)throw new E;return{id:String(p.id),replayed:String(p.id)!==c}}return await r.execute({sql:`INSERT INTO operation_runs
+      (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
+       job_id, request_id, deployment_commit, normalized_flags_json, started_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:g}),{id:c,replayed:!1}}async function N(e){let r=e.endedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE operation_runs
+      SET status = ?, failure_class = ?, ended_at = ?, updated_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,l(e.failureClass,200),r,r,e.runId]})).rowsAffected===1)return;let i=l(e.failureClass,200),n=await s().execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]});if(n.rows.length===0)throw new Error(`Unknown operation run: ${e.runId}`);let o=n.rows[0]?.failure_class==null?null:String(n.rows[0]?.failure_class);if(String(n.rows[0]?.status)!==e.status||o!==i)throw new Error(`Operation run ${e.runId} is already terminal as ${String(n.rows[0]?.status)}`)}async function A(e){if(!Number.isInteger(e.attemptNumber)||e.attemptNumber<1)throw new Error("attemptNumber must be a positive integer");if(e.attemptNumber===1&&(e.parentAttemptId||e.edgeKind||e.retryKind))throw new Error("The first attempt cannot have a parent or retry edge");if(e.attemptNumber>1&&(!e.parentAttemptId||!e.edgeKind))throw new Error("Every non-first attempt requires a parentAttemptId and edgeKind");if(e.retryKind&&!["same_method_retry","provider_fallback","durable_resume","reconciliation"].includes(e.edgeKind??""))throw new Error("retryKind requires a retry or reconciliation edgeKind");let r=e.id??m();return await s().execute({sql:`INSERT INTO operation_attempts
+      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason, retry_owner,
+       method, provider, normalized_flags_json, started_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[r,e.runId,e.parentAttemptId??null,e.attemptNumber,e.edgeKind??null,e.retryKind??null,l(e.retryReason,500),l(e.retryOwner,100),a(e.method,"method"),e.provider??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()]}),r}async function T(e){let r=m();return await s().execute({sql:`INSERT INTO operation_attempts
+      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason,
+       method, provider, normalized_flags_json, started_at)
+      SELECT ?, ?, ?, COALESCE(MAX(attempt_number), 0) + 1, ?, ?, ?, ?, ?, ?, ?
+      FROM operation_attempts WHERE run_id = ?`,args:[r,e.runId,e.parentAttemptId,e.edgeKind,e.retryKind??null,l(e.retryReason,500),a(e.method,"method"),e.provider??null,S(e.normalizedFlags),new Date().toISOString(),e.runId]}),r}async function C(e){if(!Number.isInteger(e.durationMs)||e.durationMs<0)throw new Error("durationMs must be a non-negative integer");let r=e.endedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE operation_attempts SET
+      status = ?, error_code = ?, error_message = ?, ended_at = ?, duration_ms = ?, updated_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,l(e.errorCode,200),l(e.errorMessage),r,e.durationMs,r,e.attemptId]})).rowsAffected===1)return;let i=await s().execute({sql:"SELECT status, duration_ms FROM operation_attempts WHERE id = ?",args:[e.attemptId]});if(i.rows.length===0)throw new Error(`Unknown operation attempt: ${e.attemptId}`);if(String(i.rows[0]?.status)!==e.status||Number(i.rows[0]?.duration_ms)!==e.durationMs)throw new Error(`Operation attempt ${e.attemptId} is already terminal as ${String(i.rows[0]?.status)}`)}async function L(e){let r=a(e.provider,"provider"),t=a(e.method,"method"),i=u(a(e.accountScope,"accountScope")),n=e.providerSourceKey?.trim()||null,o=n?u(n):null,d=e.id??m(),c=await s().execute({sql:`INSERT OR IGNORE INTO provider_usage_receipts
+      (id, attempt_id, provider, account_scope_hash, method, provider_product, billing_tier,
+       provider_source_key, provider_source_key_hash, provider_status)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[d,e.attemptId,r,i,t,e.providerProduct??null,e.billingTier??null,n,o,e.providerStatus??null]});if(!o){if(c.rowsAffected===1)return{id:d,replayed:!1};let w=(await s().execute({sql:"SELECT attempt_id, provider, method FROM provider_usage_receipts WHERE id = ?",args:[d]})).rows[0];if(!w||String(w.attempt_id)!==e.attemptId||String(w.provider)!==r||String(w.method)!==t)throw new Error("Provider receipt ID is already used by a different receipt");return{id:d,replayed:!0}}let _=(await s().execute({sql:`SELECT id, attempt_id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[r,i,t,o]})).rows[0];if(!_)throw new Error("Provider receipt insert did not produce a source receipt");if(String(_.attempt_id)!==e.attemptId)throw new Error("Provider source identity is already attached to a different attempt");return{id:String(_.id),replayed:String(_.id)!==d}}async function x(e,r){let t=a(r,"providerSourceKey"),i=u(t);if((await s().execute({sql:`UPDATE provider_usage_receipts
+      SET provider_source_key = ?, provider_source_key_hash = ?, updated_at = datetime('now')
+      WHERE id = ? AND provider_source_key IS NULL`,args:[t,i,e]})).rowsAffected===1)return;let o=await s().execute({sql:"SELECT provider_source_key_hash FROM provider_usage_receipts WHERE id = ?",args:[e]});if(!o.rows[0])throw new Error(`Unknown provider receipt: ${e}`);if(String(o.rows[0].provider_source_key_hash)!==i)throw new Error(`Provider receipt ${e} already has a different source identity`)}async function h(e){if(!Number.isSafeInteger(e.units)||e.units<0)throw new Error("Observed units must be a non-negative safe integer");if((await s().execute({sql:`UPDATE provider_usage_receipts SET units = ?, unit_type = ?,
+      provider_status = COALESCE(?, provider_status), checked_at = ?, updated_at = datetime('now')
+      WHERE id = ? AND cost_state = 'pending'`,args:[e.units,a(e.unitType,"unitType"),e.providerStatus??null,e.checkedAt??new Date().toISOString(),e.receiptId]})).rowsAffected!==1)throw new Error(`Unknown or terminal pending provider receipt: ${e.receiptId}`)}async function k(e){let t=(await s().execute({sql:`SELECT id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),u(a(e.accountScope,"accountScope")),a(e.method,"method"),u(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];return t?(await h({receiptId:String(t.id),units:e.units,unitType:e.unitType,providerStatus:e.providerStatus,checkedAt:e.checkedAt}),!0):!1}async function y(e){if(e.units!=null&&(!Number.isFinite(e.units)||e.units<0))throw new Error("units must be non-negative");if(e.amountUsdNanos!=null&&(!Number.isSafeInteger(e.amountUsdNanos)||e.amountUsdNanos<0))throw new Error("amountUsdNanos must be a non-negative safe integer");if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos==null)throw new Error(`${e.costState} receipts require amountUsdNanos`);if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos===0&&!e.zeroCostReason?.trim())throw new Error(`${e.costState} zero-cost receipts require zeroCostReason`);if(e.costState==="measured"&&e.amountUsdNanos!==0&&(!e.providerProduct||!e.billingTier||!e.billableEvent||!e.measurementSource||!e.rateSource||e.rateNumerator==null||e.rateDenominator==null||!e.ratePolicyVersion||!e.rateEffectiveAt||e.units==null||!e.unitType))throw new Error("measured receipts require observed product, units, and an applied rate snapshot");if(e.rateNumerator!=null&&(!Number.isSafeInteger(e.rateNumerator)||e.rateNumerator<0))throw new Error("rateNumerator must be a non-negative safe integer");if(e.rateDenominator!=null&&(!Number.isSafeInteger(e.rateDenominator)||e.rateDenominator<1))throw new Error("rateDenominator must be a positive safe integer");let r=e.checkedAt??new Date().toISOString(),t=e.finalizedAt??r;if((await s().execute({sql:`UPDATE provider_usage_receipts SET
+      cost_state = ?, units = COALESCE(?, units), unit_type = COALESCE(?, unit_type),
+      amount_usd_nanos = ?, rate_source = COALESCE(rate_source, ?), zero_cost_reason = COALESCE(?, zero_cost_reason),
+      provider_product = COALESCE(provider_product, ?), billing_tier = COALESCE(billing_tier, ?),
+      billable_event = COALESCE(billable_event, ?), measurement_source = COALESCE(measurement_source, ?),
+      rate_numerator = COALESCE(rate_numerator, ?), rate_denominator = COALESCE(rate_denominator, ?),
+      rate_policy_version = COALESCE(rate_policy_version, ?), rate_effective_at = COALESCE(rate_effective_at, ?),
+      provider_status = ?, provider_duration_ms = ?, provider_captcha = ?, error_code = ?, error_message = ?,
+      checked_at = ?, finalized_at = ?, updated_at = ?
+      WHERE id = ? AND (cost_state = 'pending'
+        OR (cost_state = 'estimated' AND ? IN ('measured', 'actual'))
+        OR (cost_state = 'measured' AND ? = 'actual'))`,args:[e.costState,e.units??null,e.unitType??null,e.amountUsdNanos??null,e.rateSource??null,e.zeroCostReason??null,e.providerProduct??null,e.billingTier??null,e.billableEvent??null,e.measurementSource??null,e.rateNumerator??null,e.rateDenominator??null,e.ratePolicyVersion??null,e.rateEffectiveAt??null,e.providerStatus??null,e.providerDurationMs??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,l(e.errorCode,200),l(e.errorMessage),r,t,t,e.receiptId,e.costState,e.costState]})).rowsAffected===1)return;let n=await s().execute({sql:"SELECT cost_state, units, unit_type, amount_usd_nanos FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]});if(n.rows.length===0)throw new Error(`Unknown provider receipt: ${e.receiptId}`);let o=n.rows[0],d=o?.units==null&&e.units==null||Number(o?.units)===e.units,c=o?.amount_usd_nanos==null&&e.amountUsdNanos==null||Number(o?.amount_usd_nanos)===e.amountUsdNanos,g=o?.unit_type==null&&e.unitType==null||String(o?.unit_type)===e.unitType;if(String(o?.cost_state)!==e.costState||!d||!c||!g)throw new Error(`Provider receipt ${e.receiptId} is already finalized as ${String(n.rows[0]?.cost_state)}`)}async function U(e){let t=(await s().execute({sql:`SELECT id FROM provider_usage_receipts
+      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),u(a(e.accountScope,"accountScope")),a(e.method,"method"),u(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];if(!t)return!1;let{provider:i,accountScope:n,method:o,providerSourceKey:d,...c}=e;return await y({...c,receiptId:String(t.id)}),!0}async function P(e=30){if(!Number.isInteger(e)||e<1||e>365)throw new Error("retentionDays must be an integer between 1 and 365");let r=`-${e} days`,t=s(),[i,n,o]=await Promise.all([t.execute({sql:`UPDATE operation_attempts
+        SET error_message = NULL, updated_at = datetime('now')
+        WHERE ended_at < datetime('now', ?) AND error_message IS NOT NULL`,args:[r]}),t.execute({sql:`UPDATE provider_usage_receipts
+        SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
+        WHERE finalized_at < datetime('now', ?)
+          AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[r]}),t.execute({sql:`UPDATE harvest_attempts
+        SET provider_session_id = NULL, \u006b\u0065\u0072\u006e\u0065\u006c_session_id = NULL, error = NULL,
+            \u006b\u0065\u0072\u006e\u0065\u006c_delete_error = NULL, browser_close_error = NULL,
+            provider_error_message = NULL, provider_detail_json = NULL,
+            provider_disconnect_message = NULL, debug_json = NULL
+        WHERE completed_at < datetime('now', ?)
+          AND (provider_session_id IS NOT NULL OR \u006b\u0065\u0072\u006e\u0065\u006c_session_id IS NOT NULL OR error IS NOT NULL
+            OR \u006b\u0065\u0072\u006e\u0065\u006c_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
+            OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
+            OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[r]})]);return{operationAttempts:i.rowsAffected,providerReceipts:n.rowsAffected,legacyHarvestAttempts:o.rowsAffected}}async function D(e){let t={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],i=a(e.billingKey,"billingKey"),n=await s().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
+      FROM operation_runs r
+      JOIN ${e.billingTable} b ON CAST(b.${t.column} AS TEXT) = ?
+      WHERE r.id = ?`,args:[i,e.runId]});if(!n.rows[0])throw new Error("Billing link target or operation run does not exist");let o=n.rows[0].billing_user_id,d=n.rows[0].run_user_id;if(d!=null&&o!=null&&String(d)!==String(o))throw new Error("Billing link owner does not match operation run owner");await s().execute({sql:`INSERT OR IGNORE INTO operation_billing_links
+      (run_id, billing_table, billing_key, relation, amount_mc) VALUES (?, ?, ?, ?, ?)`,args:[e.runId,e.billingTable,i,e.relation,e.amountMc??null]})}async function q(e,r=new Date().toISOString()){let t=m();return await s().execute({sql:"INSERT INTO provider_reconciliation_runs (id, provider, started_at) VALUES (?, ?, ?)",args:[t,a(e,"provider"),r]}),t}async function K(e){if([e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount].some(n=>!Number.isInteger(n)||n<0))throw new Error("Reconciliation counts must be non-negative integers");if(e.completedCount+e.pendingCount+e.unavailableCount+e.failedCount>e.checkedCount)throw new Error("Reconciliation outcome counts cannot exceed checkedCount");let t=e.completedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE provider_reconciliation_runs SET
+      status = ?, checked_count = ?, completed_count = ?, pending_count = ?, unavailable_count = ?,
+      failed_count = ?, error_code = ?, completed_at = ?
+      WHERE id = ? AND status = 'running'`,args:[e.status,e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount,l(e.errorCode,200),t,e.id]})).rowsAffected!==1)throw new Error(`Unknown or terminal provider reconciliation run: ${e.id}`)}async function F(e){let r=s(),[t,i,n,o]=await Promise.all([r.execute({sql:"SELECT * FROM operation_runs WHERE id = ?",args:[e]}),r.execute({sql:"SELECT * FROM operation_attempts WHERE run_id = ? ORDER BY attempt_number",args:[e]}),r.execute({sql:`SELECT r.* FROM provider_usage_receipts r
+        JOIN operation_attempts a ON a.id = r.attempt_id
+        WHERE a.run_id = ? ORDER BY a.attempt_number, r.rowid`,args:[e]}),r.execute({sql:"SELECT * FROM operation_billing_links WHERE run_id = ? ORDER BY created_at, billing_table, billing_key",args:[e]})]);return t.rows[0]?{run:t.rows[0],attempts:[...i.rows],providerReceipts:[...n.rows],billingLinks:[...o.rows]}:null}export{O as a,I as b,N as c,A as d,T as e,C as f,L as g,x as h,h as i,k as j,y as k,U as l,P as m,D as n,q as o,K as p,F as q};

package/dist/{chunk-LPEIHMYH.js → chunk-POTKJRLE.js}

@@ -1,4 +1,4 @@
-import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{_ as w}from"./chunk-CTX3NMZQ.js";import{Jb as N,Sb as D,d as m,e as I,i as u,t as s}from"./chunk-TOJBPATJ.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{_ as w}from"./chunk-72YO6PKN.js";import{Kb as N,Tb as D,d as m,e as I,i as u,t as s}from"./chunk-WJ4XFLS4.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
             (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
           VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs

package/dist/{chunk-IGJKAECE.js → chunk-QTDLZTQ7.js}

@@ -1,4 +1,4 @@
-import{Q as p,t as o}from"./chunk-TOJBPATJ.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
+import{R as p,t as o}from"./chunk-WJ4XFLS4.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
       CREATE TABLE IF NOT EXISTS service_connections (
         id                         TEXT PRIMARY KEY,
         user_id                    INTEGER NOT NULL REFERENCES users(id),

package/dist/chunk-RK2VCTZI.js

@@ -0,0 +1 @@
+import{a as w,b as c,c as u,d as m,e as _,f as l,g as y,n as g,q as p}from"./chunk-ORB4RHCK.js";import{b as f,c as s}from"./chunk-6DTXIZY2.js";async function C(t){let r=s(),n=!r?.operationRunId,o=r?.operationRunId??(await c({userId:r?.userId??null,ownerScope:r?.userId!=null?`user:${r.userId}`:"system:unscoped",tool:t.tool,normalizedFlags:{subOperation:r?.subOp??null},deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,a=n?null:await p(o),i=a?.attempts.at(-1),e=t.parentAttemptId??r?.operationRootAttemptId,d=e?await _({runId:o,parentAttemptId:e,edgeKind:t.edgeKind??"parallel_child",method:t.method,provider:t.provider,retryKind:t.retryKind,retryReason:t.retryReason}):await m({runId:o,attemptNumber:(a?.attempts.length??0)+1,method:t.method,provider:t.provider,...i?{parentAttemptId:String(i.id),edgeKind:"next_phase"}:{}}),b;try{b=(await y({attemptId:d,provider:t.provider,providerProduct:t.providerProduct,billingTier:t.billingTier,accountScope:r?.operationAccountScope??"production-default",method:t.method,providerStatus:"started"})).id}catch(I){throw await w({tool:t.tool,method:t.method,phase:"pending_receipt_create",errorCode:I instanceof Error?I.name:"UnknownError",runId:o,attemptId:d}).catch(()=>{}),await l({attemptId:d,status:"failed",errorCode:"pending_receipt_create_failed",durationMs:0}).catch(()=>{}),I}return{runId:o,attemptId:d,receiptId:b,ownsRun:n,startedAtMs:Date.now()}}async function S(t,r,n){await l({attemptId:t.attemptId,status:r,errorCode:n??null,durationMs:Date.now()-t.startedAtMs}),t.ownsRun&&await u({runId:t.runId,status:r,failureClass:n??null})}async function R(t,r){let n=s(),o=null,a=Date.now();if(n?.operationRunId&&n.operationRootAttemptId)try{o=await _({runId:n.operationRunId,parentAttemptId:n.operationRootAttemptId,edgeKind:"parallel_child",method:t})}catch(e){console.warn(JSON.stringify({event:"local_cost_phase_start_failed",method:t,message:e instanceof Error?e.message:String(e)}))}let i="failed";try{let e=await r();return i="succeeded",e}finally{o&&await l({attemptId:o,status:i,durationMs:Date.now()-a,errorCode:i==="failed"?"local_phase_failed":null}).catch(e=>console.warn(JSON.stringify({event:"local_cost_phase_finish_failed",method:t,message:e instanceof Error?e.message:String(e)})))}}async function T(t,r){let n=s()?.operationRunId;n&&await g({runId:n,billingTable:"billing_debits",billingKey:t,relation:"debit",amountMc:r}).catch(o=>console.warn(JSON.stringify({event:"cost_billing_link_failed",message:o instanceof Error?o.message:String(o)})))}async function M(t,r){let n=s();if(n?.operationRunId)return f({...n,op:t.op,userId:t.userId},r);let o=null,a=null;try{o=(await c({userId:t.userId,ownerScope:`user:${t.userId}`,tool:t.tool,normalizedFlags:t.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,t.billingDebitKey&&await g({runId:o,billingTable:"billing_debits",billingKey:t.billingDebitKey,relation:"debit"}).catch(e=>console.warn(JSON.stringify({event:"cost_billing_link_failed",tool:t.tool,message:e instanceof Error?e.message:String(e)}))),a=await m({runId:o,attemptNumber:1,method:`${t.tool}_root`,normalizedFlags:t.normalizedFlags})}catch(e){console.warn(JSON.stringify({event:"logical_cost_run_start_failed",tool:t.tool,message:e instanceof Error?e.message:String(e)}))}let i="failed";try{let e=await f({...n,op:t.op,userId:t.userId,operationRunId:o,operationRootAttemptId:a},r);return i=e instanceof Response&&e.status>=400?"failed":"succeeded",e}finally{o&&a?await h({runId:o,rootAttemptId:a,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(e=>console.warn(JSON.stringify({event:"logical_cost_run_finish_failed",tool:t.tool,message:e instanceof Error?e.message:String(e)}))):o&&await u({runId:o,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(()=>{})}}async function O(t){let r=await c({...t.jobId?{id:`extract-url:${t.jobId}`,idempotencyKey:`job:${t.jobId}`,requestFingerprint:t.jobId,jobId:t.jobId}:{},userId:t.userId,ownerScope:`user:${t.userId}`,tool:"extract_url",normalizedFlags:t.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null}),n=r.replayed?await p(r.id):null,o=n?.attempts[0]?.id==null?await m({runId:r.id,attemptNumber:1,method:"extract_url_root",normalizedFlags:t.normalizedFlags}):String(n.attempts[0].id);return t.billingDebitKey&&await g({runId:r.id,billingTable:"billing_debits",billingKey:t.billingDebitKey,relation:"debit"}),{runId:r.id,rootAttemptId:o}}async function h(t){let r=await p(t.runId),n=r?.attempts.find(o=>String(o.id)===t.rootAttemptId);n?.status==="running"&&await l({attemptId:t.rootAttemptId,status:t.status,errorCode:t.failureClass??null,durationMs:Math.max(0,Date.now()-Date.parse(String(n.started_at)))}),r?.run.status==="running"&&await u({runId:t.runId,status:t.status,failureClass:t.failureClass??null})}export{C as a,S as b,R as c,T as d,M as e,O as f,h as g};

package/dist/{chunk-OT2HUS2G.js → chunk-VEP42LGR.js}

@@ -1,4 +1,4 @@
-import{k as K}from"./chunk-YTEAWZEN.js";import{a as X}from"./chunk-IGJKAECE.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-TOJBPATJ.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
+import{k as K}from"./chunk-C3QWOZUH.js";import{a as X}from"./chunk-QTDLZTQ7.js";import{c as J,d as H}from"./chunk-HUV2WTRW.js";import{t as f}from"./chunk-WJ4XFLS4.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
       CREATE TABLE IF NOT EXISTS gmail_selections (
         selection_id TEXT PRIMARY KEY,
         owner_identity TEXT NOT NULL,