mcp-scraper 0.95.1 → 0.96.0

package/dist/chunk-ZUGTYD2I.js

@@ -1,63 +0,0 @@
-import{t as s}from"./chunk-ZPK5LYEN.js";import{createHash as v,randomUUID as m}from"crypto";async function O(e){let r=m();return await s().execute({sql:`INSERT INTO operation_coverage_defects (id, tool, method, phase, error_code, run_id, attempt_id)
-      VALUES (?, ?, ?, ?, ?, ?, ?)`,args:[r,a(e.tool,"tool"),a(e.method,"method"),a(e.phase,"phase"),a(e.errorCode,"errorCode").slice(0,200),e.runId??null,e.attemptId??null]}),r}var E=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function u(e){return v("sha256").update(e).digest("hex")}function f(e){return Array.isArray(e)?e.map(f):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([r],[t])=>r.localeCompare(t)).map(([r,t])=>[r,f(t)])):e}function S(e){return JSON.stringify(f(e??{}))}function a(e,r){let t=e.trim();if(!t)throw new Error(`${r} is required`);return t}function l(e,r=2e3){return e==null?null:e.slice(0,r)}async function I(e){let r=s(),t=a(e.tool,"tool"),i=a(e.ownerScope,"ownerScope"),n=e.idempotencyKey?.trim()||null,o=e.requestFingerprint?.trim()||null;if(n==null!=(o==null))throw new Error("idempotencyKey and requestFingerprint must be supplied together");let d=n?u(`${i}\0${t}\0${n}`):null,c=e.id??m(),g=[c,e.userId??null,u(i),t,d,o,e.jobId??null,e.requestId??null,e.deploymentCommit??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()];if(d){await r.execute({sql:`INSERT OR IGNORE INTO operation_runs
-        (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
-         job_id, request_id, deployment_commit, normalized_flags_json, started_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:g});let p=(await r.execute({sql:"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?",args:[d]})).rows[0];if(!p)throw new Error("Operation run insert did not produce an idempotency receipt");if(String(p.request_fingerprint)!==o)throw new E;return{id:String(p.id),replayed:String(p.id)!==c}}return await r.execute({sql:`INSERT INTO operation_runs
-      (id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
-       job_id, request_id, deployment_commit, normalized_flags_json, started_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:g}),{id:c,replayed:!1}}async function N(e){let r=e.endedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE operation_runs
-      SET status = ?, failure_class = ?, ended_at = ?, updated_at = ?
-      WHERE id = ? AND status = 'running'`,args:[e.status,l(e.failureClass,200),r,r,e.runId]})).rowsAffected===1)return;let i=l(e.failureClass,200),n=await s().execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]});if(n.rows.length===0)throw new Error(`Unknown operation run: ${e.runId}`);let o=n.rows[0]?.failure_class==null?null:String(n.rows[0]?.failure_class);if(String(n.rows[0]?.status)!==e.status||o!==i)throw new Error(`Operation run ${e.runId} is already terminal as ${String(n.rows[0]?.status)}`)}async function A(e){if(!Number.isInteger(e.attemptNumber)||e.attemptNumber<1)throw new Error("attemptNumber must be a positive integer");if(e.attemptNumber===1&&(e.parentAttemptId||e.edgeKind||e.retryKind))throw new Error("The first attempt cannot have a parent or retry edge");if(e.attemptNumber>1&&(!e.parentAttemptId||!e.edgeKind))throw new Error("Every non-first attempt requires a parentAttemptId and edgeKind");if(e.retryKind&&!["same_method_retry","provider_fallback","durable_resume","reconciliation"].includes(e.edgeKind??""))throw new Error("retryKind requires a retry or reconciliation edgeKind");let r=e.id??m();return await s().execute({sql:`INSERT INTO operation_attempts
-      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason, retry_owner,
-       method, provider, normalized_flags_json, started_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[r,e.runId,e.parentAttemptId??null,e.attemptNumber,e.edgeKind??null,e.retryKind??null,l(e.retryReason,500),l(e.retryOwner,100),a(e.method,"method"),e.provider??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()]}),r}async function T(e){let r=m();return await s().execute({sql:`INSERT INTO operation_attempts
-      (id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason,
-       method, provider, normalized_flags_json, started_at)
-      SELECT ?, ?, ?, COALESCE(MAX(attempt_number), 0) + 1, ?, ?, ?, ?, ?, ?, ?
-      FROM operation_attempts WHERE run_id = ?`,args:[r,e.runId,e.parentAttemptId,e.edgeKind,e.retryKind??null,l(e.retryReason,500),a(e.method,"method"),e.provider??null,S(e.normalizedFlags),new Date().toISOString(),e.runId]}),r}async function C(e){if(!Number.isInteger(e.durationMs)||e.durationMs<0)throw new Error("durationMs must be a non-negative integer");let r=e.endedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE operation_attempts SET
-      status = ?, error_code = ?, error_message = ?, ended_at = ?, duration_ms = ?, updated_at = ?
-      WHERE id = ? AND status = 'running'`,args:[e.status,l(e.errorCode,200),l(e.errorMessage),r,e.durationMs,r,e.attemptId]})).rowsAffected===1)return;let i=await s().execute({sql:"SELECT status, duration_ms FROM operation_attempts WHERE id = ?",args:[e.attemptId]});if(i.rows.length===0)throw new Error(`Unknown operation attempt: ${e.attemptId}`);if(String(i.rows[0]?.status)!==e.status||Number(i.rows[0]?.duration_ms)!==e.durationMs)throw new Error(`Operation attempt ${e.attemptId} is already terminal as ${String(i.rows[0]?.status)}`)}async function L(e){let r=a(e.provider,"provider"),t=a(e.method,"method"),i=u(a(e.accountScope,"accountScope")),n=e.providerSourceKey?.trim()||null,o=n?u(n):null,d=e.id??m(),c=await s().execute({sql:`INSERT OR IGNORE INTO provider_usage_receipts
-      (id, attempt_id, provider, account_scope_hash, method, provider_product, billing_tier,
-       provider_source_key, provider_source_key_hash, provider_status)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[d,e.attemptId,r,i,t,e.providerProduct??null,e.billingTier??null,n,o,e.providerStatus??null]});if(!o){if(c.rowsAffected===1)return{id:d,replayed:!1};let w=(await s().execute({sql:"SELECT attempt_id, provider, method FROM provider_usage_receipts WHERE id = ?",args:[d]})).rows[0];if(!w||String(w.attempt_id)!==e.attemptId||String(w.provider)!==r||String(w.method)!==t)throw new Error("Provider receipt ID is already used by a different receipt");return{id:d,replayed:!0}}let _=(await s().execute({sql:`SELECT id, attempt_id FROM provider_usage_receipts
-      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[r,i,t,o]})).rows[0];if(!_)throw new Error("Provider receipt insert did not produce a source receipt");if(String(_.attempt_id)!==e.attemptId)throw new Error("Provider source identity is already attached to a different attempt");return{id:String(_.id),replayed:String(_.id)!==d}}async function x(e,r){let t=a(r,"providerSourceKey"),i=u(t);if((await s().execute({sql:`UPDATE provider_usage_receipts
-      SET provider_source_key = ?, provider_source_key_hash = ?, updated_at = datetime('now')
-      WHERE id = ? AND provider_source_key IS NULL`,args:[t,i,e]})).rowsAffected===1)return;let o=await s().execute({sql:"SELECT provider_source_key_hash FROM provider_usage_receipts WHERE id = ?",args:[e]});if(!o.rows[0])throw new Error(`Unknown provider receipt: ${e}`);if(String(o.rows[0].provider_source_key_hash)!==i)throw new Error(`Provider receipt ${e} already has a different source identity`)}async function h(e){if(!Number.isSafeInteger(e.units)||e.units<0)throw new Error("Observed units must be a non-negative safe integer");if((await s().execute({sql:`UPDATE provider_usage_receipts SET units = ?, unit_type = ?,
-      provider_status = COALESCE(?, provider_status), checked_at = ?, updated_at = datetime('now')
-      WHERE id = ? AND cost_state = 'pending'`,args:[e.units,a(e.unitType,"unitType"),e.providerStatus??null,e.checkedAt??new Date().toISOString(),e.receiptId]})).rowsAffected!==1)throw new Error(`Unknown or terminal pending provider receipt: ${e.receiptId}`)}async function k(e){let t=(await s().execute({sql:`SELECT id FROM provider_usage_receipts
-      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),u(a(e.accountScope,"accountScope")),a(e.method,"method"),u(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];return t?(await h({receiptId:String(t.id),units:e.units,unitType:e.unitType,providerStatus:e.providerStatus,checkedAt:e.checkedAt}),!0):!1}async function y(e){if(e.units!=null&&(!Number.isFinite(e.units)||e.units<0))throw new Error("units must be non-negative");if(e.amountUsdNanos!=null&&(!Number.isSafeInteger(e.amountUsdNanos)||e.amountUsdNanos<0))throw new Error("amountUsdNanos must be a non-negative safe integer");if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos==null)throw new Error(`${e.costState} receipts require amountUsdNanos`);if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos===0&&!e.zeroCostReason?.trim())throw new Error(`${e.costState} zero-cost receipts require zeroCostReason`);if(e.costState==="measured"&&e.amountUsdNanos!==0&&(!e.providerProduct||!e.billingTier||!e.billableEvent||!e.measurementSource||!e.rateSource||e.rateNumerator==null||e.rateDenominator==null||!e.ratePolicyVersion||!e.rateEffectiveAt||e.units==null||!e.unitType))throw new Error("measured receipts require observed product, units, and an applied rate snapshot");if(e.rateNumerator!=null&&(!Number.isSafeInteger(e.rateNumerator)||e.rateNumerator<0))throw new Error("rateNumerator must be a non-negative safe integer");if(e.rateDenominator!=null&&(!Number.isSafeInteger(e.rateDenominator)||e.rateDenominator<1))throw new Error("rateDenominator must be a positive safe integer");let r=e.checkedAt??new Date().toISOString(),t=e.finalizedAt??r;if((await s().execute({sql:`UPDATE provider_usage_receipts SET
-      cost_state = ?, units = COALESCE(?, units), unit_type = COALESCE(?, unit_type),
-      amount_usd_nanos = ?, rate_source = COALESCE(rate_source, ?), zero_cost_reason = COALESCE(?, zero_cost_reason),
-      provider_product = COALESCE(provider_product, ?), billing_tier = COALESCE(billing_tier, ?),
-      billable_event = COALESCE(billable_event, ?), measurement_source = COALESCE(measurement_source, ?),
-      rate_numerator = COALESCE(rate_numerator, ?), rate_denominator = COALESCE(rate_denominator, ?),
-      rate_policy_version = COALESCE(rate_policy_version, ?), rate_effective_at = COALESCE(rate_effective_at, ?),
-      provider_status = ?, provider_duration_ms = ?, provider_captcha = ?, error_code = ?, error_message = ?,
-      checked_at = ?, finalized_at = ?, updated_at = ?
-      WHERE id = ? AND (cost_state = 'pending'
-        OR (cost_state = 'estimated' AND ? IN ('measured', 'actual'))
-        OR (cost_state = 'measured' AND ? = 'actual'))`,args:[e.costState,e.units??null,e.unitType??null,e.amountUsdNanos??null,e.rateSource??null,e.zeroCostReason??null,e.providerProduct??null,e.billingTier??null,e.billableEvent??null,e.measurementSource??null,e.rateNumerator??null,e.rateDenominator??null,e.ratePolicyVersion??null,e.rateEffectiveAt??null,e.providerStatus??null,e.providerDurationMs??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,l(e.errorCode,200),l(e.errorMessage),r,t,t,e.receiptId,e.costState,e.costState]})).rowsAffected===1)return;let n=await s().execute({sql:"SELECT cost_state, units, unit_type, amount_usd_nanos FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]});if(n.rows.length===0)throw new Error(`Unknown provider receipt: ${e.receiptId}`);let o=n.rows[0],d=o?.units==null&&e.units==null||Number(o?.units)===e.units,c=o?.amount_usd_nanos==null&&e.amountUsdNanos==null||Number(o?.amount_usd_nanos)===e.amountUsdNanos,g=o?.unit_type==null&&e.unitType==null||String(o?.unit_type)===e.unitType;if(String(o?.cost_state)!==e.costState||!d||!c||!g)throw new Error(`Provider receipt ${e.receiptId} is already finalized as ${String(n.rows[0]?.cost_state)}`)}async function U(e){let t=(await s().execute({sql:`SELECT id FROM provider_usage_receipts
-      WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),u(a(e.accountScope,"accountScope")),a(e.method,"method"),u(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];if(!t)return!1;let{provider:i,accountScope:n,method:o,providerSourceKey:d,...c}=e;return await y({...c,receiptId:String(t.id)}),!0}async function P(e=30){if(!Number.isInteger(e)||e<1||e>365)throw new Error("retentionDays must be an integer between 1 and 365");let r=`-${e} days`,t=s(),[i,n,o]=await Promise.all([t.execute({sql:`UPDATE operation_attempts
-        SET error_message = NULL, updated_at = datetime('now')
-        WHERE ended_at < datetime('now', ?) AND error_message IS NOT NULL`,args:[r]}),t.execute({sql:`UPDATE provider_usage_receipts
-        SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
-        WHERE finalized_at < datetime('now', ?)
-          AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[r]}),t.execute({sql:`UPDATE harvest_attempts
-        SET provider_session_id = NULL, kernel_session_id = NULL, error = NULL,
-            kernel_delete_error = NULL, browser_close_error = NULL,
-            provider_error_message = NULL, provider_detail_json = NULL,
-            provider_disconnect_message = NULL, debug_json = NULL
-        WHERE completed_at < datetime('now', ?)
-          AND (provider_session_id IS NOT NULL OR kernel_session_id IS NOT NULL OR error IS NOT NULL
-            OR kernel_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
-            OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
-            OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[r]})]);return{operationAttempts:i.rowsAffected,providerReceipts:n.rowsAffected,legacyHarvestAttempts:o.rowsAffected}}async function D(e){let t={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],i=a(e.billingKey,"billingKey"),n=await s().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
-      FROM operation_runs r
-      JOIN ${e.billingTable} b ON CAST(b.${t.column} AS TEXT) = ?
-      WHERE r.id = ?`,args:[i,e.runId]});if(!n.rows[0])throw new Error("Billing link target or operation run does not exist");let o=n.rows[0].billing_user_id,d=n.rows[0].run_user_id;if(d!=null&&o!=null&&String(d)!==String(o))throw new Error("Billing link owner does not match operation run owner");await s().execute({sql:`INSERT OR IGNORE INTO operation_billing_links
-      (run_id, billing_table, billing_key, relation, amount_mc) VALUES (?, ?, ?, ?, ?)`,args:[e.runId,e.billingTable,i,e.relation,e.amountMc??null]})}async function q(e,r=new Date().toISOString()){let t=m();return await s().execute({sql:"INSERT INTO provider_reconciliation_runs (id, provider, started_at) VALUES (?, ?, ?)",args:[t,a(e,"provider"),r]}),t}async function K(e){if([e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount].some(n=>!Number.isInteger(n)||n<0))throw new Error("Reconciliation counts must be non-negative integers");if(e.completedCount+e.pendingCount+e.unavailableCount+e.failedCount>e.checkedCount)throw new Error("Reconciliation outcome counts cannot exceed checkedCount");let t=e.completedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE provider_reconciliation_runs SET
-      status = ?, checked_count = ?, completed_count = ?, pending_count = ?, unavailable_count = ?,
-      failed_count = ?, error_code = ?, completed_at = ?
-      WHERE id = ? AND status = 'running'`,args:[e.status,e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount,l(e.errorCode,200),t,e.id]})).rowsAffected!==1)throw new Error(`Unknown or terminal provider reconciliation run: ${e.id}`)}async function F(e){let r=s(),[t,i,n,o]=await Promise.all([r.execute({sql:"SELECT * FROM operation_runs WHERE id = ?",args:[e]}),r.execute({sql:"SELECT * FROM operation_attempts WHERE run_id = ? ORDER BY attempt_number",args:[e]}),r.execute({sql:`SELECT r.* FROM provider_usage_receipts r
-        JOIN operation_attempts a ON a.id = r.attempt_id
-        WHERE a.run_id = ? ORDER BY a.attempt_number, r.rowid`,args:[e]}),r.execute({sql:"SELECT * FROM operation_billing_links WHERE run_id = ? ORDER BY created_at, billing_table, billing_key",args:[e]})]);return t.rows[0]?{run:t.rows[0],attempts:[...i.rows],providerReceipts:[...n.rows],billingLinks:[...o.rows]}:null}export{O as a,I as b,N as c,A as d,T as e,C as f,L as g,x as h,h as i,k as j,y as k,U as l,P as m,D as n,q as o,K as p,F as q};

package/dist/db-B6RNXKEI.js

@@ -1 +0,0 @@
-import{$ as M,$a as Ma,$b as Mb,A as l,Aa as la,Ab as lb,B as m,Ba as ma,Bb as mb,C as n,Ca as na,Cb as nb,D as o,Da as oa,Db as ob,E as p,Ea as pa,Eb as pb,F as q,Fa as qa,Fb as qb,G as r,Ga as ra,Gb as rb,H as s,Ha as sa,Hb as sb,I as t,Ia as ta,Ib as tb,J as u,Ja as ua,Jb as ub,K as v,Ka as va,Kb as vb,L as w,La as wa,Lb as wb,M as x,Ma as xa,Mb as xb,N as y,Na as ya,Nb as yb,O as z,Oa as za,Ob as zb,P as A,Pa as Aa,Pb as Ab,Q as B,Qa as Ba,Qb as Bb,R as C,Ra as Ca,Rb as Cb,S as D,Sa as Da,Sb as Db,T as E,Ta as Ea,Tb as Eb,U as F,Ua as Fa,Ub as Fb,V as G,Va as Ga,Vb as Gb,W as H,Wa as Ha,Wb as Hb,X as I,Xa as Ia,Xb as Ib,Y as J,Ya as Ja,Yb as Jb,Z as K,Za as Ka,Zb as Kb,_ as L,_a as La,_b as Lb,aa as N,ab as Na,ac as Nb,ba as O,bb as Oa,bc as Ob,ca as P,cb as Pa,cc as Pb,da as Q,db as Qa,dc as Qb,ea as R,eb as Ra,fa as S,fb as Sa,ga as T,gb as Ta,ha as U,hb as Ua,ia as V,ib as Va,ja as W,jb as Wa,ka as X,kb as Xa,la as Y,lb as Ya,ma as Z,mb as Za,na as _,nb as _a,oa as $,ob as $a,p as a,pa as aa,pb as ab,q as b,qa as ba,qb as bb,r as c,ra as ca,rb as cb,s as d,sa as da,sb as db,t as e,ta as ea,tb as eb,u as f,ua as fa,ub as fb,v as g,va as ga,vb as gb,w as h,wa as ha,wb as hb,x as i,xa as ia,xb as ib,y as j,ya as ja,yb as jb,z as k,za as ka,zb as kb}from"./chunk-ZPK5LYEN.js";export{b as CORE_SCHEMA_VERSION,a as CREDIT_LOTS_EPOCH,vb as CREDIT_LOT_TTL,c as SITE_EXTRACT_COUNTER_REPAIR_VERSION,Jb as SiteAuditJobRowSchema,Kb as SiteAuditPhaseLogRowSchema,Aa as advanceWorkflowRunStep,ya as appendWorkflowArtifacts,mb as applySubscriptionInvoiceGrant,W as attachHarvestAttemptProviderSession,Ya as cancelJob,K as checkRateLimit,Ja as checkpointDurablePaaJob,Ia as checkpointJob,t as claimDueStripeEvents,Ha as claimDurablePaaJob,Bb as claimMonthlyFreeRefresh,Ga as claimPendingJob,_a as claimPendingKpoJob,La as completeDurablePaaJob,Oa as completeExpiredDurablePaaJob,Ua as completeJob,bb as completeKpoJob,Va as completeRunningJob,u as completeStripeEvent,va as completeWorkflowRunRecord,n as consumeCode,M as consumePasswordResetToken,Fa as countActiveJobsForUser,Ea as countActiveUsers,R as createJob,S as createJobWithId,Za as createKpoJob,L as createPasswordResetToken,T as createRunningJob,J as createUser,qa as createWorkflowRun,ja as createWorkflowSchedule,yb as creditMc,zb as creditMcIdempotent,Q as deactivateUser,w as deadLetterStripeEvent,Cb as debitMc,Db as debitMcIdempotent,na as deleteWorkflowSchedule,s as enqueueStripeEvent,wb as expireOldLots,Ma as failDurablePaaJob,Pa as failExpiredDurablePaaJob,Wa as failJob,cb as failKpoJob,Xa as failRunningJob,wa as failWorkflowRunRecord,X as finalizeExpiredHarvestAttempts,Y as finishHarvestAttempt,z as generateApiKey,Gb as getBillingLedgerSummary,l as getClient,nb as getConnectedAccountBillingState,e as getDb,Mb as getInboxMessage,aa as getJob,$a as getKpoJob,eb as getKpoPhaseLog,Fb as getLedger,rb as getMemoryPlan,sb as getMemoryProvisioned,U as getOrCreateRunningJob,fa as getPageSnapshot,p as getRefresh,x as getStripeEventQueueHealth,D as getUserByApiKey,C as getUserByEmail,E as getUserById,gb as getUserByStripeCustomerId,F as getUserMemoryCreds,I as getUserStats,Ca as getWorkflowArtifact,ra as getWorkflowRun,sa as getWorkflowRunByIdempotencyKey,za as getWorkflowRunStepState,ka as getWorkflowSchedule,A as hashPassword,Pb as insertInboxMessage,d as inspectCurrentCoreSchema,Ab as ledgerExistsForOperation,Hb as ledgerExistsForStripePI,Qb as listActiveUserIds,oa as listDueWorkflowSchedules,$ as listHarvestAttempts,_ as listHarvestAttemptsPendingProviderTelemetry,Lb as listInboxMessages,ea as listJobs,fb as listKpoJobs,ca as listPendingExtractUrlDispatches,Qa as listRecentTerminalDurablePaaJobs,Sa as listRecoverableDurablePaaJobs,ia as listRequestEvents,Ra as listStaleRunningDurablePaaJobs,P as listUsers,pb as listUsersDueForConnectedAccountBillingReconciliation,Ba as listWorkflowArtifacts,ta as listWorkflowRuns,la as listWorkflowSchedules,db as logKpoPhaseComplete,ha as logRequestEvent,ba as markDispatchedJobRunning,Nb as markInboxMessageOpened,Ta as markJobBillingSettled,ua as markWorkflowRunRunning,pa as markWorkflowScheduleRan,i as migrate,j as migrateCoreSchemaForRelease,xb as migrateExistingBalancesToLots,g as migrateSecuredMainOAuthSchema,ma as patchWorkflowSchedule,m as putCode,o as putRefresh,Ib as reconcileBalanceMc,da as recordExtractUrlDispatchAttempt,Da as recordWorkflowWebhookDelivery,k as registerClient,Ka as renewDurablePaaLease,xa as replaceWorkflowArtifacts,Na as requeueExpiredDurablePaaJob,ub as resolveUserByMemorySubscription,v as retryStripeEvent,O as revokeApiKey,q as revokeRefresh,N as rotateApiKey,r as rotateRefresh,jb as setConcurrencyPackQuantity,kb as setConcurrencySubId,ob as setConnectedAccountBillingState,f as setDbForTesting,ib as setExtraConcurrencySlots,qb as setMemoryPlan,tb as setMemoryProvisioned,H as setPassword,hb as setStripeCustomerId,lb as setSubscriptionTier,G as setUserMemoryCreds,Eb as settleDebitMcIdempotent,Ob as softDeleteInboxMessage,V as startHarvestAttempt,y as stripeEventAlreadyProcessed,Z as updateHarvestAttemptProviderTelemetry,ab as updateKpoJobState,ga as upsertPageSnapshot,B as verifyPassword,h as verifySecuredMainOAuthSchema};

package/dist/extract-bundle-X7SRN63Y.js

@@ -1,26 +0,0 @@
-import{c as Ge,h as ke,i as Qe,j as Ve,l as et}from"./chunk-OS4XFDIG.js";import"./chunk-AK6HVF5F.js";import{b as Ye,c as Xe,h as Ze,j as Ke}from"./chunk-FCJXL4PU.js";import"./chunk-Y6STAKF3.js";import{c as ze,e as Fe,f as He,g as We}from"./chunk-W2BVJ7S2.js";import{c as Je}from"./chunk-DOJMTFLB.js";import"./chunk-ZUGTYD2I.js";import"./chunk-TMB56NCA.js";import"./chunk-EDQ57S4P.js";import"./chunk-6DTXIZY2.js";import"./chunk-HE45FFBU.js";import{t as qe}from"./chunk-ZPK5LYEN.js";import{createWriteStream as se,mkdirSync as ie,readFileSync as xt,readdirSync as kt,rmSync as _t,statSync as Pt,writeFileSync as g}from"fs";import{createHash as G}from"crypto";import{basename as Mt,join as a}from"path";import{tmpdir as It}from"os";import{createGzip as Ct}from"zlib";import{once as Y}from"events";import{finished as ae}from"stream/promises";import{ZipFile as At}from"yazl";import Rt from"p-limit";import{createHash as _e}from"crypto";var gt=500,ht=.9,ft=1e4,yt=5e4;function ge(r,s){let n=10**s;return Math.round(r*n)/n}function wt(r,s){if(r.length===0||r.length!==s.length)throw new Error("Similarity vectors must have the same non-zero dimension.");let n=0,c=0,b=0;for(let w=0;w<r.length;w++)n+=r[w]*s[w],c+=r[w]*r[w],b+=s[w]*s[w];return c===0||b===0?0:Math.max(-1,Math.min(1,n/(Math.sqrt(c)*Math.sqrt(b))))}function St(r){let s=_e("sha256");for(let n of[...r].sort((c,b)=>c.url.localeCompare(b.url)))s.update(n.url),s.update("\0"),s.update(n.contentHash),s.update("\0");return s.digest("hex")}function tt(r){let s=r.replace(/!\[([^\]]*)\]\([^)]*\)/g,"$1").replace(/\[([^\]]+)\]\([^)]*\)/g,"$1").replace(/https?:\/\/\S+/gi," ").replace(/&(?:amp|nbsp|quot|#39);/gi," ").replace(/[#>*_`~|\\-]+/g," ").replace(/[^\p{L}\p{N}]+/gu," ").replace(/\s+/g," ").trim().toLowerCase();return s.length>=12?s:null}function bt(r){let s=r.map(f=>f.bodyMarkdown.split(/\n{2,}/).map(M=>M.trim()).filter(Boolean)),n=r.length>=4?Math.max(3,Math.ceil(r.length*.5)):null,c=new Map;for(let f of s){let M=new Set(f.map(tt).filter(I=>!!I));for(let I of M)c.set(I,(c.get(I)??0)+1)}let b=new Set;if(n!=null)for(let[f,M]of c)M>=n&&b.add(f);let w=0;return{texts:r.map((f,M)=>{let I=[];for(let _ of s[M]){let T=tt(_);if(T&&b.has(T)){w+=_.length;continue}I.push(_)}let A=I.join(`
-
-`).trim(),C=A.length>=60?A:f.bodyMarkdown;return`${f.title?.trim()||"Untitled page"}
-
-${C}`}),minimumPageCount:n,removedBlockSignatures:b.size,removedCharacters:w}}var Pe=class{parent;constructor(s){this.parent=Array.from({length:s},(n,c)=>c)}find(s){let n=this.parent[s];return n!==s&&(this.parent[s]=this.find(n)),this.parent[s]}union(s,n){let c=this.find(s),b=this.find(n);c!==b&&(this.parent[b]=c)}};async function nt(r,s={}){let n=r.filter(i=>i.bodyMarkdown.trim().length>0).slice(0,gt),c=Number.isFinite(s.threshold)?s.threshold:ht,b=Number.isFinite(s.maxPairs)?s.maxPairs:ft,w=Math.max(0,Math.min(1,c)),$=Math.max(1,Math.min(yt,Math.floor(b))),f=s.model??Qe(),M=s.dimensions??Ve(),I=s.embedTexts??et,A=bt(n),C=[],_=new Map,T=[];for(let[i,p]of n.entries()){let y=p.contentHash||_e("sha256").update(p.bodyMarkdown).digest("hex"),k=_.get(y);k==null&&(k=C.length,_.set(y,k),C.push(A.texts[i])),T.push(k)}let j=[];for(let i=0;i<C.length;i+=64)j.push(...await I(C.slice(i,i+64)));if(j.length!==C.length)throw new Error(`Embedding provider returned ${j.length} vectors for ${C.length} unique page bodies.`);let X=T.map(i=>j[i]),le=new Pe(n.length),L=[],O=[];for(let i=0;i<n.length;i++)for(let p=i+1;p<n.length;p++){let y=wt(X[i],X[p]);L.push(y),!(y<w)&&(O.push({source:i,target:p,score:y}),le.union(i,p))}O.sort((i,p)=>p.score-i.score||n[i.source].url.localeCompare(n[p.source].url)||n[i.target].url.localeCompare(n[p.target].url));let v=[...L].sort((i,p)=>i-p),U=i=>v.length?ge(v[Math.floor((v.length-1)*i)],6):null,D=new Map;for(let i=0;i<n.length;i++){let p=le.find(i),y=D.get(p)??[];y.push(i),D.set(p,y)}let N=new Map,z=[...D.values()].filter(i=>i.length>1).sort((i,p)=>p.length-i.length||n[i[0]].url.localeCompare(n[p[0]].url)).map((i,p)=>{let y=`cluster-${String(p+1).padStart(3,"0")}`;for(let k of i)N.set(k,y);return{clusterId:y,pageCount:i.length,urls:i.map(k=>n[k].url).sort()}}),F=O.slice(0,$).map((i,p)=>{let y=n[i.source],k=n[i.target],q=ge(i.score,6);return{sourceUrl:y.url,sourceTitle:y.title,targetUrl:k.url,targetTitle:k.title,similarity:q,similarityPercent:ge(q*100,2),corpusPercentile:v.length<=1?100:ge((1-p/(v.length-1))*100,2),exactContentDuplicate:!!(y.contentHash&&y.contentHash===k.contentHash),sourceWordCount:y.wordCount,targetWordCount:k.wordCount,clusterId:N.get(i.source)??null}}),H=St(n),ue=_e("sha256").update(JSON.stringify({corpusSha256:H,model:f,dimensions:M,threshold:w,maxPairs:$,boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures}})).digest("hex");return{generatedAt:new Date().toISOString(),provider:"jina",model:f,dimensions:M,threshold:w,requestedMaxPairs:$,comparedPages:n.length,possiblePairs:n.length*Math.max(0,n.length-1)/2,qualifyingPairs:O.length,returnedPairs:F.length,pairsTruncated:O.length>F.length,corpusSha256:H,analysisSha256:ue,scoreDistribution:{min:U(0),p25:U(.25),median:U(.5),p75:U(.75),p90:U(.9),max:U(1)},boilerplateRemoval:{method:"corpus_repeated_markdown_blocks",minimumPageCount:A.minimumPageCount,removedBlockSignatures:A.removedBlockSignatures,removedCharacters:A.removedCharacters},rows:F,clusters:z}}var Tt=25,Me=20,Ie=500,vt=8,rt=10*1024*1024,st=100*1024*1024,Ce=25e4;function oe(r){return r.split("#")[0].replace(/\/$/,"")}function it(r){let s=r.replace(/^www\./,"").toLowerCase(),n=s.split(".");return n.length<=2?s:n.slice(-2).join(".")}function jt(r,s){try{let n=new URL(r);return Mt(n.pathname).replace(/[^a-zA-Z0-9._-]/g,"_").slice(0,80)||`image-${s}`}catch{return`image-${s}`}}function Et(){let r=new Map;return s=>{let n=s.replace(/^https?:\/\//,"").replace(/[^a-zA-Z0-9]+/g,"-").replace(/^-+|-+$/g,"").slice(0,80)||"page",c=r.get(n)??0;return r.set(n,c+1),c?`${n}-${c}`:n}}async function*Ae(r){let s=qe(),n=0;for(;;){let c=await s.execute({sql:`SELECT rowid AS cursor_id, page
-            FROM site_extract_pages
-            WHERE job_id = ? AND rowid > ?
-            ORDER BY rowid
-            LIMIT ${Tt}`,args:[r,n]});if(!c.rows.length)return;if(n=Number(c.rows.at(-1)?.cursor_id??n),!Number.isSafeInteger(n)||n<=0)throw new Error("site extract page cursor is invalid");yield c.rows.map(b=>JSON.parse(String(b.page)))}}function $t(r){let s=r==null?"":String(r);return/[",\r\n]/.test(s)?`"${s.replace(/"/g,'""')}"`:s}function Ut(r){return{source_url:r.sourceUrl,source_title:r.sourceTitle,target_url:r.targetUrl,target_title:r.targetTitle,similarity:r.similarity,similarity_percent:r.similarityPercent,corpus_percentile:r.corpusPercentile,exact_content_duplicate:r.exactContentDuplicate,source_word_count:r.sourceWordCount,target_word_count:r.targetWordCount,cluster_id:r.clusterId}}async function an(r,s={}){let n=a(It(),`extract-bundle-${r.id}-${Date.now()}`),c=r.options.downloadImages===!0;if(r.userId==null)throw new Error("site extract artifact owner is missing");let b=Ze(String(r.userId));ie(n,{recursive:!0}),c&&ie(a(n,"images"),{recursive:!0});let w=r.options.captureRenderedDom===!0,$=r.options.semanticSimilarity===!0;w&&ie(a(n,"rendered-dom"),{recursive:!0});try{let f=[],M=[],I=[],A=Et(),C=[],_=r.options.waybackTimeline,T=new Map;for await(let t of Ae(r.id))for(let e of t){T.set(oe(e.url),e.status),f.push({url:e.url,status:e.status,title:e.title,titleLength:e.titleLength,titlePixels:e.titlePixels,metaDescription:e.metaDescription,metaDescLength:e.metaDescLength,h1:e.h1,h1_2:e.h1_2,h2Count:e.h2Count,extractionStatus:e.extractionStatus,indexable:e.indexable,indexabilityReason:e.indexabilityReason,canonicalUrl:e.canonicalUrl?e.url:null,wordCount:e.wordCount,contentHash:e.contentHash,imagesMissingAlt:e.imagesMissingAlt,schemaTypes:e.schemaTypes.length>0?["present"]:[],outlinks:[]});let o=e.pageId??G("sha256").update(e.archivedUrl??e.url).digest("hex"),x=a(n,"pages",o);ie(x,{recursive:!0});let m=e.contentRef?await b(e.contentRef):null,d=m?.html??null,l=m?.markdown??e.bodyMarkdown??null;if($&&e.extractionStatus==="successful"&&l?.trim()&&M.push({url:e.url,title:e.title,bodyMarkdown:l,contentHash:e.contentHash,wordCount:e.wordCount}),w&&d!=null){let Ne=`rendered-dom/${A(e.url)}.html.txt`;g(a(n,Ne),d),I.push({url:e.url,path:Ne,bytes:Buffer.byteLength(d),truncated:e.renderedDomTruncated===!0,sanitized:e.renderedDomSanitized===!0})}let u=d==null?null:`pages/${o}/page.html`,h=l==null?null:`pages/${o}/page.md`,S=l==null||!e.archiveRequestedMonth?null:`pages/${e.archiveRequestedMonth}/${o}.md`;d!=null&&u&&g(a(n,u),d),l!=null&&h&&g(a(n,h),l),l!=null&&S&&(ie(a(n,"pages",e.archiveRequestedMonth),{recursive:!0}),g(a(n,S),l));let{bodyMarkdown:R,contentRef:te,discoveryLinks:ne,...pt}=e,re={version:"site-export-page.v1",...pt,pageId:o,content:{htmlPath:u,markdownPath:h,htmlBytes:d==null?0:Buffer.byteLength(d),markdownBytes:l==null?0:Buffer.byteLength(l),htmlSha256:d==null?null:G("sha256").update(d).digest("hex"),markdownSha256:l==null?null:G("sha256").update(l).digest("hex")}},Le=JSON.stringify(re,null,2),Oe=`pages/${o}/page.json`;g(a(n,Oe),Le),C.push({pageId:o,url:e.archivedUrl??e.url,jsonPath:Oe,htmlPath:u,markdownPath:h,legacyMarkdownPath:S,jsonSha256:G("sha256").update(Le).digest("hex"),htmlSha256:re.content.htmlSha256,markdownSha256:re.content.markdownSha256,htmlBytes:re.content.htmlBytes,markdownBytes:re.content.markdownBytes})}let j=Ct(),X=se(a(n,"links.jsonl.gz"));j.pipe(X);let le=async t=>{j.write(t)||await Y(j,"drain")},L=se(a(n,"schema.jsonl")),O=async t=>{L.write(t)||await Y(L,"drain")},v=se(a(n,"failures.jsonl")),U=async t=>{v.write(t)||await Y(v,"drain")},D=new Map,N=new Map,z=new Map,F=new Map,H=new Map,ue=new Set,i=0,p=0,y=0,k=0,q=0,he=!1,fe="";try{fe=it(new URL(r.startUrl).hostname)}catch{fe=""}let at=Rt(vt),Re=[],Te=[],ve=[],Z=0,ce=0,ye=0,ot=t=>ye+t>st?!1:(ye+=t,!0);for await(let t of Ae(r.id))for(let e of t){if(e.schema?.length&&await O(JSON.stringify({url:e.url,schema:e.schema})+`
-`),e.extractionStatus==="failed"){let{failureCode:m,failureReason:d}=ke(e.failureCode,e.failureReason);await U(JSON.stringify({url:e.url,status:e.status,via:e.via,failureCode:m,failureReason:d,fetchAttempts:e.fetchAttempts??[]})+`
-`)}if(e.imageLinks?.length)for(let[m,d]of e.imageLinks.entries()){let l=G("sha256").update(`${e.url}\0${d}`).digest("hex"),u={imageId:l,sourcePage:e.url,sourceUrl:d,path:null,status:"not_requested",mimeType:null,bytes:null,sha256:null,failureCode:null,failureReason:null,artifactId:null};if(Te.push(u),!c)continue;if(m>=Me){u.status="skipped_page_limit",u.failureCode="page_image_limit",u.failureReason=`Only the first ${Me} image candidates are downloaded per page.`;continue}if(Z>=Ie){u.status="skipped_site_limit",u.failureCode="site_image_limit",u.failureReason=`Only the first ${Ie} image candidates are downloaded per site.`;continue}Z++;let h=`${l}-${jt(d,m)}`,S=a(n,"images");Re.push(at(()=>Ge(d,S,h,{expectedType:"image",maxBytes:rt,consumeBytes:ot})).then(async R=>{let te=xt(R.savedPath),ne=await Xe({ownerId:String(r.userId),jobId:r.id,imageId:l,filename:h,contentType:R.mimeType??"application/octet-stream",content:te,sourceUrl:d,sourcePage:e.url});ve.push(ne),u.status="downloaded",u.path=`images/${h}`,u.artifactId=ne.key,u.mimeType=R.mimeType,u.bytes=R.sizeBytes,u.sha256=G("sha256").update(te).digest("hex")},R=>{ce++,u.status="failed",u.failureCode="image_download_failed",u.failureReason=(R instanceof Error?R.message:String(R)).replace(/\s+/g," ").slice(0,500)}))}let o=oe(e.url),x={int:0,ext:0};for(let m of e.outlinks??[]){let d=(m.rel??"").split(/\s+/).includes("nofollow"),l=oe(m.href),u=T.get(l)??null;if(await le(JSON.stringify({from:e.url,to:m.href,anchor:m.anchor,rel:m.rel,internal:m.internal,nofollow:d,targetStatus:u})+`
-`),k++,q>=Ce){he=!0;continue}if(q++,m.internal){if(x.int++,i++,D.has(l)||D.set(l,new Set),D.get(l).add(o),m.anchor){N.has(l)||N.set(l,new Map);let h=N.get(l);h.set(m.anchor,(h.get(m.anchor)??0)+1)}!d&&(T.get(l)===200||!T.has(l))&&(z.has(o)||z.set(o,new Set),z.get(o).add(l)),u!=null&&u>=400&&(y++,ue.add(e.url))}else{x.ext++;let h="";try{h=it(new URL(m.href).hostname)}catch{continue}if(!h||h===fe)continue;p++;let S=H.get(h)??{links:0,nofollow:0,pages:new Set};S.links++,d&&S.nofollow++,S.pages.add(e.url),H.set(h,S)}}F.set(e.url,x)}let lt=Promise.all([ae(X),ae(L),ae(v)]);j.end(),L.end(),v.end(),await lt,await Promise.all(Re);let W=new Map,we=oe(r.startUrl);W.set(we,0);let Se=[we];for(;Se.length>0;){let t=Se.shift(),e=W.get(t);for(let o of z.get(t)??[])W.has(o)||(W.set(o,e+1),Se.push(o))}let K=new Map;for(let t of f){let e=oe(t.url),o=D.get(e)??new Set,x=N.get(e),m=x?[...x.entries()].sort((l,u)=>u[1]-l[1]).slice(0,3).map(l=>l[0]):[],d=F.get(t.url)??{int:0,ext:0};K.set(t.url,{url:t.url,inlinks:o.size,uniqueInlinks:o.size,outlinksInternal:d.int,outlinksExternal:d.ext,crawlDepth:W.has(e)?W.get(e):null,orphan:o.size===0&&e!==we,topAnchors:m})}let je=He(f,K,ue),Q=We(r.startUrl,f,je,K),V=Ke(r),J={jobId:r.id,startUrl:r.startUrl,status:r.successfulUrls===0?"failed":r.failedUrls>0||r.remainingUrls>0||V.creditTruncated?"partial":"complete",...V,discovered:r.totalUrls,attempted:r.attemptedUrls,successful:r.successfulUrls,failed:r.failedUrls,remaining:r.remainingUrls,linkAnalysis:{truncated:he,analyzedEdges:q,streamedEdges:k,edgeCap:Ce},generatedAt:new Date().toISOString()};Q=["## Crawl delivery",`- Discovered: ${J.discovered}`,`- Attempted: ${J.attempted}`,`- Successful: ${J.successful}`,`- Failed: ${J.failed}`,`- Remaining: ${J.remaining}`,`- Requested page cap: ${V.requestedMaxPages}`,`- Funded page cap: ${V.effectiveMaxPages}`,`- Credit-truncated: ${V.creditTruncated?"yes":"no"}`,`- Link analysis: ${q}/${k} edges${he?` (capped at ${Ce})`:""}`,"",Q].join(`
-`),s.imageAudit&&(Q+=`
-
-${ze(s.imageAudit)}`);let me=[...K.values()],Ee=t=>Math.round(t*10)/10,ee={zero:0,oneToTwo:0,threeToTen:0,elevenPlus:0},$e=0,Ue=0;for(let t of me)$e+=t.inlinks,Ue+=t.outlinksInternal+t.outlinksExternal,t.inlinks===0?ee.zero++:t.inlinks<=2?ee.oneToTwo++:t.inlinks<=10?ee.threeToTen++:ee.elevenPlus++;let be=[...H.entries()].map(([t,e])=>({domain:t,links:e.links,nofollow:e.nofollow,pages:e.pages.size})).sort((t,e)=>e.links-t.links),xe={summary:{internal:{totalLinks:i,pages:f.length,orphans:me.filter(t=>t.orphan).length,brokenInternal:y,avgInlinks:f.length?Ee($e/f.length):0,avgOutlinks:f.length?Ee(Ue/f.length):0,distribution:ee,topByInlinks:[...me].sort((t,e)=>e.inlinks-t.inlinks).slice(0,20).map(t=>({url:t.url,inlinks:t.inlinks,outlinksInternal:t.outlinksInternal,outlinksExternal:t.outlinksExternal}))},external:{totalLinks:p,uniqueDomains:be.length,topDomains:be.slice(0,20)}},externalDomains:be},de=se(a(n,"pages.jsonl")),ut=new Map(C.map(t=>[t.pageId,t])),E=_?se(a(n,"capture-matrix.jsonl")):null,De=new Set,Be=(t,e)=>{let o=e;try{o=new URL(e).href}catch{}return`${t??""}\0${o}`};for await(let t of Ae(r.id))for(let e of t){let o=e.extractionStatus==="failed"?ke(e.failureCode,e.failureReason):{failureCode:null,failureReason:null},{bodyMarkdown:x,schema:m,outlinks:d,contentRef:l,...u}=e,h=K.get(e.url),S=e.pageId?ut.get(e.pageId):void 0,R={...u,...e.extractionStatus==="failed"?o:{},representations:S?{jsonPath:S.jsonPath,htmlPath:S.htmlPath,markdownPath:S.markdownPath,jsonSha256:S.jsonSha256,htmlSha256:S.htmlSha256,markdownSha256:S.markdownSha256,htmlBytes:S.htmlBytes,markdownBytes:S.markdownBytes}:null,inlinks:h?.inlinks??0,crawlDepth:h?.crawlDepth??null,orphan:h?.orphan??!1};if(de.write(JSON.stringify(R)+`
-`)||await Y(de,"drain"),E){let te=Be(e.archiveRequestedMonth,e.originalUrl??e.url);De.add(te);let ne={requestedMonth:e.archiveRequestedMonth??null,captureTimestamp:e.archiveTimestamp??null,originalUrl:e.originalUrl??e.url,archivedUrl:e.archivedUrl??null,status:e.extractionStatus==="successful"?"captured":"failed",httpStatus:e.status,contentHash:e.contentHash||null,wordCount:e.wordCount,failureCode:o.failureCode,failureReason:o.failureReason};E.write(JSON.stringify(ne)+`
-`)||await Y(E,"drain")}}if(E&&_?.timeline?.urls){let t=_.timeline,e=t.urls,o=(()=>{if(t.months?.length)return[...new Set(t.months)].sort();if(!t.from||!t.to)return[];let x=[],m=new Date(`${t.from}-01T00:00:00Z`),d=new Date(`${t.to}-01T00:00:00Z`);for(;m<=d&&x.length<60;)x.push(`${m.getUTCFullYear()}-${String(m.getUTCMonth()+1).padStart(2,"0")}`),m.setUTCMonth(m.getUTCMonth()+(t.intervalMonths??1));return x})();for(let x of o)for(let m of e)De.has(Be(x,m))||E.write(JSON.stringify({requestedMonth:x,captureTimestamp:null,originalUrl:m,archivedUrl:null,status:"missing",httpStatus:null,contentHash:null,wordCount:0,failureCode:"wayback_capture_missing",failureReason:"No successful HTML capture was available in the requested month."})+`
-`)||await Y(E,"drain")}let ct=ae(de),mt=E?ae(E):Promise.resolve();if(de.end(),E?.end(),await Promise.all([ct,mt]),w&&g(a(n,"rendered-dom","manifest.jsonl"),I.map(t=>JSON.stringify(t)).join(`
-`)),$){let t=await nt(M,{threshold:Number(r.options.similarityThreshold??void 0),maxPairs:Number(r.options.similarityMaxPairs??void 0),embedTexts:s.embedSimilarityTexts}),e=t.rows.map(Ut),o=e.length>0?Object.keys(e[0]):["source_url","source_title","target_url","target_title","similarity","similarity_percent","corpus_percentile","exact_content_duplicate","source_word_count","target_word_count","cluster_id"],x=[o.join(","),...e.map(u=>o.map(h=>$t(u[h]??null)).join(","))].join(`
-`)+`
-`,{rows:m,clusters:d,...l}=t;g(a(n,"similarity-table.csv"),x),g(a(n,"similarity.jsonl"),e.map(u=>JSON.stringify(u)).join(`
-`)),g(a(n,"similarity-table-schema.json"),JSON.stringify({tableNameSuggestion:`site_similarity_${new URL(r.startUrl).hostname.replace(/[^a-z0-9]+/gi,"_").replace(/^_+|_+$/g,"").toLowerCase()}`,defaultSort:[{column:"similarity",direction:"desc"}],columns:{source_url:"text",source_title:"text",target_url:"text",target_title:"text",similarity:"number",similarity_percent:"number",corpus_percentile:"number",exact_content_duplicate:"boolean",source_word_count:"integer",target_word_count:"integer",cluster_id:"text"}},null,2)),g(a(n,"similarity-summary.json"),JSON.stringify(l,null,2)),g(a(n,"content-clusters.json"),JSON.stringify(t.clusters,null,2)),Q+=["","## Rendered content similarity",`- Compared pages: ${t.comparedPages}`,`- Raw cosine threshold: ${t.threshold}`,`- Corpus score distribution: min ${t.scoreDistribution.min??"n/a"} / median ${t.scoreDistribution.median??"n/a"} / p90 ${t.scoreDistribution.p90??"n/a"} / max ${t.scoreDistribution.max??"n/a"}`,`- Qualifying pairs: ${t.qualifyingPairs}`,`- Returned table rows: ${t.returnedPairs}${t.pairsTruncated?" (capped)":""}`,`- Multi-page clusters: ${t.clusters.length}`,`- Embedding model: ${t.model} (${t.dimensions} dimensions)`,`- Corpus boilerplate removed: ${t.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${t.boilerplateRemoval.removedCharacters} characters`,`- Corpus SHA-256: ${t.corpusSha256}`].join(`
-`)}g(a(n,"report.md"),Q),g(a(n,"crawl-summary.json"),JSON.stringify(J,null,2)),g(a(n,"images-manifest.jsonl"),Te.map(t=>JSON.stringify(t)).join(`
-`)),g(a(n,"manifest.json"),JSON.stringify({version:"site-export.v1",jobId:r.id,startUrl:r.startUrl,generatedAt:J.generatedAt,outputProfile:{json:!0,html:!0,markdown:!0,imageManifest:!0,imageBytes:c},pages:[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId)),imagesManifestPath:"images-manifest.jsonl",download:"This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."},null,2)),_&&g(a(n,"wayback-manifest.json"),JSON.stringify({rootUrl:_.rootUrl??r.startUrl,requested:_.timeline??null,maxPagesPerSnapshot:_.maxPagesPerSnapshot??null,captureMatrix:"capture-matrix.jsonl"},null,2)),g(a(n,"issues.json"),JSON.stringify(je,null,2)),g(a(n,"link-metrics.jsonl"),me.map(t=>JSON.stringify(t)).join(`
-`)),g(a(n,"link-report.md"),Fe(xe)),g(a(n,"links-summary.json"),JSON.stringify(xe.summary,null,2)),g(a(n,"external-domains.json"),JSON.stringify(xe.externalDomains,null,2)),s.imageAudit&&(g(a(n,"images.jsonl"),s.imageAudit.rows.map(t=>JSON.stringify(t)).join(`
-`)),g(a(n,"images-summary.json"),JSON.stringify(s.imageAudit.summary,null,2))),s.seoAudit&&g(a(n,"seo-audit.json"),JSON.stringify(s.seoAudit,null,2)),s.branding&&g(a(n,"branding.json"),JSON.stringify(s.branding,null,2)),c&&g(a(n,"images-download-summary.json"),JSON.stringify({queued:Z,downloaded:Z-ce,failed:ce,downloadedBytes:ye,perPageCap:Me,perSiteCap:Ie,perFileByteCap:rt,totalByteCap:st},null,2));let P=[{name:"manifest.json",type:"application/json"},{name:"images-manifest.jsonl",type:"application/x-ndjson"},{name:"report.md",type:"text/markdown"},{name:"crawl-summary.json",type:"application/json"},{name:"failures.jsonl",type:"application/x-ndjson"},{name:"schema.jsonl",type:"application/x-ndjson"},{name:"issues.json",type:"application/json"},{name:"pages.jsonl",type:"application/x-ndjson"},{name:"links.jsonl.gz",type:"application/gzip"},{name:"link-metrics.jsonl",type:"application/x-ndjson"},{name:"link-report.md",type:"text/markdown"},{name:"links-summary.json",type:"application/json"},{name:"external-domains.json",type:"application/json"}];if(w){P.push({name:"rendered-dom/manifest.jsonl",type:"application/x-ndjson"});for(let t of I)P.push({name:t.path,type:"text/plain"})}$&&(P.push({name:"similarity-table.csv",type:"text/csv"}),P.push({name:"similarity.jsonl",type:"application/x-ndjson"}),P.push({name:"similarity-table-schema.json",type:"application/json"}),P.push({name:"similarity-summary.json",type:"application/json"}),P.push({name:"content-clusters.json",type:"application/json"})),_&&(P.push({name:"wayback-manifest.json",type:"application/json"}),P.push({name:"capture-matrix.jsonl",type:"application/x-ndjson"})),s.imageAudit&&(P.push({name:"images.jsonl",type:"application/x-ndjson"}),P.push({name:"images-summary.json",type:"application/json"})),s.seoAudit&&P.push({name:"seo-audit.json",type:"application/json"}),s.branding&&P.push({name:"branding.json",type:"application/json"}),c&&P.push({name:"images-download-summary.json",type:"application/json"});let B=new At,dt=Ye({ownerId:String(r.userId),jobId:r.id,createdAt:new Date,content:B.outputStream});for(let t of P)B.addFile(a(n,t.name),t.name);let pe=[...C].sort((t,e)=>t.pageId.localeCompare(e.pageId));for(let t of pe)t.markdownPath&&B.addFile(a(n,t.markdownPath),t.markdownPath);for(let t of pe)t.legacyMarkdownPath&&B.addFile(a(n,t.legacyMarkdownPath),t.legacyMarkdownPath);for(let t of pe)t.htmlPath&&B.addFile(a(n,t.htmlPath),t.htmlPath);for(let t of pe)B.addFile(a(n,t.jsonPath),t.jsonPath);if(c&&Z>ce){let t=a(n,"images");for(let e of kt(t,{recursive:!0})){let o=a(t,e);Pt(o).isFile()&&B.addFile(o,`images/${e.split("\\").join("/")}`)}}return B.end(),[await Je("export_packaging",()=>dt),...ve]}finally{_t(n,{recursive:!0,force:!0})}}export{Ce as MAX_ANALYZED_LINK_EDGES,Tt as SITE_EXTRACT_PAGE_CHUNK,an as assembleExtractArtifacts};

package/dist/gmail-service-FZVJBSUT.js

@@ -1 +0,0 @@
-import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-FN7U6KRI.js";import"./chunk-ZJTUAWZA.js";import"./chunk-L552AQQY.js";import"./chunk-HUV2WTRW.js";import"./chunk-EDQ57S4P.js";import"./chunk-NA76FFGD.js";import"./chunk-6DTXIZY2.js";import"./chunk-ZPK5LYEN.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};

package/dist/lead-list-enrichment-repository-SXYSS7ZX.js

@@ -1 +0,0 @@
-import{A,B,C,D,E,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z}from"./chunk-QQ7OD7BJ.js";import"./chunk-ZPK5LYEN.js";export{a as LEAD_LIST_MAX_DISPATCH_ATTEMPTS,c as LEAD_LIST_RUNNING_STALE_MINUTES,b as LEAD_LIST_START_DEADLINE_MINUTES,s as cancelPendingLeadListEnrichmentRows,C as claimLeadListEnrichmentOutbox,n as claimLeadListEnrichmentRow,u as completeLeadListEnrichmentJob,j as createOrGetLeadListEnrichmentJob,g as ensureLeadListEnrichmentRepositorySchema,z as failExpiredLeadListEnrichmentStart,v as failLeadListEnrichmentJob,B as failStaleRunningLeadListEnrichmentJob,h as getLeadListEnrichmentJob,i as getLeadListEnrichmentJobByIdempotencyKey,o as getLeadListEnrichmentRow,e as leadListEnrichmentFingerprint,f as leadListRowInputDigest,y as listExpiredLeadListEnrichmentStarts,x as listLeadListEnrichmentJobsNeedingSettlement,p as listLeadListEnrichmentRows,A as listStaleRunningLeadListEnrichmentJobs,l as markLeadListEnrichmentJobQueued,m as markLeadListEnrichmentJobRunning,D as markLeadListEnrichmentOutboxDispatched,E as markLeadListEnrichmentOutboxFailed,w as recordLeadListEnrichmentSettlement,r as requestLeadListEnrichmentCancellation,t as saveLeadListEnrichmentDeliverable,q as saveLeadListEnrichmentRowResult,k as setInitializingLeadListEnrichmentConcurrencyLock,d as stableCanonicalJson};

package/dist/operation-metering-OBKSWJVJ.js

@@ -1 +0,0 @@
-import{a,b,c,d,e,f,g}from"./chunk-DOJMTFLB.js";import"./chunk-ZUGTYD2I.js";import"./chunk-6DTXIZY2.js";import"./chunk-ZPK5LYEN.js";export{g as finishExtractUrlCostRun,b as finishMeteredProviderAction,d as linkCurrentCostDebit,f as startExtractUrlCostRun,a as startMeteredProviderAction,c as withLocalCostPhase,e as withLogicalCostRun};