mcp-scraper 0.95.1 → 0.96.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/CHANGELOG.md +98 -39
  2. package/README.md +25 -7
  3. package/THIRD_PARTY_NOTICES.html +203 -0
  4. package/dist/actions-YCMJY3PF.js +1 -0
  5. package/dist/artifacts-4ZDNP2JW.js +1 -0
  6. package/dist/artifacts-N62OPIT5.js +1 -0
  7. package/dist/artifacts-PSADD5OC.js +1 -0
  8. package/dist/artifacts-RUR3OHAK.js +1 -0
  9. package/dist/bin/api-server.js +1 -1
  10. package/dist/bin/mcp-scraper-cli.js +22 -22
  11. package/dist/bin/mcp-scraper-core.js +1 -1
  12. package/dist/bin/mcp-scraper-install.js +1 -1
  13. package/dist/bin/mcp-stdio-server.js +1 -1
  14. package/dist/bin/paa-harvest.js +1 -1
  15. package/dist/blob-store-GIH26EDO.js +1 -0
  16. package/dist/bundle-TTJP6DHU.js +1 -0
  17. package/dist/chunk-24YDQ3EO.js +5 -0
  18. package/dist/chunk-2PRH4Z3N.js +181 -0
  19. package/dist/chunk-3F7AFM53.js +1 -0
  20. package/dist/chunk-4HZXATHB.js +27 -0
  21. package/dist/chunk-55ZANTHL.js +3 -0
  22. package/dist/chunk-5BKSKLHX.js +1 -0
  23. package/dist/chunk-63VWBMYR.js +3 -0
  24. package/dist/chunk-6N5EC7ZO.js +1 -0
  25. package/dist/chunk-6SNOUSSZ.js +182 -0
  26. package/dist/chunk-7POHNTVW.js +1 -0
  27. package/dist/chunk-7SR7MEU5.js +62 -0
  28. package/dist/chunk-7UHDOIUE.js +1 -0
  29. package/dist/chunk-ARSQZCTJ.js +945 -0
  30. package/dist/chunk-B3JRL45T.js +1 -0
  31. package/dist/chunk-B4ZKPLSP.js +1 -0
  32. package/dist/chunk-C7MGJUTU.js +1 -0
  33. package/dist/{chunk-W2BVJ7S2.js → chunk-CNSVZTTT.js} +4 -4
  34. package/dist/chunk-DGAW7FD7.js +12 -0
  35. package/dist/chunk-FRXKMOXI.js +41 -0
  36. package/dist/chunk-GVYGAZV7.js +1 -0
  37. package/dist/chunk-HCK6PLM3.js +3 -0
  38. package/dist/{chunk-I3HNHCIJ.js → chunk-HJIOH7HF.js} +199 -84
  39. package/dist/chunk-HNNYXMSI.js +1 -0
  40. package/dist/chunk-HUNSEGUN.js +57 -0
  41. package/dist/chunk-IFPION5D.js +121 -0
  42. package/dist/{chunk-DYDAQQMC.js → chunk-JAV5V3JT.js} +41 -8
  43. package/dist/chunk-JMZIDHYQ.js +2 -0
  44. package/dist/{chunk-FN7U6KRI.js → chunk-JW4GDY7S.js} +1 -1
  45. package/dist/chunk-KXJCGV3H.js +1 -0
  46. package/dist/chunk-LFRAA3MY.js +4 -0
  47. package/dist/{chunk-Y6STAKF3.js → chunk-M5QHXNFZ.js} +3 -3
  48. package/dist/chunk-MVMWPKRC.js +2 -0
  49. package/dist/chunk-NALXKTUU.js +1 -0
  50. package/dist/chunk-NNOX3FYG.js +198 -0
  51. package/dist/chunk-OB6E6UTD.js +1 -0
  52. package/dist/chunk-OILOHQNL.js +2 -0
  53. package/dist/chunk-OLYPLX5C.js +1 -0
  54. package/dist/chunk-QER2ELZP.js +1 -0
  55. package/dist/chunk-QWE7VKIX.js +1 -0
  56. package/dist/chunk-R5LJL6IR.js +225 -0
  57. package/dist/chunk-RDHGQ2WZ.js +1 -0
  58. package/dist/chunk-SDQW7IJN.js +1 -0
  59. package/dist/chunk-SEZETSB3.js +1 -0
  60. package/dist/chunk-SNJR5N36.js +1 -0
  61. package/dist/chunk-TPN25AFZ.js +159 -0
  62. package/dist/{chunk-AK6HVF5F.js → chunk-U4WIJZCX.js} +1 -1
  63. package/dist/chunk-UIOSADPB.js +1718 -0
  64. package/dist/{chunk-DYKSPHHR.js → chunk-UNEJYZ4T.js} +1 -1
  65. package/dist/chunk-VAC3SQ5T.js +16 -0
  66. package/dist/chunk-VVSHZV5S.js +73 -0
  67. package/dist/chunk-WNUXMPVX.js +1 -0
  68. package/dist/chunk-WP3QS4CJ.js +1 -0
  69. package/dist/chunk-ZPQNY5TI.js +1 -0
  70. package/dist/contracts-GUXGFCVV.js +1 -0
  71. package/dist/core-schema-423BSTE6.js +936 -0
  72. package/dist/dispatch-3HY4735M.js +1 -0
  73. package/dist/index.cjs +104 -214
  74. package/dist/index.d.cts +192 -180
  75. package/dist/index.d.ts +192 -180
  76. package/dist/index.js +1 -1
  77. package/dist/media-artifacts-67CJWI7O.js +1 -0
  78. package/dist/metering-ZH2P62IC.js +1 -0
  79. package/dist/migrations-3AKEC5QE.js +1 -0
  80. package/dist/queue-HUP2GNXF.js +1 -0
  81. package/dist/reconciliation-ANXNJWME.js +1 -0
  82. package/dist/report-artifact-offload-6OOY2PTM.js +1 -0
  83. package/dist/repository-4EC5EXVN.js +1 -0
  84. package/dist/{location-data-repository-NWWKDYG6.js → repository-CYXVMYAI.js} +1 -1
  85. package/dist/server-VQU6S24A.js +5825 -0
  86. package/dist/settlement-6CCE2PDH.js +1 -0
  87. package/dist/worker-5G5BK23B.js +1 -0
  88. package/package.json +18 -138
  89. package/dist/analytics-repository-A7VMA24D.js +0 -1
  90. package/dist/chunk-56C243PX.js +0 -2595
  91. package/dist/chunk-6ND5AFV5.js +0 -27
  92. package/dist/chunk-6NE4OIXR.js +0 -6
  93. package/dist/chunk-DOJMTFLB.js +0 -1
  94. package/dist/chunk-EDQ57S4P.js +0 -1
  95. package/dist/chunk-FCJXL4PU.js +0 -182
  96. package/dist/chunk-GPSPSJBJ.js +0 -1
  97. package/dist/chunk-HE45FFBU.js +0 -1
  98. package/dist/chunk-HUV2WTRW.js +0 -1
  99. package/dist/chunk-L552AQQY.js +0 -84
  100. package/dist/chunk-MVDMZ3N2.js +0 -1282
  101. package/dist/chunk-NA76FFGD.js +0 -73
  102. package/dist/chunk-NZSDHDX3.js +0 -1
  103. package/dist/chunk-OS4XFDIG.js +0 -21
  104. package/dist/chunk-QQ7OD7BJ.js +0 -100
  105. package/dist/chunk-RPA5ADXW.js +0 -16
  106. package/dist/chunk-TMB56NCA.js +0 -1
  107. package/dist/chunk-WO3N5FH2.js +0 -5
  108. package/dist/chunk-XPEJB4BZ.js +0 -1
  109. package/dist/chunk-YK4OAZRQ.js +0 -4
  110. package/dist/chunk-YWUNN24S.js +0 -1
  111. package/dist/chunk-ZJTUAWZA.js +0 -19
  112. package/dist/chunk-ZPK5LYEN.js +0 -1565
  113. package/dist/chunk-ZUGTYD2I.js +0 -63
  114. package/dist/db-B6RNXKEI.js +0 -1
  115. package/dist/extract-bundle-X7SRN63Y.js +0 -26
  116. package/dist/gmail-service-FZVJBSUT.js +0 -1
  117. package/dist/lead-list-enrichment-repository-SXYSS7ZX.js +0 -1
  118. package/dist/operation-metering-OBKSWJVJ.js +0 -1
  119. package/dist/server-JV6L7F2X.js +0 -5014
  120. package/dist/site-extract-repository-UVE7TWJO.js +0 -1
  121. package/dist/stripe-event-worker-2LSUVQOM.js +0 -1
  122. package/dist/worker-XPIO5TAV.js +0 -1
  123. /package/dist/{chunk-YGBTTW5D.js → chunk-2IATUJO6.js} +0 -0
  124. /package/dist/{chunk-6DTXIZY2.js → chunk-4ZVO6GX4.js} +0 -0
@@ -1,182 +0,0 @@
1
- import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{ba as w}from"./chunk-EDQ57S4P.js";import{Kb as N,Tb as D,d as m,e as I,i as u,t as s}from"./chunk-ZPK5LYEN.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
2
- VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
3
- (id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
4
- VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs
5
- WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')
6
- AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))
7
- ORDER BY updated_at ASC LIMIT ?`,args:[Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(r=>_(r))}async function ft(t=25,e=2){let n=s(),r=Math.max(1,Math.min(60,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
8
- LEFT JOIN billing_debits d
9
- ON d.idempotency_key = json_extract(j.options, '$.debitKey')
10
- AND d.user_id = j.user_id
11
- AND d.status = 'applied'
12
- WHERE j.status = 'pending'
13
- AND j.billed_mc IS NULL
14
- AND (
15
- d.idempotency_key IS NOT NULL
16
- OR (
17
- json_extract(j.options, '$.billingClass') = ?
18
- AND json_extract(j.options, '$.heldMc') = 0
19
- AND json_extract(j.options, '$.debitKey') IS NULL
20
- AND json_extract(j.options, '$.xraySetupReceipt.billingClass') = ?
21
- AND json_extract(j.options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
22
- )
23
- )
24
- AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
25
- AND j.updated_at <= datetime('now', ?)
26
- ORDER BY j.updated_at ASC LIMIT ?`,args:[g,g,`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function bt(t){return await s().execute({sql:`UPDATE site_extract_jobs
27
- SET dispatch_attempts = dispatch_attempts + 1,
28
- updated_at = datetime('now'),
29
- next_dispatch_at = datetime('now', '+5 minutes'),
30
- dispatch_error = NULL,
31
- status = CASE
32
- WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
33
- ELSE status
34
- END,
35
- error = CASE
36
- WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
37
- THEN 'Background crawl was acknowledged repeatedly but never started; the hold will be refunded.'
38
- ELSE error
39
- END
40
- WHERE id = ? AND status = 'pending'`,args:[t]}),x(t)}async function wt(t,e){let n=s(),r=u(e).slice(0,1e3);return await n.execute({sql:`UPDATE site_extract_jobs SET
41
- dispatch_attempts = dispatch_attempts + 1,
42
- dispatch_error = ?,
43
- next_dispatch_at = datetime('now', '+5 minutes'),
44
- updated_at = datetime('now'),
45
- status = CASE
46
- WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
47
- ELSE status
48
- END,
49
- error = CASE
50
- WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
51
- THEN 'Background crawl could not be dispatched after repeated attempts; the hold will be refunded.'
52
- ELSE error
53
- END
54
- WHERE id = ? AND status = 'pending'`,args:[r,t]}),x(t)}async function St(t,e){await s().execute({sql:`UPDATE site_extract_jobs SET
55
- settlement_attempts = settlement_attempts + 1,
56
- settlement_error = ?,
57
- next_settlement_at = datetime('now', '+15 minutes'),
58
- updated_at = datetime('now')
59
- WHERE id = ? AND billed_mc IS NULL`,args:[u(e).slice(0,1e3),t]})}async function At(t=25,e=60){let n=s(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
60
- LEFT JOIN billing_debits d
61
- ON d.idempotency_key = json_extract(j.options, '$.debitKey')
62
- AND d.user_id = j.user_id
63
- AND d.status = 'applied'
64
- WHERE j.status = 'running'
65
- AND j.billed_mc IS NULL
66
- AND (
67
- json_extract(j.options, '$.debitKey') IS NULL
68
- OR d.idempotency_key IS NOT NULL
69
- )
70
- AND j.updated_at <= datetime('now', ?)
71
- ORDER BY j.updated_at ASC LIMIT ?`,args:[`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function Nt(t,e=60){let n=s(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`UPDATE site_extract_jobs
72
- SET status = 'failed',
73
- error = 'Background crawl stopped without a heartbeat; completed pages were preserved and settlement is pending.',
74
- public_error_json = ?,
75
- updated_at = datetime('now')
76
- WHERE id = ?
77
- AND status = 'running'
78
- AND billed_mc IS NULL
79
- AND updated_at <= datetime('now', ?)`,args:[m({error_code:"extraction_failed",error_type:"extraction",message:"The background crawl stopped without a heartbeat. Completed pages were preserved; retry after settlement completes.",retryable:!0,charge_status:"refund_pending"}),t,`-${r} minutes`]})).rowsAffected===1}async function Tt(t){return(await s().execute({sql:`UPDATE site_extract_jobs
80
- SET status = 'running', updated_at = datetime('now')
81
- WHERE id = ? AND status = 'failed'`,args:[t]})).rowsAffected!==1?null:x(t)}async function yt(t,e){await s().execute({sql:`UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')
82
- WHERE id = ? AND billed_mc IS NULL`,args:[u(e).slice(0,1e3),t]})}async function ht(t,e){await s().execute({sql:`UPDATE site_extract_jobs
83
- SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')
84
- WHERE id = ? AND status IN ('pending', 'running')`,args:[e,t]})}async function Rt(t,e){if(e.length===0)return;let n=s(),r=await x(t),i=r?.userId!=null?await M({ownerId:String(r.userId),jobId:t,createdAt:r.createdAt,pages:e}):new Map;await n.batch([...e.map(a=>{let{discoveryLinks:o,acquiredHtml:c,mainHtml:d,...l}=a;return l.contentRef=i.get(a.archivedUrl??a.url)??a.contentRef,{sql:`INSERT INTO site_extract_pages (job_id, url, page)
85
- SELECT ?, ?, ?
86
- WHERE EXISTS (
87
- SELECT 1 FROM site_extract_jobs
88
- WHERE id = ? AND status IN ('pending', 'running')
89
- )
90
- ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page
91
- WHERE NOT CASE
92
- WHEN json_valid(site_extract_pages.page) = 0 THEN 0
93
- WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL
94
- THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'
95
- ELSE COALESCE(
96
- json_extract(site_extract_pages.page, '$.status') >= 200
97
- AND json_extract(site_extract_pages.page, '$.status') < 300
98
- AND json_extract(site_extract_pages.page, '$.wordCount') > 0,
99
- 0
100
- )
101
- END
102
- OR CASE
103
- WHEN json_valid(excluded.page) = 0 THEN 0
104
- WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL
105
- THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'
106
- ELSE COALESCE(
107
- json_extract(excluded.page, '$.status') >= 200
108
- AND json_extract(excluded.page, '$.status') < 300
109
- AND json_extract(excluded.page, '$.wordCount') > 0,
110
- 0
111
- )
112
- END`,args:[t,a.archivedUrl??a.url,JSON.stringify(l),t]}}),{sql:`UPDATE site_extract_jobs SET
113
- done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
114
- attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
115
- successful_urls = (
116
- SELECT COUNT(*) FROM site_extract_pages
117
- WHERE job_id = ? AND CASE
118
- WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
119
- THEN json_extract(page, '$.extractionStatus') = 'successful'
120
- ELSE json_extract(page, '$.status') >= 200
121
- AND json_extract(page, '$.status') < 300
122
- AND json_extract(page, '$.wordCount') > 0
123
- END
124
- ),
125
- failed_urls = (
126
- SELECT COUNT(*) FROM site_extract_pages
127
- WHERE job_id = ? AND NOT CASE
128
- WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
129
- THEN json_extract(page, '$.extractionStatus') = 'successful'
130
- ELSE COALESCE(
131
- json_extract(page, '$.status') >= 200
132
- AND json_extract(page, '$.status') < 300
133
- AND json_extract(page, '$.wordCount') > 0,
134
- 0
135
- )
136
- END
137
- ),
138
- updated_at = datetime('now')
139
- WHERE id = ? AND status IN ('pending', 'running')`,args:[t,t,t,t,t]}],"write")}async function It(t){return(await s().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function Dt(t){return(await s().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ?",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function jt(t,e=181){let n=s(),r=Math.max(1,Math.min(5001,Math.round(e)));return(await n.execute({sql:`SELECT DISTINCT CAST(images.value AS TEXT) AS url
140
- FROM site_extract_pages p,
141
- json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images
142
- WHERE p.job_id = ?
143
- AND images.type = 'text'
144
- AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096
145
- LIMIT ?`,args:[t,r]})).rows.map(a=>String(a.url))}async function Pt(t){let n=await s().execute({sql:`SELECT COUNT(*) AS n FROM site_extract_pages
146
- WHERE job_id = ? AND CASE
147
- WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
148
- THEN json_extract(page, '$.extractionStatus') = 'successful'
149
- ELSE COALESCE(
150
- json_extract(page, '$.status') >= 200
151
- AND json_extract(page, '$.status') < 300
152
- AND json_extract(page, '$.wordCount') > 0,
153
- 0
154
- )
155
- END`,args:[t]});return Number(n.rows[0]?.n??0)}async function Ct(t){let n=await s().execute({sql:"SELECT url FROM site_extract_pages WHERE job_id = ?",args:[t]});return new Set(n.rows.map(r=>String(r.url)))}async function z(t,e,n,r=null,i=!1,a=null){return(await s().execute({sql:`UPDATE site_extract_jobs
156
- SET status = ?, artifacts = ?, error = ?, public_error_json = ?, updated_at = datetime('now')
157
- WHERE id = ? AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed'))`,args:[n,JSON.stringify(e??[]),r?u(r).slice(0,2e3):null,m(a),t,i?1:0]})).rowsAffected===1}async function Lt(t,e){await z(t,e,"complete")}async function Ut(t,e,n){await s().execute({sql:`UPDATE site_extract_jobs
158
- SET status = 'failed', error = ?, public_error_json = ?, updated_at = datetime('now')
159
- WHERE id = ? AND status IN ('pending', 'running')`,args:[u(e).slice(0,2e3),m(n),t]})}async function Ot(t,e,n){await s().execute({sql:`UPDATE site_extract_jobs
160
- SET status = 'failed', billed_mc = 0, error = ?, public_error_json = ?, updated_at = datetime('now')
161
- WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,args:[u(e).slice(0,2e3),m(n),t]})}async function kt(t,e){let r=(await x(t))?.options.billingClass===g&&e?{...e,charge_status:"not_charged"}:e;await s().execute({sql:`UPDATE site_extract_jobs SET public_error_json = ?, updated_at = datetime('now')
162
- WHERE id = ? AND status IN ('complete', 'partial', 'failed')`,args:[m(r),t]})}async function Mt(t,e,n,r,i){let a=s(),o=await x(t);if(!o||o.billedMc!=null)return;if(o.options.billingClass===g){if(!await X(t))throw new Error("xray_extract_zero_charge_finalization_rejected");return}let c=typeof o.options.debitKey=="string"?o.options.debitKey:null;if(c){let f=await D(e,c,Math.max(0,r),w.EXTRACT_SITE_REFUND,i,"extract_site");await a.execute({sql:"UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL",args:[f.final_amount_mc,t]});return}let d=Math.max(0,n),l=Math.max(0,r);await a.batch([{sql:"UPDATE site_extract_jobs SET billed_mc = -1 WHERE id = ? AND billed_mc IS NULL",args:[t]},{sql:`INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
163
- SELECT ?, ?, ?, ?, datetime(j.created_at, ?)
164
- FROM site_extract_jobs j
165
- WHERE j.id = ?
166
- AND j.billed_mc = -1
167
- AND ? > 0
168
- AND changes() = 1
169
- AND datetime(j.created_at, ?) > datetime('now')`,args:[e,d,d,w.EXTRACT_SITE_REFUND,N,t,d,N]},{sql:`UPDATE users SET balance_mc = balance_mc + ?
170
- WHERE id = ? AND ? > 0 AND changes() = 1`,args:[d,e,d]},{sql:`INSERT INTO ledger (user_id, amount_mc, operation, description)
171
- SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,args:[e,d,w.EXTRACT_SITE_REFUND,i,d]},{sql:`UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now')
172
- WHERE id = ? AND billed_mc = -1`,args:[l,t]}],"write")}async function X(t){return(await s().execute({sql:`UPDATE site_extract_jobs
173
- SET billed_mc = 0, updated_at = datetime('now')
174
- WHERE id = ?
175
- AND billed_mc IS NULL
176
- AND json_extract(options, '$.billingClass') = ?
177
- AND json_extract(options, '$.heldMc') = 0
178
- AND json_extract(options, '$.debitKey') IS NULL
179
- AND json_extract(options, '$.xraySetupReceipt.billingClass') = ?
180
- AND json_extract(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
181
- AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
182
- AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0`,args:[t,g,g]})).rowsAffected===1}export{y as a,V as b,G as c,Q as d,Z as e,tt as f,et as g,it as h,g as i,pt as j,Et as k,mt as l,K as m,gt as n,x as o,_t as p,xt as q,ft as r,bt as s,wt as t,St as u,At as v,Nt as w,Tt as x,yt as y,ht as z,Rt as A,It as B,Dt as C,jt as D,Pt as E,Ct as F,z as G,Lt as H,Ut as I,Ot as J,kt as K,Mt as L,X as M};
@@ -1 +0,0 @@
1
- function o(e,_=!1,n="light",r=1,R=!1,t=!1){let E=_&&(R||t)?24e4:_&&n==="full"?r>=2?75e3:45e3:_?4e4:4e5,S=Math.min(E+5e3,795e3);return{serverMs:E,clientMs:S}}function M(){return(process.env.BROWSER_SERVICE_API_KEY??process.env.KERNEL_API_KEY)?.trim()||void 0}function u(){return(process.env.BROWSER_AGENT_PROFILE_NAME??process.env.BROWSER_SERVICE_PROFILE_NAME??process.env.KERNEL_BROWSER_PROFILE_NAME??process.env.KERNEL_PROFILE_NAME)?.trim()||void 0}function i(){let e=(process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES??process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES??process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES??process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();if(e){if(["1","true","yes","on"].includes(e))return!0;if(["0","false","no","off"].includes(e))return!1}}export{o as a,M as b,u as c,i as d};
@@ -1 +0,0 @@
1
- import{isIP as I}from"net";import{lookup as T}from"dns/promises";import p from"ipaddr.js";function P(n){return n.startsWith("[")&&n.endsWith("]")?n.slice(1,-1):n}function b(n){let t=P(n);if(!I(t))return!1;try{let e=p.parse(t),s=o=>[["192.0.0.0",24],["192.0.2.0",24],["198.18.0.0",15],["198.51.100.0",24],["203.0.113.0",24]].some(([m,u])=>o.match(p.parse(m),u));if(e instanceof p.IPv6&&e.isIPv4MappedAddress()){let o=e.toIPv4Address();return o.range()!=="unicast"||s(o)}return e instanceof p.IPv4&&s(e)?!0:e.range()!=="unicast"}catch{return!0}}async function N(n){let t=P(n.toLowerCase());if(t==="localhost"||t.endsWith(".localhost")||t.endsWith(".local")||b(t))return!0;try{let e=await T(t,{all:!0,verbatim:!0});return e.length===0||e.some(s=>b(s.address))}catch{return!0}}async function S(n,t){let e;try{e=new URL(n.trim())}catch{return{error:`Invalid ${t.field}`}}return(t.requireHttps?["https:"]:["http:","https:"]).includes(e.protocol)?await N(e.hostname)?{error:`${t.field} must resolve to a public internet host`}:{parsed:e}:{error:t.requireHttps?`${t.field} must use https`:`${t.field} must use http or https`}}import{load as C}from"cheerio";var _="script,style,noscript,svg,iframe,canvas,nav,header,footer,aside",k=4096;function d(n){return(n??"").replace(/\s+/g," ").trim()}function L(n){let t=n.replace(/[\x00-\x09\x0b\x0c\x0e-\x1f]/g," ").trim();if(!t)return null;try{return JSON.parse(t)}catch(e){let s=/at position (\d+)/.exec(e instanceof Error?e.message:"")?.[1];if(!s)return null;try{return JSON.parse(t.slice(0,Number(s)))}catch{return null}}}function g(n,t,e={value:2e3}){if(e.value--<=0||n==null)return;if(Array.isArray(n)){for(let l of n)g(l,t,e);return}if(typeof n!="object")return;let s=n,o=s["@type"];if(typeof o=="string"&&o&&t.add(o),Array.isArray(o))for(let l of o)typeof l=="string"&&l&&t.add(l);Array.isArray(s["@graph"])&&g(s["@graph"],t,e)}function H(n){let t=n("main").first();if(t.length)return t;let e=n("article").first();if(e.length)return e;let s=n("body").first();return s.length?s:n.root()}function w(n,t){let e=Number(n.attr(t)??0);return Number.isFinite(e)&&e>0?e:null}function W(n){let t=C(n),e={};t("meta").each((f,i)=>{let r=t(i),a=d(r.attr("name")??r.attr("property")).toLowerCase(),c=d(r.attr("content"));a&&c&&(e[a]=c)});let s=[];t("h1,h2,h3,h4,h5,h6").each((f,i)=>{let r=t(i),a=d(r.text());a&&s.push({level:Number(i.tagName.slice(1)),text:a})});let o=[],l=new Set;t('script[type="application/ld+json"]').each((f,i)=>{let r=L(t(i).text());r!=null&&(Array.isArray(r)?o.push(...r):o.push(r),g(r,l))});let m=new Set;t("[itemtype]").each((f,i)=>{let r=t(i).attr("itemtype")??"";for(let a of r.split(/\s+/))try{let c=new URL(a);if(c.hostname==="schema.org"||c.hostname==="www.schema.org"){let h=c.pathname.replace(/^\/+/,"");h&&m.add(h)}}catch{}});let u=H(t).clone();u.find(_).remove(),u.find("[href],[src],[srcset],[data-src],[data-srcset]").each((f,i)=>{let r=t(i);for(let a of["href","src","srcset","data-src","data-srcset"]){let c=r.attr(a);c&&c.length>k&&r.removeAttr(a)}}),u.find("img").each((f,i)=>{let r=t(i),a=r.attr("src");if(!a||a.startsWith("data:")){let c=r.attr("data-src")??r.attr("data-lazy-src")??r.attr("data-original");c&&r.attr("src",c)}});let A=u.html()??"",v=d(u.text()),y=[];t("a[href]").each((f,i)=>{let r=t(i),a=d(r.attr("href"));a&&y.push({href:a,text:d(r.text()),rel:d(r.attr("rel")).toLowerCase()||null})});let x=[];return t("img").each((f,i)=>{let r=t(i),a=Object.fromEntries(Object.entries(i.attribs??{}).map(([c,h])=>[c.toLowerCase(),h]));x.push({src:r.attr("src")??r.attr("data-src")??r.attr("data-lazy-src")??r.attr("data-original")??null,srcset:r.attr("srcset")??r.attr("data-srcset")??null,alt:r.attr("alt")??null,width:w(r,"width"),height:w(r,"height"),attributes:a})}),{$:t,sourceHtml:n,title:d(t("title").first().text())||null,meta:e,headings:s,schema:o,schemaTypes:[...l],microdataTypes:[...m],canonicalUrl:d(t('link[rel~="canonical"]').first().attr("href"))||null,mainHtml:A,mainText:v,bodyClass:t("body").first().attr("class")??"",links:y,images:x}}export{b as a,S as b,W as c};
@@ -1 +0,0 @@
1
- import{createHash as B,randomUUID as N}from"crypto";import{mkdir as O,readFile as T,stat as v,writeFile as U}from"fs/promises";import{homedir as k}from"os";import{dirname as S,join as A}from"path";var d="connected-data-exports/",b=10080*60*1e3,R=900*1e3;function p(){return process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function D(){return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim()||A(k(),"Downloads","mcp-scraper")}function g(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function x(t){return(t.toLowerCase().replace(/[^a-z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||"connected-data-export").slice(0,120)}function C(t){if(!t.startsWith(d))return null;let e=(t.split("/").at(-1)??"").match(/^(\d{13})-/);if(!e)return null;let r=Number(e[1]);return Number.isFinite(r)?r:null}function _(t){return t.startsWith(d)&&t.slice(d.length).split("/")[0]||null}function P(t){let n=C(t);return n===null?null:new Date(n+b)}function I(t,n=Date.now()){let e=C(t);return e===null||e+b<=n}async function h(t,n){let e=p();if(!e)return null;let r=Math.min(Date.now()+R,n.getTime());if(r<=Date.now())return null;let{issueSignedToken:a,presignUrl:i}=await import("@vercel/blob"),s=await a({token:e,pathname:t,operations:["get"],validUntil:r}),{presignedUrl:o}=await i(s,{access:"private",operation:"get",pathname:t,validUntil:r});return{url:o,expiresAt:new Date(r).toISOString()}}async function H(t){return L({...t,contentType:"application/x-ndjson"})}async function L(t){let n=t.stableCreatedAt?Date.parse(t.stableCreatedAt):Number.NaN,e=Number.isFinite(n)?n:Date.now(),r=x(t.filename),a=r.includes(".")?`.${r.split(".").at(-1)}`:".bin",i=t.stableKey?x(t.stableKey):N(),s=`${d}${t.ownerId}/${e}-${x(t.exportId)}-${i}--${r}`,o=(typeof t.content=="string",Buffer.from(t.content)),u=o.byteLength,c=B("sha256").update(o).digest("hex"),l=new Date(e+b),f=p(),m=s;if(f){let{put:w}=await import("@vercel/blob");m=(await w(s,o,{access:"private",token:f,contentType:t.contentType,addRandomSuffix:!t.stableKey,...t.stableKey?{allowOverwrite:!0}:{},cacheControlMaxAge:60,multipart:u>100*1024*1024})).pathname}else{if(g())throw new Error("connected_data_private_blob_not_configured");let w=A(D(),"blobs",s);await O(S(w),{recursive:!0}),await U(w,o)}let y=await h(m,l);return{artifactId:m,filename:r,contentType:t.contentType,bytes:u,sha256:c,createdAt:new Date(e).toISOString(),expiresAt:l.toISOString(),downloadUrl:y?.url??null,downloadUrlExpiresAt:y?.expiresAt??null,readback:{tool:"report_artifact_read",arguments:{artifactId:m,offset:0,maxBytes:2e4},continuation:"Repeat with offset set to the previous result nextOffset until nextOffset is null."}}}async function V(t){if(_(t.artifactId)!==t.ownerId)return null;let n=P(t.artifactId);if(!n||n.getTime()<=Date.now())return null;let e=await h(t.artifactId,n);return e?{downloadUrl:e.url,downloadUrlExpiresAt:e.expiresAt,expiresAt:n.toISOString()}:null}async function E(t,n=Number.POSITIVE_INFINITY){let e=t.getReader(),r=[],a=0;try{for(;;){let{done:i,value:s}=await e.read();if(i)break;if(s){let o=Buffer.from(s);if(a+=o.byteLength,a>n)return await e.cancel("artifact_too_large").catch(()=>{}),null;r.push(o)}}}finally{e.releaseLock()}return Buffer.concat(r)}async function j(t,n,e){if(I(t))return null;let r=p(),a;if(r){let{get:u}=await import("@vercel/blob"),c=await u(t,{access:"private",token:r,useCache:!1});if(!c||c.statusCode!==200)return null;let l=await E(c.stream);if(!l)return null;a=l}else{if(g())return null;try{a=await T(A(D(),"blobs",t))}catch{return null}}let i=a.length,s=Math.max(0,n),o=Math.min(i,s+e);return{text:a.subarray(s,o).toString("utf8"),totalBytes:i,nextOffset:o<i?o:null}}async function q(t){if(_(t.artifactId)!==t.ownerId||I(t.artifactId))return null;let n=p(),e;if(n){let{get:r}=await import("@vercel/blob"),a=await r(t.artifactId,{access:"private",token:n,useCache:!1});if(!a||a.statusCode!==200)return null;let i=await E(a.stream,t.maxBytes??25*1024*1024);if(!i)return null;e=i}else{if(g())return null;try{let r=A(D(),"blobs",t.artifactId);if((await v(r)).size>(t.maxBytes??25*1024*1024))return null;e=await T(r)}catch{return null}}return e.byteLength>(t.maxBytes??25*1024*1024)?null:e}async function z(t={}){let n=t.now??new Date,e=p();if(!e)return{deleted:0,store:g()?"none":"local"};if(!t.force&&!(n.getUTCHours()===3&&n.getUTCMinutes()===17))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=n.getTime()-b,{list:a,del:i}=await import("@vercel/blob"),s,o=0;for(let u=0;u<20;u++){let c=await a({prefix:d,token:e,limit:1e3,cursor:s}),l=c.blobs.filter(f=>new Date(f.uploadedAt).getTime()<=r);if(l.length>0&&(await i(l.map(f=>f.pathname),{token:e}),o+=l.length),!c.hasMore||!c.cursor)break;s=c.cursor}return{deleted:o,store:"private-vercel-blob"}}export{d as a,_ as b,H as c,L as d,V as e,j as f,q as g,z as h};
@@ -1,84 +0,0 @@
1
- import{R as p,t as o}from"./chunk-ZPK5LYEN.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
2
- CREATE TABLE IF NOT EXISTS service_connections (
3
- id TEXT PRIMARY KEY,
4
- user_id INTEGER NOT NULL REFERENCES users(id),
5
- provider_config_key TEXT NOT NULL,
6
- provider TEXT,
7
- transport TEXT NOT NULL,
8
- upstream_connection_id TEXT,
9
- label TEXT,
10
- user_label TEXT,
11
- provider_account_id TEXT,
12
- provider_account_email TEXT,
13
- provider_account_name TEXT,
14
- provider_identity_checked_at TEXT,
15
- provider_identity_source_updated_at TEXT,
16
- lifecycle_status TEXT NOT NULL DEFAULT 'pending',
17
- operational_status TEXT NOT NULL DEFAULT 'unknown',
18
- reconnect_required INTEGER NOT NULL DEFAULT 0,
19
- actions_enabled INTEGER NOT NULL DEFAULT 0,
20
- read_tools_json TEXT NOT NULL DEFAULT '[]',
21
- action_tools_json TEXT NOT NULL DEFAULT '[]',
22
- tool_revision TEXT,
23
- last_checked_at TEXT,
24
- last_successful_call_at TEXT,
25
- last_failure_at TEXT,
26
- last_failure_code TEXT,
27
- last_failure_retryable INTEGER,
28
- consecutive_failures INTEGER NOT NULL DEFAULT 0,
29
- vault_name TEXT,
30
- table_name TEXT,
31
- legacy_owner_identity TEXT,
32
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
33
- updated_at TEXT NOT NULL DEFAULT (datetime('now')),
34
- UNIQUE(user_id, provider_config_key, upstream_connection_id)
35
- )
36
- `);try{await t.execute("ALTER TABLE service_connections ADD COLUMN user_label TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT")}catch{}await t.execute("CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)"),await t.execute("CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)"),await t.execute(`
37
- CREATE TABLE IF NOT EXISTS service_connection_health_events (
38
- id TEXT PRIMARY KEY,
39
- connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
40
- operational_status TEXT NOT NULL,
41
- failure_code TEXT,
42
- retryable INTEGER,
43
- evidence_source TEXT NOT NULL,
44
- created_at TEXT NOT NULL DEFAULT (datetime('now'))
45
- )
46
- `),await t.execute("CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)"),await t.execute(`
47
- CREATE TABLE IF NOT EXISTS service_connection_action_audit (
48
- id TEXT PRIMARY KEY,
49
- connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
50
- user_id INTEGER NOT NULL REFERENCES users(id),
51
- tool TEXT NOT NULL,
52
- request_id TEXT NOT NULL,
53
- status TEXT NOT NULL,
54
- error_code TEXT,
55
- created_at TEXT NOT NULL DEFAULT (datetime('now')),
56
- UNIQUE(user_id, request_id)
57
- )
58
- `);try{await t.execute("ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT")}catch{}try{await t.execute("ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT")}catch{}})().catch(t=>{throw a=null,u=null,t})),a}function f(e){if(typeof e!="string")return[];try{let t=JSON.parse(e);return Array.isArray(t)?t.filter(n=>typeof n=="string"):[]}catch{return[]}}function g(e){return{id:String(e.id),userId:Number(e.user_id),providerConfigKey:String(e.provider_config_key),provider:typeof e.provider=="string"?e.provider:null,transport:e.transport==="remote_mcp"?"remote_mcp":e.transport==="credential_reference"?"credential_reference":"nango",upstreamConnectionId:typeof e.upstream_connection_id=="string"?e.upstream_connection_id:null,label:typeof e.label=="string"?e.label:null,userLabel:typeof e.user_label=="string"?e.user_label:null,providerAccountId:typeof e.provider_account_id=="string"?e.provider_account_id:null,providerAccountEmail:typeof e.provider_account_email=="string"?e.provider_account_email:null,providerAccountName:typeof e.provider_account_name=="string"?e.provider_account_name:null,providerIdentityCheckedAt:typeof e.provider_identity_checked_at=="string"?e.provider_identity_checked_at:null,providerIdentitySourceUpdatedAt:typeof e.provider_identity_source_updated_at=="string"?e.provider_identity_source_updated_at:null,lifecycleStatus:String(e.lifecycle_status),operationalStatus:String(e.operational_status),reconnectRequired:Number(e.reconnect_required)===1,actionsEnabled:Number(e.actions_enabled)===1,readTools:f(e.read_tools_json),actionTools:f(e.action_tools_json),toolRevision:typeof e.tool_revision=="string"?e.tool_revision:null,lastCheckedAt:typeof e.last_checked_at=="string"?e.last_checked_at:null,lastSuccessfulCallAt:typeof e.last_successful_call_at=="string"?e.last_successful_call_at:null,lastFailureAt:typeof e.last_failure_at=="string"?e.last_failure_at:null,lastFailureCode:typeof e.last_failure_code=="string"?e.last_failure_code:null,lastFailureRetryable:e.last_failure_retryable===null||e.last_failure_retryable===void 0?null:Number(e.last_failure_retryable)===1,consecutiveFailures:Number(e.consecutive_failures??0),vaultName:typeof e.vault_name=="string"?e.vault_name:null,tableName:typeof e.table_name=="string"?e.table_name:null,createdAt:String(e.created_at),updatedAt:String(e.updated_at)}}async function E(e){let t=await p(e.normalize("NFKC").trim().toLowerCase());if(!t)throw new Error("service_connection_user_not_found");return Number(t.id)}async function D(e,t){await c();let n=o(),r=await E(e);for(let i of t)await n.execute({sql:`
59
- INSERT INTO service_connections (
60
- id, user_id, provider_config_key, provider, transport, upstream_connection_id,
61
- label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
62
- ) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
63
- ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
64
- provider = excluded.provider,
65
- label = COALESCE(excluded.label, service_connections.label),
66
- lifecycle_status = excluded.lifecycle_status,
67
- reconnect_required = excluded.reconnect_required,
68
- updated_at = excluded.updated_at
69
- `,args:[_(),r,i.providerConfigKey,i.provider,i.upstreamConnectionId,i.label,i.lifecycleStatus,i.reconnectRequired?1:0,e,i.createdAt,i.updatedAt]});return A(e,"nango")}async function A(e,t){await c();let n=await E(e);return(await o().execute({sql:`SELECT * FROM service_connections WHERE user_id = ?${t?" AND transport = ?":""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,args:t?[n,t]:[n]})).rows.map(i=>g(i))}async function d(e,t){await c();let n=await E(e),i=(await o().execute({sql:"SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",args:[t,n]})).rows[0];return i?g(i):null}async function w(e,t,n,r){await c(),await o().execute({sql:"UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?",args:[JSON.stringify(t),JSON.stringify(n),r,e]})}async function U(e){await c(),await o().execute({sql:`
70
- UPDATE service_connections SET
71
- provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
72
- provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
73
- updated_at = datetime('now')
74
- WHERE id = ?
75
- `,args:[e.providerAccountId??null,e.providerAccountEmail?.toLowerCase()??null,e.providerAccountName??null,e.sourceUpdatedAt,e.connectionId]})}async function h(e,t,n){let r=await d(e,t);if(!r||r.lifecycleStatus==="disconnected")throw new Error("service_connection_not_found");return await o().execute({sql:"UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[n,t,r.userId]}),n}async function X(e){await c();let t=new Date().toISOString();await o().execute({sql:`
76
- UPDATE service_connections SET
77
- operational_status = ?, last_checked_at = ?,
78
- last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
79
- last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
80
- last_failure_code = ?, last_failure_retryable = ?,
81
- consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
82
- updated_at = datetime('now')
83
- WHERE id = ?
84
- `,args:[e.operationalStatus,t,e.operationalStatus,t,e.operationalStatus,t,e.failureCode??null,e.retryable==null?null:e.retryable?1:0,e.operationalStatus,e.connectionId]}),await o().execute({sql:"INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)",args:[_(),e.connectionId,e.operationalStatus,e.failureCode??null,e.retryable==null?null:e.retryable?1:0,e.evidenceSource]})}async function q(e,t,n){let r=await d(e,t);if(!r||r.lifecycleStatus==="disconnected")throw new Error("service_connection_not_found");return await o().execute({sql:"UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[n?1:0,t,r.userId]}),n}async function k(e,t){let n=await d(e,t);if(!n)throw new Error("service_connection_not_found");await o().execute({sql:"UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[t,n.userId]});let r=await o().execute({sql:"SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'",args:[n.userId]});return Number(r.rows[0]?.count??0)}async function F(e){await c();let t=await d(e.identity,e.connectionId);if(!t)throw new Error("service_connection_not_found");await o().execute({sql:"UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?",args:[e.status,e.errorCode??null,e.result===void 0?null:JSON.stringify(e.result),t.userId,e.requestId,t.id]})}async function P(e){await c();let t=await d(e.identity,e.connectionId);if(!t)throw new Error("service_connection_not_found");let n=await o().execute({sql:"INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)",args:[_(),t.id,t.userId,e.tool,e.requestId,e.requestDigest]});if(Number(n.rowsAffected??0)===1)return{claimed:!0};let i=(await o().execute({sql:"SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1",args:[t.userId,e.requestId]})).rows[0],s=String(i?.connection_id??"")!==t.id||String(i?.tool??"")!==e.tool||String(i?.request_digest??"")!==e.requestDigest,l;if(!s&&typeof i?.result_json=="string")try{l=JSON.parse(i.result_json)}catch{l=void 0}return{claimed:!1,status:String(i?.status??"unknown"),errorCode:typeof i?.error_code=="string"?i.error_code:null,conflict:s,...l!==void 0?{result:l}:{}}}export{S as a,L as b,I as c,x as d,c as e,D as f,A as g,d as h,w as i,U as j,h as k,X as l,q as m,k as n,F as o,P as p};