mcp-scraper 0.95.1 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -39
- package/README.md +25 -7
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/actions-YCMJY3PF.js +1 -0
- package/dist/artifacts-4ZDNP2JW.js +1 -0
- package/dist/artifacts-N62OPIT5.js +1 -0
- package/dist/artifacts-PSADD5OC.js +1 -0
- package/dist/artifacts-RUR3OHAK.js +1 -0
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +22 -22
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/blob-store-GIH26EDO.js +1 -0
- package/dist/bundle-TTJP6DHU.js +1 -0
- package/dist/chunk-24YDQ3EO.js +5 -0
- package/dist/chunk-2PRH4Z3N.js +181 -0
- package/dist/chunk-3F7AFM53.js +1 -0
- package/dist/chunk-4HZXATHB.js +27 -0
- package/dist/chunk-55ZANTHL.js +3 -0
- package/dist/chunk-5BKSKLHX.js +1 -0
- package/dist/chunk-63VWBMYR.js +3 -0
- package/dist/chunk-6N5EC7ZO.js +1 -0
- package/dist/chunk-6SNOUSSZ.js +182 -0
- package/dist/chunk-7POHNTVW.js +1 -0
- package/dist/chunk-7SR7MEU5.js +62 -0
- package/dist/chunk-7UHDOIUE.js +1 -0
- package/dist/chunk-ARSQZCTJ.js +945 -0
- package/dist/chunk-B3JRL45T.js +1 -0
- package/dist/chunk-B4ZKPLSP.js +1 -0
- package/dist/chunk-C7MGJUTU.js +1 -0
- package/dist/{chunk-W2BVJ7S2.js → chunk-CNSVZTTT.js} +4 -4
- package/dist/chunk-DGAW7FD7.js +12 -0
- package/dist/chunk-FRXKMOXI.js +41 -0
- package/dist/chunk-GVYGAZV7.js +1 -0
- package/dist/chunk-HCK6PLM3.js +3 -0
- package/dist/{chunk-I3HNHCIJ.js → chunk-HJIOH7HF.js} +199 -84
- package/dist/chunk-HNNYXMSI.js +1 -0
- package/dist/chunk-HUNSEGUN.js +57 -0
- package/dist/chunk-IFPION5D.js +121 -0
- package/dist/{chunk-DYDAQQMC.js → chunk-JAV5V3JT.js} +41 -8
- package/dist/chunk-JMZIDHYQ.js +2 -0
- package/dist/{chunk-FN7U6KRI.js → chunk-JW4GDY7S.js} +1 -1
- package/dist/chunk-KXJCGV3H.js +1 -0
- package/dist/chunk-LFRAA3MY.js +4 -0
- package/dist/{chunk-Y6STAKF3.js → chunk-M5QHXNFZ.js} +3 -3
- package/dist/chunk-MVMWPKRC.js +2 -0
- package/dist/chunk-NALXKTUU.js +1 -0
- package/dist/chunk-NNOX3FYG.js +198 -0
- package/dist/chunk-OB6E6UTD.js +1 -0
- package/dist/chunk-OILOHQNL.js +2 -0
- package/dist/chunk-OLYPLX5C.js +1 -0
- package/dist/chunk-QER2ELZP.js +1 -0
- package/dist/chunk-QWE7VKIX.js +1 -0
- package/dist/chunk-R5LJL6IR.js +225 -0
- package/dist/chunk-RDHGQ2WZ.js +1 -0
- package/dist/chunk-SDQW7IJN.js +1 -0
- package/dist/chunk-SEZETSB3.js +1 -0
- package/dist/chunk-SNJR5N36.js +1 -0
- package/dist/chunk-TPN25AFZ.js +159 -0
- package/dist/{chunk-AK6HVF5F.js → chunk-U4WIJZCX.js} +1 -1
- package/dist/chunk-UIOSADPB.js +1718 -0
- package/dist/{chunk-DYKSPHHR.js → chunk-UNEJYZ4T.js} +1 -1
- package/dist/chunk-VAC3SQ5T.js +16 -0
- package/dist/chunk-VVSHZV5S.js +73 -0
- package/dist/chunk-WNUXMPVX.js +1 -0
- package/dist/chunk-WP3QS4CJ.js +1 -0
- package/dist/chunk-ZPQNY5TI.js +1 -0
- package/dist/contracts-GUXGFCVV.js +1 -0
- package/dist/core-schema-423BSTE6.js +936 -0
- package/dist/dispatch-3HY4735M.js +1 -0
- package/dist/index.cjs +104 -214
- package/dist/index.d.cts +192 -180
- package/dist/index.d.ts +192 -180
- package/dist/index.js +1 -1
- package/dist/media-artifacts-67CJWI7O.js +1 -0
- package/dist/metering-ZH2P62IC.js +1 -0
- package/dist/migrations-3AKEC5QE.js +1 -0
- package/dist/queue-HUP2GNXF.js +1 -0
- package/dist/reconciliation-ANXNJWME.js +1 -0
- package/dist/report-artifact-offload-6OOY2PTM.js +1 -0
- package/dist/repository-4EC5EXVN.js +1 -0
- package/dist/{location-data-repository-NWWKDYG6.js → repository-CYXVMYAI.js} +1 -1
- package/dist/server-VQU6S24A.js +5825 -0
- package/dist/settlement-6CCE2PDH.js +1 -0
- package/dist/worker-5G5BK23B.js +1 -0
- package/package.json +18 -138
- package/dist/analytics-repository-A7VMA24D.js +0 -1
- package/dist/chunk-56C243PX.js +0 -2595
- package/dist/chunk-6ND5AFV5.js +0 -27
- package/dist/chunk-6NE4OIXR.js +0 -6
- package/dist/chunk-DOJMTFLB.js +0 -1
- package/dist/chunk-EDQ57S4P.js +0 -1
- package/dist/chunk-FCJXL4PU.js +0 -182
- package/dist/chunk-GPSPSJBJ.js +0 -1
- package/dist/chunk-HE45FFBU.js +0 -1
- package/dist/chunk-HUV2WTRW.js +0 -1
- package/dist/chunk-L552AQQY.js +0 -84
- package/dist/chunk-MVDMZ3N2.js +0 -1282
- package/dist/chunk-NA76FFGD.js +0 -73
- package/dist/chunk-NZSDHDX3.js +0 -1
- package/dist/chunk-OS4XFDIG.js +0 -21
- package/dist/chunk-QQ7OD7BJ.js +0 -100
- package/dist/chunk-RPA5ADXW.js +0 -16
- package/dist/chunk-TMB56NCA.js +0 -1
- package/dist/chunk-WO3N5FH2.js +0 -5
- package/dist/chunk-XPEJB4BZ.js +0 -1
- package/dist/chunk-YK4OAZRQ.js +0 -4
- package/dist/chunk-YWUNN24S.js +0 -1
- package/dist/chunk-ZJTUAWZA.js +0 -19
- package/dist/chunk-ZPK5LYEN.js +0 -1565
- package/dist/chunk-ZUGTYD2I.js +0 -63
- package/dist/db-B6RNXKEI.js +0 -1
- package/dist/extract-bundle-X7SRN63Y.js +0 -26
- package/dist/gmail-service-FZVJBSUT.js +0 -1
- package/dist/lead-list-enrichment-repository-SXYSS7ZX.js +0 -1
- package/dist/operation-metering-OBKSWJVJ.js +0 -1
- package/dist/server-JV6L7F2X.js +0 -5014
- package/dist/site-extract-repository-UVE7TWJO.js +0 -1
- package/dist/stripe-event-worker-2LSUVQOM.js +0 -1
- package/dist/worker-XPIO5TAV.js +0 -1
- /package/dist/{chunk-YGBTTW5D.js → chunk-2IATUJO6.js} +0 -0
- /package/dist/{chunk-6DTXIZY2.js → chunk-4ZVO6GX4.js} +0 -0
package/dist/chunk-FCJXL4PU.js
DELETED
|
@@ -1,182 +0,0 @@
|
|
|
1
|
-
import{a as j,b as T,c as P,d as C,f as S}from"./chunk-TMB56NCA.js";import{ba as w}from"./chunk-EDQ57S4P.js";import{Kb as N,Tb as D,d as m,e as I,i as u,t as s}from"./chunk-ZPK5LYEN.js";import{createHash as W}from"crypto";import{gunzipSync as $,gzipSync as q}from"zlib";var y="site-extracts/",L=10080*60*1e3,H=900*1e3;function U(){return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function v(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function p(){return{prefix:y,artifactTtlMs:L,downloadTtlMs:H,token:U()}}function h(t){return j(t,y)}async function V(t){let e=await P({policy:p(),ownerId:t.ownerId,artifactKey:`${t.jobId}.zip`,createdAt:t.createdAt,filename:`${t.jobId}-site-export.zip`,contentType:"application/zip",content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"bundle"}}async function G(t){let e=await T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"images"],artifactKey:`${t.imageId}.bin`,createdAt:new Date,filename:t.filename,contentType:t.contentType,content:t.content});return{key:e.artifactId,url:e.downloadUrl??"",bytes:e.bytes,contentType:e.contentType,filename:e.filename,sha256:e.sha256,expiresAt:e.expiresAt,downloadUrlExpiresAt:e.downloadUrlExpiresAt,kind:"image",imageId:t.imageId,sourceUrl:t.sourceUrl,sourcePage:t.sourcePage}}async function O(t){return T({policy:p(),ownerId:t.ownerId,scopeSegments:[t.jobId,"content"],artifactKey:`${t.chunkKey}.json.gz`,createdAt:t.createdAt,filename:`${t.chunkKey}.json.gz`,contentType:"application/gzip",content:t.content})}async function Q(t){return C({policy:p(),artifactId:t.artifactId,ownerId:t.ownerId})}async function J(t){return S({policy:p(),artifactId:t,maxBytes:50*1024*1024})}async function Z(t){return h(t.artifactId)!==t.ownerId?null:J(t.artifactId)}async function k(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:25*1024*1024})}async function tt(t){return h(t.artifactId)!==t.ownerId?null:S({policy:p(),artifactId:t.artifactId,maxBytes:10*1024*1024})}async function et(t={}){let e=t.now??new Date,n=U();if(!n)return{deleted:0,store:v()?"none":"local"};if(!t.force&&!(e.getUTCHours()===3&&e.getUTCMinutes()===21))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=e.getTime()-L,{list:i,del:a}=await import("@vercel/blob"),o,c=0;for(let d=0;d<20;d+=1){let l=await i({prefix:y,token:n,limit:1e3,cursor:o}),f=l.blobs.filter(A=>new Date(A.uploadedAt).getTime()<=r);if(f.length>0&&(await a(f.map(A=>A.pathname),{token:n}),c+=f.length),!l.hasMore||!l.cursor)break;o=l.cursor}return{deleted:c,store:"private-vercel-blob"}}var F=10,b=20*1024*1024;function E(t){return W("sha256").update(t).digest("hex")}function B(t){return!t.acquiredHtml||t.extractionStatus==="failed"?null:{pageId:t.pageId??E(t.archivedUrl??t.url),url:t.archivedUrl??t.url,html:t.acquiredHtml,mainHtml:t.mainHtml??"",markdown:t.bodyMarkdown,htmlSha256:t.htmlSha256??E(t.acquiredHtml),markdownSha256:t.markdownSha256??E(t.bodyMarkdown)}}function R(t){return Buffer.from(JSON.stringify({version:"site-extract-content.v1",pages:t}))}async function M(t){let e=t.pages.flatMap(o=>{let c=B(o);return c?[c]:[]}),n=new Map,r=[],i=0,a=async()=>{if(!r.length)return;let o=R(r);if(o.length>b)throw new Error(`site export content chunk exceeds ${b} bytes`);let c=E(r.map(l=>l.pageId).join("\0")).slice(0,16),d=await O({ownerId:t.ownerId,jobId:t.jobId,createdAt:t.createdAt,chunkKey:`content-${String(i).padStart(4,"0")}-${c}`,content:q(o,{level:6})});for(let l of r)n.set(l.url,{artifactId:d.artifactId,artifactSha256:d.sha256,pageId:l.pageId,uncompressedBytes:o.length});i++,r=[]};for(let o of e){let c=[...r,o];if(r.length>0&&(c.length>F||R(c).length>b)&&await a(),r.push(o),R(r).length>b)throw new Error(`page ${o.pageId} exceeds the durable content chunk limit`)}return await a(),n}function it(t){let e=new Map;return async n=>{let r=e.get(n.artifactId);if(!r){let a=await k({artifactId:n.artifactId,ownerId:t});if(!a)throw new Error("site export content chunk is missing or unauthorized");if(E(a)!==n.artifactSha256)throw new Error("site export content chunk checksum mismatch");let o=$(a);if(o.length>b)throw new Error("site export content chunk exceeds its read limit");if(r=JSON.parse(o.toString("utf8")),r.version!=="site-extract-content.v1"||!Array.isArray(r.pages))throw new Error("site export content chunk has an unsupported contract");for(e.set(n.artifactId,r);e.size>2;)e.delete(e.keys().next().value)}let i=r.pages.find(a=>a.pageId===n.pageId);if(!i)throw new Error("site export page is missing from its content chunk");if(E(i.html)!==i.htmlSha256||E(i.markdown)!==i.markdownSha256)throw new Error("site export page checksum mismatch");return i}}var g="xray_entitlement";function pt(t){let e=Math.max(1,Number(t.options.effectiveMaxPages??t.options.maxPages??1)),n=Math.max(e,Number(t.options.requestedMaxPages??e)),r=n>e;return{requestedMaxPages:n,effectiveMaxPages:e,creditLimited:r,creditTruncated:r&&t.totalUrls>=e}}function Et(t,e=!1){return t.successfulUrls===0?"failed":t.failedUrls>0||t.remainingUrls>0||e?"partial":"complete"}function _(t){let e=Number(t.total_urls??0),n=t.attempted_urls==null,r=Number(n?t.done_urls??0:t.attempted_urls),i=Number(n?t.done_urls??0:t.successful_urls??0),a=Number(t.failed_urls??Math.max(0,r-i));return{id:String(t.id),userId:t.user_id!=null?Number(t.user_id):null,idempotencyKey:t.idempotency_key!=null?String(t.idempotency_key):null,requestFingerprint:t.request_fingerprint!=null?String(t.request_fingerprint):null,status:t.status!=null?String(t.status):"pending",startUrl:String(t.start_url??""),options:t.options?JSON.parse(String(t.options)):{},totalUrls:e,doneUrls:r,attemptedUrls:r,successfulUrls:i,failedUrls:a,remainingUrls:Math.max(0,e-r),artifacts:t.artifacts?JSON.parse(String(t.artifacts)):null,error:t.error!=null?String(t.error):null,publicError:I(t.public_error_json),billedMc:t.billed_mc!=null?Number(t.billed_mc):null,createdAt:String(t.created_at??""),updatedAt:String(t.updated_at??"")}}async function mt(t,e,n,r){await s().execute({sql:`INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
|
|
2
|
-
VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,args:[t,e,n,JSON.stringify(r)]})}async function K(t,e){let r=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[t,e]});return r.rows[0]?_(r.rows[0]):null}async function gt(t){let n=await s().execute({sql:`INSERT OR IGNORE INTO site_extract_jobs
|
|
3
|
-
(id, user_id, status, start_url, options, idempotency_key, request_fingerprint, created_at, updated_at)
|
|
4
|
-
VALUES (?, ?, 'pending', ?, ?, ?, ?, datetime('now'), datetime('now'))`,args:[t.jobId,t.userId,t.startUrl,JSON.stringify(t.options),t.idempotencyKey,t.requestFingerprint]}),r=await K(t.userId,t.idempotencyKey);if(!r)throw new Error("idempotent extract job was not persisted");return{job:r,created:n.rowsAffected===1,conflict:r.requestFingerprint!==t.requestFingerprint}}async function x(t){let n=await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE id = ?",args:[t]});return n.rows[0]?_(n.rows[0]):null}async function _t(t){return(await s().execute({sql:"SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50",args:[t]})).rows.map(r=>_(r))}async function xt(t=25){return(await s().execute({sql:`SELECT * FROM site_extract_jobs
|
|
5
|
-
WHERE billed_mc IS NULL AND status IN ('complete', 'partial', 'failed')
|
|
6
|
-
AND (next_settlement_at IS NULL OR next_settlement_at <= datetime('now'))
|
|
7
|
-
ORDER BY updated_at ASC LIMIT ?`,args:[Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(r=>_(r))}async function ft(t=25,e=2){let n=s(),r=Math.max(1,Math.min(60,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
|
|
8
|
-
LEFT JOIN billing_debits d
|
|
9
|
-
ON d.idempotency_key = json_extract(j.options, '$.debitKey')
|
|
10
|
-
AND d.user_id = j.user_id
|
|
11
|
-
AND d.status = 'applied'
|
|
12
|
-
WHERE j.status = 'pending'
|
|
13
|
-
AND j.billed_mc IS NULL
|
|
14
|
-
AND (
|
|
15
|
-
d.idempotency_key IS NOT NULL
|
|
16
|
-
OR (
|
|
17
|
-
json_extract(j.options, '$.billingClass') = ?
|
|
18
|
-
AND json_extract(j.options, '$.heldMc') = 0
|
|
19
|
-
AND json_extract(j.options, '$.debitKey') IS NULL
|
|
20
|
-
AND json_extract(j.options, '$.xraySetupReceipt.billingClass') = ?
|
|
21
|
-
AND json_extract(j.options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
|
|
22
|
-
)
|
|
23
|
-
)
|
|
24
|
-
AND (j.next_dispatch_at IS NULL OR j.next_dispatch_at <= datetime('now'))
|
|
25
|
-
AND j.updated_at <= datetime('now', ?)
|
|
26
|
-
ORDER BY j.updated_at ASC LIMIT ?`,args:[g,g,`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function bt(t){return await s().execute({sql:`UPDATE site_extract_jobs
|
|
27
|
-
SET dispatch_attempts = dispatch_attempts + 1,
|
|
28
|
-
updated_at = datetime('now'),
|
|
29
|
-
next_dispatch_at = datetime('now', '+5 minutes'),
|
|
30
|
-
dispatch_error = NULL,
|
|
31
|
-
status = CASE
|
|
32
|
-
WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
|
|
33
|
-
ELSE status
|
|
34
|
-
END,
|
|
35
|
-
error = CASE
|
|
36
|
-
WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
|
|
37
|
-
THEN 'Background crawl was acknowledged repeatedly but never started; the hold will be refunded.'
|
|
38
|
-
ELSE error
|
|
39
|
-
END
|
|
40
|
-
WHERE id = ? AND status = 'pending'`,args:[t]}),x(t)}async function wt(t,e){let n=s(),r=u(e).slice(0,1e3);return await n.execute({sql:`UPDATE site_extract_jobs SET
|
|
41
|
-
dispatch_attempts = dispatch_attempts + 1,
|
|
42
|
-
dispatch_error = ?,
|
|
43
|
-
next_dispatch_at = datetime('now', '+5 minutes'),
|
|
44
|
-
updated_at = datetime('now'),
|
|
45
|
-
status = CASE
|
|
46
|
-
WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours') THEN 'failed'
|
|
47
|
-
ELSE status
|
|
48
|
-
END,
|
|
49
|
-
error = CASE
|
|
50
|
-
WHEN dispatch_attempts + 1 >= 10 OR created_at <= datetime('now', '-6 hours')
|
|
51
|
-
THEN 'Background crawl could not be dispatched after repeated attempts; the hold will be refunded.'
|
|
52
|
-
ELSE error
|
|
53
|
-
END
|
|
54
|
-
WHERE id = ? AND status = 'pending'`,args:[r,t]}),x(t)}async function St(t,e){await s().execute({sql:`UPDATE site_extract_jobs SET
|
|
55
|
-
settlement_attempts = settlement_attempts + 1,
|
|
56
|
-
settlement_error = ?,
|
|
57
|
-
next_settlement_at = datetime('now', '+15 minutes'),
|
|
58
|
-
updated_at = datetime('now')
|
|
59
|
-
WHERE id = ? AND billed_mc IS NULL`,args:[u(e).slice(0,1e3),t]})}async function At(t=25,e=60){let n=s(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`SELECT j.* FROM site_extract_jobs j
|
|
60
|
-
LEFT JOIN billing_debits d
|
|
61
|
-
ON d.idempotency_key = json_extract(j.options, '$.debitKey')
|
|
62
|
-
AND d.user_id = j.user_id
|
|
63
|
-
AND d.status = 'applied'
|
|
64
|
-
WHERE j.status = 'running'
|
|
65
|
-
AND j.billed_mc IS NULL
|
|
66
|
-
AND (
|
|
67
|
-
json_extract(j.options, '$.debitKey') IS NULL
|
|
68
|
-
OR d.idempotency_key IS NOT NULL
|
|
69
|
-
)
|
|
70
|
-
AND j.updated_at <= datetime('now', ?)
|
|
71
|
-
ORDER BY j.updated_at ASC LIMIT ?`,args:[`-${r} minutes`,Math.max(1,Math.min(100,Math.round(t)))]})).rows.map(a=>_(a))}async function Nt(t,e=60){let n=s(),r=Math.max(15,Math.min(1440,Math.round(e)));return(await n.execute({sql:`UPDATE site_extract_jobs
|
|
72
|
-
SET status = 'failed',
|
|
73
|
-
error = 'Background crawl stopped without a heartbeat; completed pages were preserved and settlement is pending.',
|
|
74
|
-
public_error_json = ?,
|
|
75
|
-
updated_at = datetime('now')
|
|
76
|
-
WHERE id = ?
|
|
77
|
-
AND status = 'running'
|
|
78
|
-
AND billed_mc IS NULL
|
|
79
|
-
AND updated_at <= datetime('now', ?)`,args:[m({error_code:"extraction_failed",error_type:"extraction",message:"The background crawl stopped without a heartbeat. Completed pages were preserved; retry after settlement completes.",retryable:!0,charge_status:"refund_pending"}),t,`-${r} minutes`]})).rowsAffected===1}async function Tt(t){return(await s().execute({sql:`UPDATE site_extract_jobs
|
|
80
|
-
SET status = 'running', updated_at = datetime('now')
|
|
81
|
-
WHERE id = ? AND status = 'failed'`,args:[t]})).rowsAffected!==1?null:x(t)}async function yt(t,e){await s().execute({sql:`UPDATE site_extract_jobs SET billed_mc = 0, settlement_error = ?, updated_at = datetime('now')
|
|
82
|
-
WHERE id = ? AND billed_mc IS NULL`,args:[u(e).slice(0,1e3),t]})}async function ht(t,e){await s().execute({sql:`UPDATE site_extract_jobs
|
|
83
|
-
SET status = 'running', total_urls = MAX(total_urls, ?), updated_at = datetime('now')
|
|
84
|
-
WHERE id = ? AND status IN ('pending', 'running')`,args:[e,t]})}async function Rt(t,e){if(e.length===0)return;let n=s(),r=await x(t),i=r?.userId!=null?await M({ownerId:String(r.userId),jobId:t,createdAt:r.createdAt,pages:e}):new Map;await n.batch([...e.map(a=>{let{discoveryLinks:o,acquiredHtml:c,mainHtml:d,...l}=a;return l.contentRef=i.get(a.archivedUrl??a.url)??a.contentRef,{sql:`INSERT INTO site_extract_pages (job_id, url, page)
|
|
85
|
-
SELECT ?, ?, ?
|
|
86
|
-
WHERE EXISTS (
|
|
87
|
-
SELECT 1 FROM site_extract_jobs
|
|
88
|
-
WHERE id = ? AND status IN ('pending', 'running')
|
|
89
|
-
)
|
|
90
|
-
ON CONFLICT(job_id, url) DO UPDATE SET page = excluded.page
|
|
91
|
-
WHERE NOT CASE
|
|
92
|
-
WHEN json_valid(site_extract_pages.page) = 0 THEN 0
|
|
93
|
-
WHEN json_extract(site_extract_pages.page, '$.extractionStatus') IS NOT NULL
|
|
94
|
-
THEN json_extract(site_extract_pages.page, '$.extractionStatus') = 'successful'
|
|
95
|
-
ELSE COALESCE(
|
|
96
|
-
json_extract(site_extract_pages.page, '$.status') >= 200
|
|
97
|
-
AND json_extract(site_extract_pages.page, '$.status') < 300
|
|
98
|
-
AND json_extract(site_extract_pages.page, '$.wordCount') > 0,
|
|
99
|
-
0
|
|
100
|
-
)
|
|
101
|
-
END
|
|
102
|
-
OR CASE
|
|
103
|
-
WHEN json_valid(excluded.page) = 0 THEN 0
|
|
104
|
-
WHEN json_extract(excluded.page, '$.extractionStatus') IS NOT NULL
|
|
105
|
-
THEN json_extract(excluded.page, '$.extractionStatus') = 'successful'
|
|
106
|
-
ELSE COALESCE(
|
|
107
|
-
json_extract(excluded.page, '$.status') >= 200
|
|
108
|
-
AND json_extract(excluded.page, '$.status') < 300
|
|
109
|
-
AND json_extract(excluded.page, '$.wordCount') > 0,
|
|
110
|
-
0
|
|
111
|
-
)
|
|
112
|
-
END`,args:[t,a.archivedUrl??a.url,JSON.stringify(l),t]}}),{sql:`UPDATE site_extract_jobs SET
|
|
113
|
-
done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
|
|
114
|
-
attempted_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?),
|
|
115
|
-
successful_urls = (
|
|
116
|
-
SELECT COUNT(*) FROM site_extract_pages
|
|
117
|
-
WHERE job_id = ? AND CASE
|
|
118
|
-
WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
|
|
119
|
-
THEN json_extract(page, '$.extractionStatus') = 'successful'
|
|
120
|
-
ELSE json_extract(page, '$.status') >= 200
|
|
121
|
-
AND json_extract(page, '$.status') < 300
|
|
122
|
-
AND json_extract(page, '$.wordCount') > 0
|
|
123
|
-
END
|
|
124
|
-
),
|
|
125
|
-
failed_urls = (
|
|
126
|
-
SELECT COUNT(*) FROM site_extract_pages
|
|
127
|
-
WHERE job_id = ? AND NOT CASE
|
|
128
|
-
WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
|
|
129
|
-
THEN json_extract(page, '$.extractionStatus') = 'successful'
|
|
130
|
-
ELSE COALESCE(
|
|
131
|
-
json_extract(page, '$.status') >= 200
|
|
132
|
-
AND json_extract(page, '$.status') < 300
|
|
133
|
-
AND json_extract(page, '$.wordCount') > 0,
|
|
134
|
-
0
|
|
135
|
-
)
|
|
136
|
-
END
|
|
137
|
-
),
|
|
138
|
-
updated_at = datetime('now')
|
|
139
|
-
WHERE id = ? AND status IN ('pending', 'running')`,args:[t,t,t,t,t]}],"write")}async function It(t){return(await s().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function Dt(t){return(await s().execute({sql:"SELECT page FROM site_extract_pages WHERE job_id = ?",args:[t]})).rows.map(r=>JSON.parse(String(r.page)))}async function jt(t,e=181){let n=s(),r=Math.max(1,Math.min(5001,Math.round(e)));return(await n.execute({sql:`SELECT DISTINCT CAST(images.value AS TEXT) AS url
|
|
140
|
-
FROM site_extract_pages p,
|
|
141
|
-
json_each(CASE WHEN json_valid(p.page) THEN p.page ELSE '{}' END, '$.imageLinks') images
|
|
142
|
-
WHERE p.job_id = ?
|
|
143
|
-
AND images.type = 'text'
|
|
144
|
-
AND length(CAST(images.value AS TEXT)) BETWEEN 1 AND 4096
|
|
145
|
-
LIMIT ?`,args:[t,r]})).rows.map(a=>String(a.url))}async function Pt(t){let n=await s().execute({sql:`SELECT COUNT(*) AS n FROM site_extract_pages
|
|
146
|
-
WHERE job_id = ? AND CASE
|
|
147
|
-
WHEN json_extract(page, '$.extractionStatus') IS NOT NULL
|
|
148
|
-
THEN json_extract(page, '$.extractionStatus') = 'successful'
|
|
149
|
-
ELSE COALESCE(
|
|
150
|
-
json_extract(page, '$.status') >= 200
|
|
151
|
-
AND json_extract(page, '$.status') < 300
|
|
152
|
-
AND json_extract(page, '$.wordCount') > 0,
|
|
153
|
-
0
|
|
154
|
-
)
|
|
155
|
-
END`,args:[t]});return Number(n.rows[0]?.n??0)}async function Ct(t){let n=await s().execute({sql:"SELECT url FROM site_extract_pages WHERE job_id = ?",args:[t]});return new Set(n.rows.map(r=>String(r.url)))}async function z(t,e,n,r=null,i=!1,a=null){return(await s().execute({sql:`UPDATE site_extract_jobs
|
|
156
|
-
SET status = ?, artifacts = ?, error = ?, public_error_json = ?, updated_at = datetime('now')
|
|
157
|
-
WHERE id = ? AND (status IN ('pending', 'running') OR (? = 1 AND status = 'failed'))`,args:[n,JSON.stringify(e??[]),r?u(r).slice(0,2e3):null,m(a),t,i?1:0]})).rowsAffected===1}async function Lt(t,e){await z(t,e,"complete")}async function Ut(t,e,n){await s().execute({sql:`UPDATE site_extract_jobs
|
|
158
|
-
SET status = 'failed', error = ?, public_error_json = ?, updated_at = datetime('now')
|
|
159
|
-
WHERE id = ? AND status IN ('pending', 'running')`,args:[u(e).slice(0,2e3),m(n),t]})}async function Ot(t,e,n){await s().execute({sql:`UPDATE site_extract_jobs
|
|
160
|
-
SET status = 'failed', billed_mc = 0, error = ?, public_error_json = ?, updated_at = datetime('now')
|
|
161
|
-
WHERE id = ? AND status = 'pending' AND billed_mc IS NULL`,args:[u(e).slice(0,2e3),m(n),t]})}async function kt(t,e){let r=(await x(t))?.options.billingClass===g&&e?{...e,charge_status:"not_charged"}:e;await s().execute({sql:`UPDATE site_extract_jobs SET public_error_json = ?, updated_at = datetime('now')
|
|
162
|
-
WHERE id = ? AND status IN ('complete', 'partial', 'failed')`,args:[m(r),t]})}async function Mt(t,e,n,r,i){let a=s(),o=await x(t);if(!o||o.billedMc!=null)return;if(o.options.billingClass===g){if(!await X(t))throw new Error("xray_extract_zero_charge_finalization_rejected");return}let c=typeof o.options.debitKey=="string"?o.options.debitKey:null;if(c){let f=await D(e,c,Math.max(0,r),w.EXTRACT_SITE_REFUND,i,"extract_site");await a.execute({sql:"UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL",args:[f.final_amount_mc,t]});return}let d=Math.max(0,n),l=Math.max(0,r);await a.batch([{sql:"UPDATE site_extract_jobs SET billed_mc = -1 WHERE id = ? AND billed_mc IS NULL",args:[t]},{sql:`INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
|
|
163
|
-
SELECT ?, ?, ?, ?, datetime(j.created_at, ?)
|
|
164
|
-
FROM site_extract_jobs j
|
|
165
|
-
WHERE j.id = ?
|
|
166
|
-
AND j.billed_mc = -1
|
|
167
|
-
AND ? > 0
|
|
168
|
-
AND changes() = 1
|
|
169
|
-
AND datetime(j.created_at, ?) > datetime('now')`,args:[e,d,d,w.EXTRACT_SITE_REFUND,N,t,d,N]},{sql:`UPDATE users SET balance_mc = balance_mc + ?
|
|
170
|
-
WHERE id = ? AND ? > 0 AND changes() = 1`,args:[d,e,d]},{sql:`INSERT INTO ledger (user_id, amount_mc, operation, description)
|
|
171
|
-
SELECT ?, ?, ?, ? WHERE ? > 0 AND changes() = 1`,args:[e,d,w.EXTRACT_SITE_REFUND,i,d]},{sql:`UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now')
|
|
172
|
-
WHERE id = ? AND billed_mc = -1`,args:[l,t]}],"write")}async function X(t){return(await s().execute({sql:`UPDATE site_extract_jobs
|
|
173
|
-
SET billed_mc = 0, updated_at = datetime('now')
|
|
174
|
-
WHERE id = ?
|
|
175
|
-
AND billed_mc IS NULL
|
|
176
|
-
AND json_extract(options, '$.billingClass') = ?
|
|
177
|
-
AND json_extract(options, '$.heldMc') = 0
|
|
178
|
-
AND json_extract(options, '$.debitKey') IS NULL
|
|
179
|
-
AND json_extract(options, '$.xraySetupReceipt.billingClass') = ?
|
|
180
|
-
AND json_extract(options, '$.xraySetupReceipt.consumesMcpScraperCredits') = 0
|
|
181
|
-
AND json_extract(options, '$.xraySetupReceipt.heldMc') = 0
|
|
182
|
-
AND json_extract(options, '$.xraySetupReceipt.billedMc') = 0`,args:[t,g,g]})).rowsAffected===1}export{y as a,V as b,G as c,Q as d,Z as e,tt as f,et as g,it as h,g as i,pt as j,Et as k,mt as l,K as m,gt as n,x as o,_t as p,xt as q,ft as r,bt as s,wt as t,St as u,At as v,Nt as w,Tt as x,yt as y,ht as z,Rt as A,It as B,Dt as C,jt as D,Pt as E,Ct as F,z as G,Lt as H,Ut as I,Ot as J,kt as K,Mt as L,X as M};
|
package/dist/chunk-GPSPSJBJ.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
function o(e,_=!1,n="light",r=1,R=!1,t=!1){let E=_&&(R||t)?24e4:_&&n==="full"?r>=2?75e3:45e3:_?4e4:4e5,S=Math.min(E+5e3,795e3);return{serverMs:E,clientMs:S}}function M(){return(process.env.BROWSER_SERVICE_API_KEY??process.env.KERNEL_API_KEY)?.trim()||void 0}function u(){return(process.env.BROWSER_AGENT_PROFILE_NAME??process.env.BROWSER_SERVICE_PROFILE_NAME??process.env.KERNEL_BROWSER_PROFILE_NAME??process.env.KERNEL_PROFILE_NAME)?.trim()||void 0}function i(){let e=(process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES??process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES??process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES??process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();if(e){if(["1","true","yes","on"].includes(e))return!0;if(["0","false","no","off"].includes(e))return!1}}export{o as a,M as b,u as c,i as d};
|
package/dist/chunk-HE45FFBU.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{isIP as I}from"net";import{lookup as T}from"dns/promises";import p from"ipaddr.js";function P(n){return n.startsWith("[")&&n.endsWith("]")?n.slice(1,-1):n}function b(n){let t=P(n);if(!I(t))return!1;try{let e=p.parse(t),s=o=>[["192.0.0.0",24],["192.0.2.0",24],["198.18.0.0",15],["198.51.100.0",24],["203.0.113.0",24]].some(([m,u])=>o.match(p.parse(m),u));if(e instanceof p.IPv6&&e.isIPv4MappedAddress()){let o=e.toIPv4Address();return o.range()!=="unicast"||s(o)}return e instanceof p.IPv4&&s(e)?!0:e.range()!=="unicast"}catch{return!0}}async function N(n){let t=P(n.toLowerCase());if(t==="localhost"||t.endsWith(".localhost")||t.endsWith(".local")||b(t))return!0;try{let e=await T(t,{all:!0,verbatim:!0});return e.length===0||e.some(s=>b(s.address))}catch{return!0}}async function S(n,t){let e;try{e=new URL(n.trim())}catch{return{error:`Invalid ${t.field}`}}return(t.requireHttps?["https:"]:["http:","https:"]).includes(e.protocol)?await N(e.hostname)?{error:`${t.field} must resolve to a public internet host`}:{parsed:e}:{error:t.requireHttps?`${t.field} must use https`:`${t.field} must use http or https`}}import{load as C}from"cheerio";var _="script,style,noscript,svg,iframe,canvas,nav,header,footer,aside",k=4096;function d(n){return(n??"").replace(/\s+/g," ").trim()}function L(n){let t=n.replace(/[\x00-\x09\x0b\x0c\x0e-\x1f]/g," ").trim();if(!t)return null;try{return JSON.parse(t)}catch(e){let s=/at position (\d+)/.exec(e instanceof Error?e.message:"")?.[1];if(!s)return null;try{return JSON.parse(t.slice(0,Number(s)))}catch{return null}}}function g(n,t,e={value:2e3}){if(e.value--<=0||n==null)return;if(Array.isArray(n)){for(let l of n)g(l,t,e);return}if(typeof n!="object")return;let s=n,o=s["@type"];if(typeof o=="string"&&o&&t.add(o),Array.isArray(o))for(let l of o)typeof l=="string"&&l&&t.add(l);Array.isArray(s["@graph"])&&g(s["@graph"],t,e)}function H(n){let t=n("main").first();if(t.length)return t;let e=n("article").first();if(e.length)return e;let s=n("body").first();return s.length?s:n.root()}function w(n,t){let e=Number(n.attr(t)??0);return Number.isFinite(e)&&e>0?e:null}function W(n){let t=C(n),e={};t("meta").each((f,i)=>{let r=t(i),a=d(r.attr("name")??r.attr("property")).toLowerCase(),c=d(r.attr("content"));a&&c&&(e[a]=c)});let s=[];t("h1,h2,h3,h4,h5,h6").each((f,i)=>{let r=t(i),a=d(r.text());a&&s.push({level:Number(i.tagName.slice(1)),text:a})});let o=[],l=new Set;t('script[type="application/ld+json"]').each((f,i)=>{let r=L(t(i).text());r!=null&&(Array.isArray(r)?o.push(...r):o.push(r),g(r,l))});let m=new Set;t("[itemtype]").each((f,i)=>{let r=t(i).attr("itemtype")??"";for(let a of r.split(/\s+/))try{let c=new URL(a);if(c.hostname==="schema.org"||c.hostname==="www.schema.org"){let h=c.pathname.replace(/^\/+/,"");h&&m.add(h)}}catch{}});let u=H(t).clone();u.find(_).remove(),u.find("[href],[src],[srcset],[data-src],[data-srcset]").each((f,i)=>{let r=t(i);for(let a of["href","src","srcset","data-src","data-srcset"]){let c=r.attr(a);c&&c.length>k&&r.removeAttr(a)}}),u.find("img").each((f,i)=>{let r=t(i),a=r.attr("src");if(!a||a.startsWith("data:")){let c=r.attr("data-src")??r.attr("data-lazy-src")??r.attr("data-original");c&&r.attr("src",c)}});let A=u.html()??"",v=d(u.text()),y=[];t("a[href]").each((f,i)=>{let r=t(i),a=d(r.attr("href"));a&&y.push({href:a,text:d(r.text()),rel:d(r.attr("rel")).toLowerCase()||null})});let x=[];return t("img").each((f,i)=>{let r=t(i),a=Object.fromEntries(Object.entries(i.attribs??{}).map(([c,h])=>[c.toLowerCase(),h]));x.push({src:r.attr("src")??r.attr("data-src")??r.attr("data-lazy-src")??r.attr("data-original")??null,srcset:r.attr("srcset")??r.attr("data-srcset")??null,alt:r.attr("alt")??null,width:w(r,"width"),height:w(r,"height"),attributes:a})}),{$:t,sourceHtml:n,title:d(t("title").first().text())||null,meta:e,headings:s,schema:o,schemaTypes:[...l],microdataTypes:[...m],canonicalUrl:d(t('link[rel~="canonical"]').first().attr("href"))||null,mainHtml:A,mainText:v,bodyClass:t("body").first().attr("class")??"",links:y,images:x}}export{b as a,S as b,W as c};
|
package/dist/chunk-HUV2WTRW.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{createHash as B,randomUUID as N}from"crypto";import{mkdir as O,readFile as T,stat as v,writeFile as U}from"fs/promises";import{homedir as k}from"os";import{dirname as S,join as A}from"path";var d="connected-data-exports/",b=10080*60*1e3,R=900*1e3;function p(){return process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()||process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()||null}function D(){return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim()||A(k(),"Downloads","mcp-scraper")}function g(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function x(t){return(t.toLowerCase().replace(/[^a-z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||"connected-data-export").slice(0,120)}function C(t){if(!t.startsWith(d))return null;let e=(t.split("/").at(-1)??"").match(/^(\d{13})-/);if(!e)return null;let r=Number(e[1]);return Number.isFinite(r)?r:null}function _(t){return t.startsWith(d)&&t.slice(d.length).split("/")[0]||null}function P(t){let n=C(t);return n===null?null:new Date(n+b)}function I(t,n=Date.now()){let e=C(t);return e===null||e+b<=n}async function h(t,n){let e=p();if(!e)return null;let r=Math.min(Date.now()+R,n.getTime());if(r<=Date.now())return null;let{issueSignedToken:a,presignUrl:i}=await import("@vercel/blob"),s=await a({token:e,pathname:t,operations:["get"],validUntil:r}),{presignedUrl:o}=await i(s,{access:"private",operation:"get",pathname:t,validUntil:r});return{url:o,expiresAt:new Date(r).toISOString()}}async function H(t){return L({...t,contentType:"application/x-ndjson"})}async function L(t){let n=t.stableCreatedAt?Date.parse(t.stableCreatedAt):Number.NaN,e=Number.isFinite(n)?n:Date.now(),r=x(t.filename),a=r.includes(".")?`.${r.split(".").at(-1)}`:".bin",i=t.stableKey?x(t.stableKey):N(),s=`${d}${t.ownerId}/${e}-${x(t.exportId)}-${i}--${r}`,o=(typeof t.content=="string",Buffer.from(t.content)),u=o.byteLength,c=B("sha256").update(o).digest("hex"),l=new Date(e+b),f=p(),m=s;if(f){let{put:w}=await import("@vercel/blob");m=(await w(s,o,{access:"private",token:f,contentType:t.contentType,addRandomSuffix:!t.stableKey,...t.stableKey?{allowOverwrite:!0}:{},cacheControlMaxAge:60,multipart:u>100*1024*1024})).pathname}else{if(g())throw new Error("connected_data_private_blob_not_configured");let w=A(D(),"blobs",s);await O(S(w),{recursive:!0}),await U(w,o)}let y=await h(m,l);return{artifactId:m,filename:r,contentType:t.contentType,bytes:u,sha256:c,createdAt:new Date(e).toISOString(),expiresAt:l.toISOString(),downloadUrl:y?.url??null,downloadUrlExpiresAt:y?.expiresAt??null,readback:{tool:"report_artifact_read",arguments:{artifactId:m,offset:0,maxBytes:2e4},continuation:"Repeat with offset set to the previous result nextOffset until nextOffset is null."}}}async function V(t){if(_(t.artifactId)!==t.ownerId)return null;let n=P(t.artifactId);if(!n||n.getTime()<=Date.now())return null;let e=await h(t.artifactId,n);return e?{downloadUrl:e.url,downloadUrlExpiresAt:e.expiresAt,expiresAt:n.toISOString()}:null}async function E(t,n=Number.POSITIVE_INFINITY){let e=t.getReader(),r=[],a=0;try{for(;;){let{done:i,value:s}=await e.read();if(i)break;if(s){let o=Buffer.from(s);if(a+=o.byteLength,a>n)return await e.cancel("artifact_too_large").catch(()=>{}),null;r.push(o)}}}finally{e.releaseLock()}return Buffer.concat(r)}async function j(t,n,e){if(I(t))return null;let r=p(),a;if(r){let{get:u}=await import("@vercel/blob"),c=await u(t,{access:"private",token:r,useCache:!1});if(!c||c.statusCode!==200)return null;let l=await E(c.stream);if(!l)return null;a=l}else{if(g())return null;try{a=await T(A(D(),"blobs",t))}catch{return null}}let i=a.length,s=Math.max(0,n),o=Math.min(i,s+e);return{text:a.subarray(s,o).toString("utf8"),totalBytes:i,nextOffset:o<i?o:null}}async function q(t){if(_(t.artifactId)!==t.ownerId||I(t.artifactId))return null;let n=p(),e;if(n){let{get:r}=await import("@vercel/blob"),a=await r(t.artifactId,{access:"private",token:n,useCache:!1});if(!a||a.statusCode!==200)return null;let i=await E(a.stream,t.maxBytes??25*1024*1024);if(!i)return null;e=i}else{if(g())return null;try{let r=A(D(),"blobs",t.artifactId);if((await v(r)).size>(t.maxBytes??25*1024*1024))return null;e=await T(r)}catch{return null}}return e.byteLength>(t.maxBytes??25*1024*1024)?null:e}async function z(t={}){let n=t.now??new Date,e=p();if(!e)return{deleted:0,store:g()?"none":"local"};if(!t.force&&!(n.getUTCHours()===3&&n.getUTCMinutes()===17))return{deleted:0,store:"private-vercel-blob",skipped:!0};let r=n.getTime()-b,{list:a,del:i}=await import("@vercel/blob"),s,o=0;for(let u=0;u<20;u++){let c=await a({prefix:d,token:e,limit:1e3,cursor:s}),l=c.blobs.filter(f=>new Date(f.uploadedAt).getTime()<=r);if(l.length>0&&(await i(l.map(f=>f.pathname),{token:e}),o+=l.length),!c.hasMore||!c.cursor)break;s=c.cursor}return{deleted:o,store:"private-vercel-blob"}}export{d as a,_ as b,H as c,L as d,V as e,j as f,q as g,z as h};
|
package/dist/chunk-L552AQQY.js
DELETED
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
import{R as p,t as o}from"./chunk-ZPK5LYEN.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
|
|
2
|
-
CREATE TABLE IF NOT EXISTS service_connections (
|
|
3
|
-
id TEXT PRIMARY KEY,
|
|
4
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
5
|
-
provider_config_key TEXT NOT NULL,
|
|
6
|
-
provider TEXT,
|
|
7
|
-
transport TEXT NOT NULL,
|
|
8
|
-
upstream_connection_id TEXT,
|
|
9
|
-
label TEXT,
|
|
10
|
-
user_label TEXT,
|
|
11
|
-
provider_account_id TEXT,
|
|
12
|
-
provider_account_email TEXT,
|
|
13
|
-
provider_account_name TEXT,
|
|
14
|
-
provider_identity_checked_at TEXT,
|
|
15
|
-
provider_identity_source_updated_at TEXT,
|
|
16
|
-
lifecycle_status TEXT NOT NULL DEFAULT 'pending',
|
|
17
|
-
operational_status TEXT NOT NULL DEFAULT 'unknown',
|
|
18
|
-
reconnect_required INTEGER NOT NULL DEFAULT 0,
|
|
19
|
-
actions_enabled INTEGER NOT NULL DEFAULT 0,
|
|
20
|
-
read_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
21
|
-
action_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
22
|
-
tool_revision TEXT,
|
|
23
|
-
last_checked_at TEXT,
|
|
24
|
-
last_successful_call_at TEXT,
|
|
25
|
-
last_failure_at TEXT,
|
|
26
|
-
last_failure_code TEXT,
|
|
27
|
-
last_failure_retryable INTEGER,
|
|
28
|
-
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
29
|
-
vault_name TEXT,
|
|
30
|
-
table_name TEXT,
|
|
31
|
-
legacy_owner_identity TEXT,
|
|
32
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
33
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
34
|
-
UNIQUE(user_id, provider_config_key, upstream_connection_id)
|
|
35
|
-
)
|
|
36
|
-
`);try{await t.execute("ALTER TABLE service_connections ADD COLUMN user_label TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT")}catch{}await t.execute("CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)"),await t.execute("CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)"),await t.execute(`
|
|
37
|
-
CREATE TABLE IF NOT EXISTS service_connection_health_events (
|
|
38
|
-
id TEXT PRIMARY KEY,
|
|
39
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
40
|
-
operational_status TEXT NOT NULL,
|
|
41
|
-
failure_code TEXT,
|
|
42
|
-
retryable INTEGER,
|
|
43
|
-
evidence_source TEXT NOT NULL,
|
|
44
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
45
|
-
)
|
|
46
|
-
`),await t.execute("CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)"),await t.execute(`
|
|
47
|
-
CREATE TABLE IF NOT EXISTS service_connection_action_audit (
|
|
48
|
-
id TEXT PRIMARY KEY,
|
|
49
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
50
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
51
|
-
tool TEXT NOT NULL,
|
|
52
|
-
request_id TEXT NOT NULL,
|
|
53
|
-
status TEXT NOT NULL,
|
|
54
|
-
error_code TEXT,
|
|
55
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
56
|
-
UNIQUE(user_id, request_id)
|
|
57
|
-
)
|
|
58
|
-
`);try{await t.execute("ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT")}catch{}try{await t.execute("ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT")}catch{}})().catch(t=>{throw a=null,u=null,t})),a}function f(e){if(typeof e!="string")return[];try{let t=JSON.parse(e);return Array.isArray(t)?t.filter(n=>typeof n=="string"):[]}catch{return[]}}function g(e){return{id:String(e.id),userId:Number(e.user_id),providerConfigKey:String(e.provider_config_key),provider:typeof e.provider=="string"?e.provider:null,transport:e.transport==="remote_mcp"?"remote_mcp":e.transport==="credential_reference"?"credential_reference":"nango",upstreamConnectionId:typeof e.upstream_connection_id=="string"?e.upstream_connection_id:null,label:typeof e.label=="string"?e.label:null,userLabel:typeof e.user_label=="string"?e.user_label:null,providerAccountId:typeof e.provider_account_id=="string"?e.provider_account_id:null,providerAccountEmail:typeof e.provider_account_email=="string"?e.provider_account_email:null,providerAccountName:typeof e.provider_account_name=="string"?e.provider_account_name:null,providerIdentityCheckedAt:typeof e.provider_identity_checked_at=="string"?e.provider_identity_checked_at:null,providerIdentitySourceUpdatedAt:typeof e.provider_identity_source_updated_at=="string"?e.provider_identity_source_updated_at:null,lifecycleStatus:String(e.lifecycle_status),operationalStatus:String(e.operational_status),reconnectRequired:Number(e.reconnect_required)===1,actionsEnabled:Number(e.actions_enabled)===1,readTools:f(e.read_tools_json),actionTools:f(e.action_tools_json),toolRevision:typeof e.tool_revision=="string"?e.tool_revision:null,lastCheckedAt:typeof e.last_checked_at=="string"?e.last_checked_at:null,lastSuccessfulCallAt:typeof e.last_successful_call_at=="string"?e.last_successful_call_at:null,lastFailureAt:typeof e.last_failure_at=="string"?e.last_failure_at:null,lastFailureCode:typeof e.last_failure_code=="string"?e.last_failure_code:null,lastFailureRetryable:e.last_failure_retryable===null||e.last_failure_retryable===void 0?null:Number(e.last_failure_retryable)===1,consecutiveFailures:Number(e.consecutive_failures??0),vaultName:typeof e.vault_name=="string"?e.vault_name:null,tableName:typeof e.table_name=="string"?e.table_name:null,createdAt:String(e.created_at),updatedAt:String(e.updated_at)}}async function E(e){let t=await p(e.normalize("NFKC").trim().toLowerCase());if(!t)throw new Error("service_connection_user_not_found");return Number(t.id)}async function D(e,t){await c();let n=o(),r=await E(e);for(let i of t)await n.execute({sql:`
|
|
59
|
-
INSERT INTO service_connections (
|
|
60
|
-
id, user_id, provider_config_key, provider, transport, upstream_connection_id,
|
|
61
|
-
label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
|
|
62
|
-
) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
|
|
63
|
-
ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
|
|
64
|
-
provider = excluded.provider,
|
|
65
|
-
label = COALESCE(excluded.label, service_connections.label),
|
|
66
|
-
lifecycle_status = excluded.lifecycle_status,
|
|
67
|
-
reconnect_required = excluded.reconnect_required,
|
|
68
|
-
updated_at = excluded.updated_at
|
|
69
|
-
`,args:[_(),r,i.providerConfigKey,i.provider,i.upstreamConnectionId,i.label,i.lifecycleStatus,i.reconnectRequired?1:0,e,i.createdAt,i.updatedAt]});return A(e,"nango")}async function A(e,t){await c();let n=await E(e);return(await o().execute({sql:`SELECT * FROM service_connections WHERE user_id = ?${t?" AND transport = ?":""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,args:t?[n,t]:[n]})).rows.map(i=>g(i))}async function d(e,t){await c();let n=await E(e),i=(await o().execute({sql:"SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",args:[t,n]})).rows[0];return i?g(i):null}async function w(e,t,n,r){await c(),await o().execute({sql:"UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?",args:[JSON.stringify(t),JSON.stringify(n),r,e]})}async function U(e){await c(),await o().execute({sql:`
|
|
70
|
-
UPDATE service_connections SET
|
|
71
|
-
provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
|
|
72
|
-
provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
|
|
73
|
-
updated_at = datetime('now')
|
|
74
|
-
WHERE id = ?
|
|
75
|
-
`,args:[e.providerAccountId??null,e.providerAccountEmail?.toLowerCase()??null,e.providerAccountName??null,e.sourceUpdatedAt,e.connectionId]})}async function h(e,t,n){let r=await d(e,t);if(!r||r.lifecycleStatus==="disconnected")throw new Error("service_connection_not_found");return await o().execute({sql:"UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[n,t,r.userId]}),n}async function X(e){await c();let t=new Date().toISOString();await o().execute({sql:`
|
|
76
|
-
UPDATE service_connections SET
|
|
77
|
-
operational_status = ?, last_checked_at = ?,
|
|
78
|
-
last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
|
|
79
|
-
last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
|
|
80
|
-
last_failure_code = ?, last_failure_retryable = ?,
|
|
81
|
-
consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
|
|
82
|
-
updated_at = datetime('now')
|
|
83
|
-
WHERE id = ?
|
|
84
|
-
`,args:[e.operationalStatus,t,e.operationalStatus,t,e.operationalStatus,t,e.failureCode??null,e.retryable==null?null:e.retryable?1:0,e.operationalStatus,e.connectionId]}),await o().execute({sql:"INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)",args:[_(),e.connectionId,e.operationalStatus,e.failureCode??null,e.retryable==null?null:e.retryable?1:0,e.evidenceSource]})}async function q(e,t,n){let r=await d(e,t);if(!r||r.lifecycleStatus==="disconnected")throw new Error("service_connection_not_found");return await o().execute({sql:"UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[n?1:0,t,r.userId]}),n}async function k(e,t){let n=await d(e,t);if(!n)throw new Error("service_connection_not_found");await o().execute({sql:"UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[t,n.userId]});let r=await o().execute({sql:"SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'",args:[n.userId]});return Number(r.rows[0]?.count??0)}async function F(e){await c();let t=await d(e.identity,e.connectionId);if(!t)throw new Error("service_connection_not_found");await o().execute({sql:"UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?",args:[e.status,e.errorCode??null,e.result===void 0?null:JSON.stringify(e.result),t.userId,e.requestId,t.id]})}async function P(e){await c();let t=await d(e.identity,e.connectionId);if(!t)throw new Error("service_connection_not_found");let n=await o().execute({sql:"INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)",args:[_(),t.id,t.userId,e.tool,e.requestId,e.requestDigest]});if(Number(n.rowsAffected??0)===1)return{claimed:!0};let i=(await o().execute({sql:"SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1",args:[t.userId,e.requestId]})).rows[0],s=String(i?.connection_id??"")!==t.id||String(i?.tool??"")!==e.tool||String(i?.request_digest??"")!==e.requestDigest,l;if(!s&&typeof i?.result_json=="string")try{l=JSON.parse(i.result_json)}catch{l=void 0}return{claimed:!1,status:String(i?.status??"unknown"),errorCode:typeof i?.error_code=="string"?i.error_code:null,conflict:s,...l!==void 0?{result:l}:{}}}export{S as a,L as b,I as c,x as d,c as e,D as f,A as g,d as h,w as i,U as j,h as k,X as l,q as m,k as n,F as o,P as p};
|