mcp-scraper 0.95.2 → 0.96.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -1
- package/README.md +21 -3
- package/dist/actions-R4EJDLRO.js +1 -0
- package/dist/artifacts-4ZDNP2JW.js +1 -0
- package/dist/artifacts-N62OPIT5.js +1 -0
- package/dist/artifacts-PSADD5OC.js +1 -0
- package/dist/artifacts-RUR3OHAK.js +1 -0
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +22 -22
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/blob-store-GIH26EDO.js +1 -0
- package/dist/bundle-FCXZMIRB.js +1 -0
- package/dist/chunk-24YDQ3EO.js +5 -0
- package/dist/chunk-3F7AFM53.js +1 -0
- package/dist/chunk-4HZXATHB.js +27 -0
- package/dist/chunk-55ZANTHL.js +3 -0
- package/dist/chunk-5BKSKLHX.js +1 -0
- package/dist/chunk-5LYINW7Z.js +8 -0
- package/dist/chunk-63VWBMYR.js +3 -0
- package/dist/chunk-6BD75VKS.js +218 -0
- package/dist/chunk-6N5EC7ZO.js +1 -0
- package/dist/chunk-7POHNTVW.js +1 -0
- package/dist/chunk-7SR7MEU5.js +62 -0
- package/dist/chunk-7WPCGRD7.js +182 -0
- package/dist/chunk-B3JRL45T.js +1 -0
- package/dist/chunk-B4ZKPLSP.js +1 -0
- package/dist/chunk-BPKTUCV7.js +12 -0
- package/dist/chunk-C7MGJUTU.js +1 -0
- package/dist/{chunk-W2BVJ7S2.js → chunk-CNSVZTTT.js} +4 -4
- package/dist/chunk-CORWWDHL.js +1 -0
- package/dist/chunk-CVQUHVHX.js +1 -0
- package/dist/chunk-FB4KBB5W.js +16 -0
- package/dist/{chunk-Y3Q4FIKK.js → chunk-H6MODA7W.js} +199 -84
- package/dist/chunk-HCK6PLM3.js +3 -0
- package/dist/chunk-HNNYXMSI.js +1 -0
- package/dist/chunk-HUNSEGUN.js +57 -0
- package/dist/{chunk-MC4W6TZT.js → chunk-HVQZRQRJ.js} +41 -8
- package/dist/chunk-IFPION5D.js +121 -0
- package/dist/chunk-IY6ZUUY7.js +173 -0
- package/dist/chunk-JHT6A4U6.js +11 -0
- package/dist/chunk-JMZIDHYQ.js +2 -0
- package/dist/chunk-KXJCGV3H.js +1 -0
- package/dist/chunk-MVMWPKRC.js +2 -0
- package/dist/chunk-NALXKTUU.js +1 -0
- package/dist/chunk-NNOX3FYG.js +198 -0
- package/dist/chunk-OILOHQNL.js +2 -0
- package/dist/chunk-OLYPLX5C.js +1 -0
- package/dist/chunk-QER2ELZP.js +1 -0
- package/dist/chunk-RDHGQ2WZ.js +1 -0
- package/dist/chunk-RIHI3KH7.js +955 -0
- package/dist/chunk-RN4VG3VH.js +4 -0
- package/dist/chunk-RYX4FJVH.js +1 -0
- package/dist/chunk-SDQW7IJN.js +1 -0
- package/dist/{chunk-ENZUJEIM.js → chunk-SE4VKLEC.js} +1 -1
- package/dist/chunk-SEZETSB3.js +1 -0
- package/dist/chunk-SFUOSJ3A.js +159 -0
- package/dist/chunk-UIOSADPB.js +1718 -0
- package/dist/{chunk-ZH75KWBI.js → chunk-UNEJYZ4T.js} +1 -1
- package/dist/chunk-UXNR3K6I.js +1 -0
- package/dist/{chunk-VREXSMC6.js → chunk-VVSHZV5S.js} +5 -5
- package/dist/chunk-WNUXMPVX.js +1 -0
- package/dist/chunk-WP3QS4CJ.js +1 -0
- package/dist/chunk-ZTMQEDIB.js +41 -0
- package/dist/contracts-GUXGFCVV.js +1 -0
- package/dist/core-schema-423BSTE6.js +936 -0
- package/dist/dispatch-P7IY75LH.js +1 -0
- package/dist/index.cjs +103 -213
- package/dist/index.d.cts +184 -169
- package/dist/index.d.ts +184 -169
- package/dist/index.js +1 -1
- package/dist/media-artifacts-67CJWI7O.js +1 -0
- package/dist/metering-ZH2P62IC.js +1 -0
- package/dist/migrations-3AKEC5QE.js +1 -0
- package/dist/queue-AC4XKD6Z.js +1 -0
- package/dist/reconciliation-UZUZ6B74.js +1 -0
- package/dist/report-artifact-offload-6OOY2PTM.js +1 -0
- package/dist/repository-4EC5EXVN.js +1 -0
- package/dist/{location-data-repository-A2QZVP7O.js → repository-CYXVMYAI.js} +1 -1
- package/dist/server-DXZ27KQO.js +5825 -0
- package/dist/settlement-6CS4JZTY.js +1 -0
- package/dist/worker-ATRD6VKM.js +1 -0
- package/package.json +2 -2
- package/dist/analytics-repository-DZZKDZL6.js +0 -1
- package/dist/chunk-6DRKDSXY.js +0 -182
- package/dist/chunk-6ND5AFV5.js +0 -27
- package/dist/chunk-7XDBGQDY.js +0 -6
- package/dist/chunk-7YOXSMHG.js +0 -1565
- package/dist/chunk-BLVBUMH5.js +0 -21
- package/dist/chunk-EDQ57S4P.js +0 -1
- package/dist/chunk-GLTECY3U.js +0 -1282
- package/dist/chunk-GPSPSJBJ.js +0 -1
- package/dist/chunk-HE45FFBU.js +0 -1
- package/dist/chunk-HUV2WTRW.js +0 -1
- package/dist/chunk-IXJQHXDC.js +0 -2595
- package/dist/chunk-K3D2S36Q.js +0 -16
- package/dist/chunk-KFOFU3LA.js +0 -1
- package/dist/chunk-LMWU7VPZ.js +0 -63
- package/dist/chunk-O4LUUIYG.js +0 -4
- package/dist/chunk-OLSW4AN4.js +0 -19
- package/dist/chunk-RLFCJGHA.js +0 -100
- package/dist/chunk-T4VTJNDZ.js +0 -84
- package/dist/chunk-TMB56NCA.js +0 -1
- package/dist/chunk-VXOXB3IB.js +0 -1
- package/dist/chunk-WO3N5FH2.js +0 -5
- package/dist/chunk-WO46LTBW.js +0 -1
- package/dist/chunk-XPEJB4BZ.js +0 -1
- package/dist/db-CDEZHBRN.js +0 -1
- package/dist/extract-bundle-SXC6EWJR.js +0 -26
- package/dist/gmail-service-EWR76XE7.js +0 -1
- package/dist/lead-list-enrichment-repository-SIWGXWQV.js +0 -1
- package/dist/operation-metering-UJ53YX7S.js +0 -1
- package/dist/server-QVEKDIDQ.js +0 -5014
- package/dist/site-extract-repository-23ND4N77.js +0 -1
- package/dist/stripe-event-worker-LPXTITVC.js +0 -1
- package/dist/worker-EDN2TJH3.js +0 -1
- /package/dist/{chunk-YGBTTW5D.js → chunk-2IATUJO6.js} +0 -0
- /package/dist/{chunk-6DTXIZY2.js → chunk-4ZVO6GX4.js} +0 -0
- /package/dist/{chunk-XG6GCEUE.js → chunk-U4WIJZCX.js} +0 -0
package/dist/chunk-K3D2S36Q.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
var s={message:"Maps search returns candidates whose website may be unavailable without a profile lookup. Directory jobs can be polled without starting another search. Google Search local-pack features require unfiltered one-page full mode, and retryable controls automatic retry decisions."};var _={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${_[e]}${t}${_.reset}`:t}function o(t,e,n){let a=t.padEnd(9," ");return` ${r(a,"muted",n)} ${e.join(r(" . ","muted",n))}`}function p(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",a=String.raw`
|
|
2
|
-
__ __ ____ ____
|
|
3
|
-
| \/ |/ ___| _ \
|
|
4
|
-
| |\/| | | | |_) |
|
|
5
|
-
| | | | |___| __/
|
|
6
|
-
|_| |_|\____|_|
|
|
7
|
-
|
|
8
|
-
____ ____ ____ _ ____ _____ ____
|
|
9
|
-
/ ___| / ___| _ \ / \ | _ \| ____| _ \
|
|
10
|
-
\___ \| | | |_) | / _ \ | |_) | _| | |_) |
|
|
11
|
-
___) | |___| _ < / ___ \| __/| |___| _ <
|
|
12
|
-
|____/ \____|_| \_\/_/ \_\_| |_____|_| \_\
|
|
13
|
-
`,c=[`MCP_SCRAPER_API_KEY=${n} npx -y -p mcp-scraper@latest \\`," mcp-scraper-cli agent install claude --apply"].join(`
|
|
14
|
-
`),i=["[mcp_servers.mcp-scraper]",'command = "npx"','args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',`env = { MCP_SCRAPER_API_KEY = "${n}" }`].join(`
|
|
15
|
-
`);return[r(`mcp-scraper v${t.version}`,"bold",e),r("> mcp-scraper-install","muted",e),r(a,"amber",e),`${r("MCP Scraper Agent","cyan",e)} . v${t.version} . mcpscraper.dev`,"1/1 install surfaces ready",r(`Newest in v${t.version}: ${s.message}`,"lime",e),"",`${r("Tools","cyan",e)} ${r("(364 MCP tools)","muted",e)}`,o("search",["harvest_paa","search_serp","maps_search","maps_place_intel"],e),o("extract",["extract_url","map_site_urls","extract_site","audit_site","directory_workflow"],e),o("build",["create_editorial_reading_room","rank_tracker_workflow","portable HTML"],e),o("media",["youtube_harvest","youtube_transcribe","facebook_reels_inventory","facebook_ad_search","facebook_page_intel","facebook_ad_transcribe","facebook_video_transcribe","tiktok_video_transcribe","instagram_profile_content","instagram_media_download","reddit_thread"],e),o("browser",["serp_identity_create","serp_identity_list","browser_open","browser_profile_connect","browser_profile_list","browser_close","browser_screenshot","browser_read","browser_locate","browser_replay_mark","browser_replay_annotate"],e),o("connect",["list_service_connections","describe_service_connection_tool","import_service_connection_to_memory","export_connected_service_data","renew_connected_data_download","read_service_connection","call_service_connection_action"],e),o("commons",["commons_search_entities","commons_get_entity_linkset","commons_prepare_entity","commons_submit_entity","commons_prepare_publication","commons_claim_publication","commons_publish_editorial","commons_get_publication"],e),o("account",["credits_info","reports","MCP resources"],e),o("memory",["memory-put","memory-get","memory-search","list-vaults","record-fact","list-scheduled-actions"],e),`${r("Workflows","cyan",e)} ${r("(MCP + CLI + API)","muted",e)}`,o("route",["workflow_list","workflow_suggest","workflow_run","workflow_step","workflow_status","workflow_artifact_read"],e),o("seo",["directory","agent-packet","competitive audit","map/serp comparison","PAA/AIO briefs","scheduled runs"],e),"",r("Usage tips:","amber",e),"Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.","Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.","Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install","Hosted browser sessions use direct/no-proxy egress by default.","Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.","Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.","Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.","Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.","Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.","For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.","If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.","",`${r("Ready.","lime",e)} Install the combined MCP server with one command:`,"",r("Setup doctor","amber",e),"npx -y -p mcp-scraper@latest mcp-scraper-cli doctor","",r("Hosted profile setup","amber",e),'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',"Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.","",r("Claude Code one-command setup","amber",e),c,"Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list","",r("Codex config","amber",e),i,"",r("Claude Desktop Extension","amber",e),"Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb","",r("Safety note:","muted",e),"mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.","Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",""].join(`
|
|
16
|
-
`)}export{p as a};
|
package/dist/chunk-KFOFU3LA.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{a as w,b as c,c as m,d as u,e as _,f as l,g as h,n as g,q as p}from"./chunk-LMWU7VPZ.js";import{b as I,c as s}from"./chunk-6DTXIZY2.js";async function R(e){let r=s(),n=!r?.operationRunId,o=r?.operationRunId??(await c({userId:r?.userId??null,ownerScope:r?.userId!=null?`user:${r.userId}`:"system:unscoped",tool:e.tool,normalizedFlags:{subOperation:r?.subOp??null},deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,a=n?null:await p(o),i=a?.attempts.at(-1),t=e.parentAttemptId??(e.retryKind&&i?String(i.id):r?.operationRootAttemptId),b=e.edgeKind??(e.retryKind==="same_method"?"same_method_retry":e.retryKind==="provider_fallback"?"provider_fallback":void 0),d=t?await _({runId:o,parentAttemptId:t,edgeKind:b??"parallel_child",method:e.method,provider:e.provider,retryKind:e.retryKind,retryReason:e.retryReason}):await u({runId:o,attemptNumber:(a?.attempts.length??0)+1,method:e.method,provider:e.provider,...i?{parentAttemptId:String(i.id),edgeKind:b??"next_phase",retryKind:e.retryKind??null,retryReason:e.retryReason??null}:{}}),y;try{y=(await h({attemptId:d,provider:e.provider,providerProduct:e.providerProduct,billingTier:e.billingTier,accountScope:r?.operationAccountScope??"production-default",method:e.method,providerStatus:"started"})).id}catch(f){throw await w({tool:e.tool,method:e.method,phase:"pending_receipt_create",errorCode:f instanceof Error?f.name:"UnknownError",runId:o,attemptId:d}).catch(()=>{}),await l({attemptId:d,status:"failed",errorCode:"pending_receipt_create_failed",durationMs:0}).catch(()=>{}),f}return{runId:o,attemptId:d,receiptId:y,ownsRun:n,startedAtMs:Date.now()}}async function S(e,r,n){await l({attemptId:e.attemptId,status:r,errorCode:n??null,durationMs:Date.now()-e.startedAtMs}),e.ownsRun&&await m({runId:e.runId,status:r,failureClass:n??null})}async function T(e,r){let n=s(),o=null,a=Date.now();if(n?.operationRunId&&n.operationRootAttemptId)try{o=await _({runId:n.operationRunId,parentAttemptId:n.operationRootAttemptId,edgeKind:"parallel_child",method:e})}catch(t){console.warn(JSON.stringify({event:"local_cost_phase_start_failed",method:e,message:t instanceof Error?t.message:String(t)}))}let i="failed";try{let t=await r();return i="succeeded",t}finally{o&&await l({attemptId:o,status:i,durationMs:Date.now()-a,errorCode:i==="failed"?"local_phase_failed":null}).catch(t=>console.warn(JSON.stringify({event:"local_cost_phase_finish_failed",method:e,message:t instanceof Error?t.message:String(t)})))}}async function M(e,r){let n=s()?.operationRunId;n&&await g({runId:n,billingTable:"billing_debits",billingKey:e,relation:"debit",amountMc:r}).catch(o=>console.warn(JSON.stringify({event:"cost_billing_link_failed",message:o instanceof Error?o.message:String(o)})))}async function K(e,r){let n=s();if(n?.operationRunId)return I({...n,op:e.op,userId:e.userId},r);let o=null,a=null;try{o=(await c({userId:e.userId,ownerScope:`user:${e.userId}`,tool:e.tool,normalizedFlags:e.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null})).id,e.billingDebitKey&&await g({runId:o,billingTable:"billing_debits",billingKey:e.billingDebitKey,relation:"debit"}).catch(t=>console.warn(JSON.stringify({event:"cost_billing_link_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))),a=await u({runId:o,attemptNumber:1,method:`${e.tool}_root`,normalizedFlags:e.normalizedFlags})}catch(t){console.warn(JSON.stringify({event:"logical_cost_run_start_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))}let i="failed";try{let t=await I({...n,op:e.op,userId:e.userId,operationRunId:o,operationRootAttemptId:a},r);return i=t instanceof Response&&t.status>=400?"failed":"succeeded",t}finally{o&&a?await v({runId:o,rootAttemptId:a,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(t=>console.warn(JSON.stringify({event:"logical_cost_run_finish_failed",tool:e.tool,message:t instanceof Error?t.message:String(t)}))):o&&await m({runId:o,status:i,failureClass:i==="failed"?"operation_failed":null}).catch(()=>{})}}async function O(e){let r=await c({...e.jobId?{id:`extract-url:${e.jobId}`,idempotencyKey:`job:${e.jobId}`,requestFingerprint:e.jobId,jobId:e.jobId}:{},userId:e.userId,ownerScope:`user:${e.userId}`,tool:"extract_url",normalizedFlags:e.normalizedFlags,deploymentCommit:process.env.VERCEL_GIT_COMMIT_SHA?.trim()||process.env.GIT_COMMIT_SHA?.trim()||null}),n=r.replayed?await p(r.id):null,o=n?.attempts[0]?.id==null?await u({runId:r.id,attemptNumber:1,method:"extract_url_root",normalizedFlags:e.normalizedFlags}):String(n.attempts[0].id);return e.billingDebitKey&&await g({runId:r.id,billingTable:"billing_debits",billingKey:e.billingDebitKey,relation:"debit"}),{runId:r.id,rootAttemptId:o}}async function v(e){let r=await p(e.runId),n=r?.attempts.find(o=>String(o.id)===e.rootAttemptId);n?.status==="running"&&await l({attemptId:e.rootAttemptId,status:e.status,errorCode:e.failureClass??null,durationMs:Math.max(0,Date.now()-Date.parse(String(n.started_at)))}),r?.run.status==="running"&&await m({runId:e.runId,status:e.status,failureClass:e.failureClass??null})}export{R as a,S as b,T as c,M as d,K as e,O as f,v as g};
|
package/dist/chunk-LMWU7VPZ.js
DELETED
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import{t as s}from"./chunk-7YOXSMHG.js";import{createHash as v,randomUUID as m}from"crypto";async function O(e){let r=m();return await s().execute({sql:`INSERT INTO operation_coverage_defects (id, tool, method, phase, error_code, run_id, attempt_id)
|
|
2
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,args:[r,a(e.tool,"tool"),a(e.method,"method"),a(e.phase,"phase"),a(e.errorCode,"errorCode").slice(0,200),e.runId??null,e.attemptId??null]}),r}var E=class extends Error{constructor(){super("This idempotency key was already used with different normalized inputs."),this.name="OperationIdempotencyConflictError"}};function u(e){return v("sha256").update(e).digest("hex")}function f(e){return Array.isArray(e)?e.map(f):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([r],[t])=>r.localeCompare(t)).map(([r,t])=>[r,f(t)])):e}function S(e){return JSON.stringify(f(e??{}))}function a(e,r){let t=e.trim();if(!t)throw new Error(`${r} is required`);return t}function l(e,r=2e3){return e==null?null:e.slice(0,r)}async function I(e){let r=s(),t=a(e.tool,"tool"),i=a(e.ownerScope,"ownerScope"),n=e.idempotencyKey?.trim()||null,o=e.requestFingerprint?.trim()||null;if(n==null!=(o==null))throw new Error("idempotencyKey and requestFingerprint must be supplied together");let d=n?u(`${i}\0${t}\0${n}`):null,c=e.id??m(),g=[c,e.userId??null,u(i),t,d,o,e.jobId??null,e.requestId??null,e.deploymentCommit??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()];if(d){await r.execute({sql:`INSERT OR IGNORE INTO operation_runs
|
|
3
|
-
(id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
|
|
4
|
-
job_id, request_id, deployment_commit, normalized_flags_json, started_at)
|
|
5
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:g});let p=(await r.execute({sql:"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?",args:[d]})).rows[0];if(!p)throw new Error("Operation run insert did not produce an idempotency receipt");if(String(p.request_fingerprint)!==o)throw new E;return{id:String(p.id),replayed:String(p.id)!==c}}return await r.execute({sql:`INSERT INTO operation_runs
|
|
6
|
-
(id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
|
|
7
|
-
job_id, request_id, deployment_commit, normalized_flags_json, started_at)
|
|
8
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:g}),{id:c,replayed:!1}}async function N(e){let r=e.endedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE operation_runs
|
|
9
|
-
SET status = ?, failure_class = ?, ended_at = ?, updated_at = ?
|
|
10
|
-
WHERE id = ? AND status = 'running'`,args:[e.status,l(e.failureClass,200),r,r,e.runId]})).rowsAffected===1)return;let i=l(e.failureClass,200),n=await s().execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]});if(n.rows.length===0)throw new Error(`Unknown operation run: ${e.runId}`);let o=n.rows[0]?.failure_class==null?null:String(n.rows[0]?.failure_class);if(String(n.rows[0]?.status)!==e.status||o!==i)throw new Error(`Operation run ${e.runId} is already terminal as ${String(n.rows[0]?.status)}`)}async function A(e){if(!Number.isInteger(e.attemptNumber)||e.attemptNumber<1)throw new Error("attemptNumber must be a positive integer");if(e.attemptNumber===1&&(e.parentAttemptId||e.edgeKind||e.retryKind))throw new Error("The first attempt cannot have a parent or retry edge");if(e.attemptNumber>1&&(!e.parentAttemptId||!e.edgeKind))throw new Error("Every non-first attempt requires a parentAttemptId and edgeKind");if(e.retryKind&&!["same_method_retry","provider_fallback","durable_resume","reconciliation"].includes(e.edgeKind??""))throw new Error("retryKind requires a retry or reconciliation edgeKind");let r=e.id??m();return await s().execute({sql:`INSERT INTO operation_attempts
|
|
11
|
-
(id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason, retry_owner,
|
|
12
|
-
method, provider, normalized_flags_json, started_at)
|
|
13
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[r,e.runId,e.parentAttemptId??null,e.attemptNumber,e.edgeKind??null,e.retryKind??null,l(e.retryReason,500),l(e.retryOwner,100),a(e.method,"method"),e.provider??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()]}),r}async function T(e){let r=m();return await s().execute({sql:`INSERT INTO operation_attempts
|
|
14
|
-
(id, run_id, parent_attempt_id, attempt_number, edge_kind, retry_kind, retry_reason,
|
|
15
|
-
method, provider, normalized_flags_json, started_at)
|
|
16
|
-
SELECT ?, ?, ?, COALESCE(MAX(attempt_number), 0) + 1, ?, ?, ?, ?, ?, ?, ?
|
|
17
|
-
FROM operation_attempts WHERE run_id = ?`,args:[r,e.runId,e.parentAttemptId,e.edgeKind,e.retryKind??null,l(e.retryReason,500),a(e.method,"method"),e.provider??null,S(e.normalizedFlags),new Date().toISOString(),e.runId]}),r}async function C(e){if(!Number.isInteger(e.durationMs)||e.durationMs<0)throw new Error("durationMs must be a non-negative integer");let r=e.endedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE operation_attempts SET
|
|
18
|
-
status = ?, error_code = ?, error_message = ?, ended_at = ?, duration_ms = ?, updated_at = ?
|
|
19
|
-
WHERE id = ? AND status = 'running'`,args:[e.status,l(e.errorCode,200),l(e.errorMessage),r,e.durationMs,r,e.attemptId]})).rowsAffected===1)return;let i=await s().execute({sql:"SELECT status, duration_ms FROM operation_attempts WHERE id = ?",args:[e.attemptId]});if(i.rows.length===0)throw new Error(`Unknown operation attempt: ${e.attemptId}`);if(String(i.rows[0]?.status)!==e.status||Number(i.rows[0]?.duration_ms)!==e.durationMs)throw new Error(`Operation attempt ${e.attemptId} is already terminal as ${String(i.rows[0]?.status)}`)}async function L(e){let r=a(e.provider,"provider"),t=a(e.method,"method"),i=u(a(e.accountScope,"accountScope")),n=e.providerSourceKey?.trim()||null,o=n?u(n):null,d=e.id??m(),c=await s().execute({sql:`INSERT OR IGNORE INTO provider_usage_receipts
|
|
20
|
-
(id, attempt_id, provider, account_scope_hash, method, provider_product, billing_tier,
|
|
21
|
-
provider_source_key, provider_source_key_hash, provider_status)
|
|
22
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:[d,e.attemptId,r,i,t,e.providerProduct??null,e.billingTier??null,n,o,e.providerStatus??null]});if(!o){if(c.rowsAffected===1)return{id:d,replayed:!1};let w=(await s().execute({sql:"SELECT attempt_id, provider, method FROM provider_usage_receipts WHERE id = ?",args:[d]})).rows[0];if(!w||String(w.attempt_id)!==e.attemptId||String(w.provider)!==r||String(w.method)!==t)throw new Error("Provider receipt ID is already used by a different receipt");return{id:d,replayed:!0}}let _=(await s().execute({sql:`SELECT id, attempt_id FROM provider_usage_receipts
|
|
23
|
-
WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[r,i,t,o]})).rows[0];if(!_)throw new Error("Provider receipt insert did not produce a source receipt");if(String(_.attempt_id)!==e.attemptId)throw new Error("Provider source identity is already attached to a different attempt");return{id:String(_.id),replayed:String(_.id)!==d}}async function x(e,r){let t=a(r,"providerSourceKey"),i=u(t);if((await s().execute({sql:`UPDATE provider_usage_receipts
|
|
24
|
-
SET provider_source_key = ?, provider_source_key_hash = ?, updated_at = datetime('now')
|
|
25
|
-
WHERE id = ? AND provider_source_key IS NULL`,args:[t,i,e]})).rowsAffected===1)return;let o=await s().execute({sql:"SELECT provider_source_key_hash FROM provider_usage_receipts WHERE id = ?",args:[e]});if(!o.rows[0])throw new Error(`Unknown provider receipt: ${e}`);if(String(o.rows[0].provider_source_key_hash)!==i)throw new Error(`Provider receipt ${e} already has a different source identity`)}async function h(e){if(!Number.isSafeInteger(e.units)||e.units<0)throw new Error("Observed units must be a non-negative safe integer");if((await s().execute({sql:`UPDATE provider_usage_receipts SET units = ?, unit_type = ?,
|
|
26
|
-
provider_status = COALESCE(?, provider_status), checked_at = ?, updated_at = datetime('now')
|
|
27
|
-
WHERE id = ? AND cost_state = 'pending'`,args:[e.units,a(e.unitType,"unitType"),e.providerStatus??null,e.checkedAt??new Date().toISOString(),e.receiptId]})).rowsAffected!==1)throw new Error(`Unknown or terminal pending provider receipt: ${e.receiptId}`)}async function k(e){let t=(await s().execute({sql:`SELECT id FROM provider_usage_receipts
|
|
28
|
-
WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),u(a(e.accountScope,"accountScope")),a(e.method,"method"),u(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];return t?(await h({receiptId:String(t.id),units:e.units,unitType:e.unitType,providerStatus:e.providerStatus,checkedAt:e.checkedAt}),!0):!1}async function y(e){if(e.units!=null&&(!Number.isFinite(e.units)||e.units<0))throw new Error("units must be non-negative");if(e.amountUsdNanos!=null&&(!Number.isSafeInteger(e.amountUsdNanos)||e.amountUsdNanos<0))throw new Error("amountUsdNanos must be a non-negative safe integer");if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos==null)throw new Error(`${e.costState} receipts require amountUsdNanos`);if((e.costState==="actual"||e.costState==="measured")&&e.amountUsdNanos===0&&!e.zeroCostReason?.trim())throw new Error(`${e.costState} zero-cost receipts require zeroCostReason`);if(e.costState==="measured"&&e.amountUsdNanos!==0&&(!e.providerProduct||!e.billingTier||!e.billableEvent||!e.measurementSource||!e.rateSource||e.rateNumerator==null||e.rateDenominator==null||!e.ratePolicyVersion||!e.rateEffectiveAt||e.units==null||!e.unitType))throw new Error("measured receipts require observed product, units, and an applied rate snapshot");if(e.rateNumerator!=null&&(!Number.isSafeInteger(e.rateNumerator)||e.rateNumerator<0))throw new Error("rateNumerator must be a non-negative safe integer");if(e.rateDenominator!=null&&(!Number.isSafeInteger(e.rateDenominator)||e.rateDenominator<1))throw new Error("rateDenominator must be a positive safe integer");let r=e.checkedAt??new Date().toISOString(),t=e.finalizedAt??r;if((await s().execute({sql:`UPDATE provider_usage_receipts SET
|
|
29
|
-
cost_state = ?, units = COALESCE(?, units), unit_type = COALESCE(?, unit_type),
|
|
30
|
-
amount_usd_nanos = ?, rate_source = COALESCE(rate_source, ?), zero_cost_reason = COALESCE(?, zero_cost_reason),
|
|
31
|
-
provider_product = COALESCE(provider_product, ?), billing_tier = COALESCE(billing_tier, ?),
|
|
32
|
-
billable_event = COALESCE(billable_event, ?), measurement_source = COALESCE(measurement_source, ?),
|
|
33
|
-
rate_numerator = COALESCE(rate_numerator, ?), rate_denominator = COALESCE(rate_denominator, ?),
|
|
34
|
-
rate_policy_version = COALESCE(rate_policy_version, ?), rate_effective_at = COALESCE(rate_effective_at, ?),
|
|
35
|
-
provider_status = ?, provider_duration_ms = ?, provider_captcha = ?, error_code = ?, error_message = ?,
|
|
36
|
-
checked_at = ?, finalized_at = ?, updated_at = ?
|
|
37
|
-
WHERE id = ? AND (cost_state = 'pending'
|
|
38
|
-
OR (cost_state = 'estimated' AND ? IN ('measured', 'actual'))
|
|
39
|
-
OR (cost_state = 'measured' AND ? = 'actual'))`,args:[e.costState,e.units??null,e.unitType??null,e.amountUsdNanos??null,e.rateSource??null,e.zeroCostReason??null,e.providerProduct??null,e.billingTier??null,e.billableEvent??null,e.measurementSource??null,e.rateNumerator??null,e.rateDenominator??null,e.ratePolicyVersion??null,e.rateEffectiveAt??null,e.providerStatus??null,e.providerDurationMs??null,e.providerCaptcha==null?null:e.providerCaptcha?1:0,l(e.errorCode,200),l(e.errorMessage),r,t,t,e.receiptId,e.costState,e.costState]})).rowsAffected===1)return;let n=await s().execute({sql:"SELECT cost_state, units, unit_type, amount_usd_nanos FROM provider_usage_receipts WHERE id = ?",args:[e.receiptId]});if(n.rows.length===0)throw new Error(`Unknown provider receipt: ${e.receiptId}`);let o=n.rows[0],d=o?.units==null&&e.units==null||Number(o?.units)===e.units,c=o?.amount_usd_nanos==null&&e.amountUsdNanos==null||Number(o?.amount_usd_nanos)===e.amountUsdNanos,g=o?.unit_type==null&&e.unitType==null||String(o?.unit_type)===e.unitType;if(String(o?.cost_state)!==e.costState||!d||!c||!g)throw new Error(`Provider receipt ${e.receiptId} is already finalized as ${String(n.rows[0]?.cost_state)}`)}async function U(e){let t=(await s().execute({sql:`SELECT id FROM provider_usage_receipts
|
|
40
|
-
WHERE provider = ? AND account_scope_hash = ? AND method = ? AND provider_source_key_hash = ?`,args:[a(e.provider,"provider"),u(a(e.accountScope,"accountScope")),a(e.method,"method"),u(a(e.providerSourceKey,"providerSourceKey"))]})).rows[0];if(!t)return!1;let{provider:i,accountScope:n,method:o,providerSourceKey:d,...c}=e;return await y({...c,receiptId:String(t.id)}),!0}async function P(e=30){if(!Number.isInteger(e)||e<1||e>365)throw new Error("retentionDays must be an integer between 1 and 365");let r=`-${e} days`,t=s(),[i,n,o]=await Promise.all([t.execute({sql:`UPDATE operation_attempts
|
|
41
|
-
SET error_message = NULL, updated_at = datetime('now')
|
|
42
|
-
WHERE ended_at < datetime('now', ?) AND error_message IS NOT NULL`,args:[r]}),t.execute({sql:`UPDATE provider_usage_receipts
|
|
43
|
-
SET provider_source_key = NULL, error_message = NULL, updated_at = datetime('now')
|
|
44
|
-
WHERE finalized_at < datetime('now', ?)
|
|
45
|
-
AND (provider_source_key IS NOT NULL OR error_message IS NOT NULL)`,args:[r]}),t.execute({sql:`UPDATE harvest_attempts
|
|
46
|
-
SET provider_session_id = NULL, \u006b\u0065\u0072\u006e\u0065\u006c_session_id = NULL, error = NULL,
|
|
47
|
-
\u006b\u0065\u0072\u006e\u0065\u006c_delete_error = NULL, browser_close_error = NULL,
|
|
48
|
-
provider_error_message = NULL, provider_detail_json = NULL,
|
|
49
|
-
provider_disconnect_message = NULL, debug_json = NULL
|
|
50
|
-
WHERE completed_at < datetime('now', ?)
|
|
51
|
-
AND (provider_session_id IS NOT NULL OR \u006b\u0065\u0072\u006e\u0065\u006c_session_id IS NOT NULL OR error IS NOT NULL
|
|
52
|
-
OR \u006b\u0065\u0072\u006e\u0065\u006c_delete_error IS NOT NULL OR browser_close_error IS NOT NULL
|
|
53
|
-
OR provider_error_message IS NOT NULL OR provider_detail_json IS NOT NULL
|
|
54
|
-
OR provider_disconnect_message IS NOT NULL OR debug_json IS NOT NULL)`,args:[r]})]);return{operationAttempts:i.rowsAffected,providerReceipts:n.rowsAffected,legacyHarvestAttempts:o.rowsAffected}}async function D(e){let t={billing_authorizations:{column:"id"},billing_events:{column:"id"},billing_debits:{column:"idempotency_key"},billing_credits:{column:"idempotency_key"},ledger:{column:"id"}}[e.billingTable],i=a(e.billingKey,"billingKey"),n=await s().execute({sql:`SELECT b.user_id AS billing_user_id, r.user_id AS run_user_id
|
|
55
|
-
FROM operation_runs r
|
|
56
|
-
JOIN ${e.billingTable} b ON CAST(b.${t.column} AS TEXT) = ?
|
|
57
|
-
WHERE r.id = ?`,args:[i,e.runId]});if(!n.rows[0])throw new Error("Billing link target or operation run does not exist");let o=n.rows[0].billing_user_id,d=n.rows[0].run_user_id;if(d!=null&&o!=null&&String(d)!==String(o))throw new Error("Billing link owner does not match operation run owner");await s().execute({sql:`INSERT OR IGNORE INTO operation_billing_links
|
|
58
|
-
(run_id, billing_table, billing_key, relation, amount_mc) VALUES (?, ?, ?, ?, ?)`,args:[e.runId,e.billingTable,i,e.relation,e.amountMc??null]})}async function q(e,r=new Date().toISOString()){let t=m();return await s().execute({sql:"INSERT INTO provider_reconciliation_runs (id, provider, started_at) VALUES (?, ?, ?)",args:[t,a(e,"provider"),r]}),t}async function K(e){if([e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount].some(n=>!Number.isInteger(n)||n<0))throw new Error("Reconciliation counts must be non-negative integers");if(e.completedCount+e.pendingCount+e.unavailableCount+e.failedCount>e.checkedCount)throw new Error("Reconciliation outcome counts cannot exceed checkedCount");let t=e.completedAt??new Date().toISOString();if((await s().execute({sql:`UPDATE provider_reconciliation_runs SET
|
|
59
|
-
status = ?, checked_count = ?, completed_count = ?, pending_count = ?, unavailable_count = ?,
|
|
60
|
-
failed_count = ?, error_code = ?, completed_at = ?
|
|
61
|
-
WHERE id = ? AND status = 'running'`,args:[e.status,e.checkedCount,e.completedCount,e.pendingCount,e.unavailableCount,e.failedCount,l(e.errorCode,200),t,e.id]})).rowsAffected!==1)throw new Error(`Unknown or terminal provider reconciliation run: ${e.id}`)}async function F(e){let r=s(),[t,i,n,o]=await Promise.all([r.execute({sql:"SELECT * FROM operation_runs WHERE id = ?",args:[e]}),r.execute({sql:"SELECT * FROM operation_attempts WHERE run_id = ? ORDER BY attempt_number",args:[e]}),r.execute({sql:`SELECT r.* FROM provider_usage_receipts r
|
|
62
|
-
JOIN operation_attempts a ON a.id = r.attempt_id
|
|
63
|
-
WHERE a.run_id = ? ORDER BY a.attempt_number, r.rowid`,args:[e]}),r.execute({sql:"SELECT * FROM operation_billing_links WHERE run_id = ? ORDER BY created_at, billing_table, billing_key",args:[e]})]);return t.rows[0]?{run:t.rows[0],attempts:[...i.rows],providerReceipts:[...n.rows],billingLinks:[...o.rows]}:null}export{O as a,I as b,N as c,A as d,T as e,C as f,L as g,x as h,h as i,k as j,y as k,U as l,P as m,D as n,q as o,K as p,F as q};
|
package/dist/chunk-O4LUUIYG.js
DELETED
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import{A as f,Ga as T,Ia as h,Ja as w,Ka as _,La as y,Ma as C,ga as g,ia as S,va as R,wa as E,ya as v,z as u}from"./chunk-GLTECY3U.js";import{a as P}from"./chunk-K3D2S36Q.js";import{a as m}from"./chunk-VXOXB3IB.js";import{readFileSync as N}from"fs";import{homedir as I}from"os";import{join as A}from"path";import{serveStdio as b}from"@modelcontextprotocol/server/stdio";import{McpServer as k}from"@modelcontextprotocol/server";var x=new Map(w.map(r=>[r.upstreamName,r.id]));function O(r){let s=r.split(`
|
|
2
|
-
`).filter(o=>o.startsWith("data:")).map(o=>o.slice(5).trim()).filter(Boolean);if(!s.length)return JSON.parse(r);for(let o of s){let e=JSON.parse(o);if(e.result||e.error)return e}throw new Error("hosted MCP returned no JSON-RPC result")}function a(r){return{content:[{type:"text",text:JSON.stringify({ok:!1,error:r})}],isError:!0}}var c=class{baseUrl;apiKey;constructor(s,o){this.baseUrl=s.replace(/\/$/,""),this.apiKey=o}async callMemoryTool(s,o){let e=x.get(s);if(!e)return a(`unknown memory tool: ${s}`);try{let n=await fetch(`${this.baseUrl}/mcp`,{method:"POST",headers:{"content-type":"application/json",accept:"application/json, text/event-stream","x-api-key":this.apiKey},body:JSON.stringify({jsonrpc:"2.0",id:`memory:${e}`,method:"tools/call",params:{name:e,arguments:o}})}),l=await n.text();if(!n.ok)return a(`hosted memory ${e} failed (HTTP ${n.status})`);let t=O(l);return t.error?a(t.error.message??`hosted memory ${e} failed`):t.result??a(`hosted memory ${e} returned no result`)}catch(n){return a(n instanceof Error?n.message:`hosted memory ${e} call failed`)}}};function M(r,s){if(s.length===0)throw new Error("Restricted MCP tool allowlist must contain at least one tool name");let o=new Set;for(let t of s){if(typeof t!="string"||t.trim()!==t||t.length===0)throw new Error("Restricted MCP tool allowlist contains a malformed tool name");if(o.has(t))throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${t}`);o.add(t)}let e=new Set,n=r,l=n.registerTool.bind(r);return n.registerTool=(t,i,p)=>{if(o.has(t))return e.add(t),l(t,i,p)},{assertComplete(){let t=[...o].filter(i=>!e.has(i));if(t.length>0)throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${t.join(", ")}`)},registeredToolNames(){return[...e]}}}var L=new Set(["paa","serp","browser-agent","scheduled-results","memory"]);function $(){let s=[process.env.MCP_SCRAPER_KEY_PATH?.trim(),A(I(),".mcp-scraper-key")].filter(Boolean);for(let o of s)try{let e=N(o,"utf8").trim();if(e)return e}catch{}}function Y(r,s={}){let o=s.toolsets??L,e=process.env.MCP_SCRAPER_BASE_URL?.trim()||process.env.MCP_BASE_URL?.trim()||"https://mcpscraper.dev",n=g(),l=S({deploymentProfile:n,transportProfile:"stdio",baseUrl:e,explicitlyEnabled:process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK==="1"}),t=process.env.BROWSER_AGENT_CONSOLE_URL?.trim()||e,i=new k({name:"mcp-scraper",version:m},{instructions:u,cacheHints:{"server/discover":{ttlMs:3e5,cacheScope:"private"},"tools/list":{ttlMs:3e5,cacheScope:"private"},"prompts/list":{ttlMs:3e5,cacheScope:"private"},"resources/list":{ttlMs:3e5,cacheScope:"private"},"resources/templates/list":{ttlMs:3e5,cacheScope:"private"},"resources/read":{ttlMs:6e4,cacheScope:"private"}}});f(i);let p=s.allowedToolNames?M(i,s.allowedToolNames):void 0,d=s.httpExecutor??new T(e,r,{localNetworkAccess:l});return o.has("paa")&&v(i,d,{ownerId:R(r),deploymentProfile:n,transportProfile:"stdio",baseUrl:e,localNetworkAccess:l,taskHandleSecret:r}),o.has("serp")&&E(i,d,{exposeDevelopmentDiagnostics:n==="development"||n==="test"}),o.has("browser-agent")&&h(i,{baseUrl:e,apiKey:r,consoleBaseUrl:t}),o.has("scheduled-results")&&C(i,new y(e,r)),o.has("memory")&&_(i,new c(e,r)),p?.assertComplete(),i}function se(r={}){let s=process.argv.includes("--stdio")||process.env.MCP_SCRAPER_FORCE_STDIO==="1",o=!!(process.stdin.isTTY&&process.stdout.isTTY),e=process.argv.includes("--help")||process.argv.includes("-h");if(!s&&(o||e)){let l=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY;process.stdout.write(P({version:m,color:!l,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()})),process.exit(0)}let n=(process.env.MCP_SCRAPER_API_KEY??process.env.MCP_SCRAPER_KEY??process.env.MCP_API_KEY??$())?.trim();n||(process.stderr.write(`MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required
|
|
3
|
-
`),process.exit(1)),b(()=>Y(n,r),{legacy:"serve",onerror(l){process.stderr.write(`${l.message}
|
|
4
|
-
`)}})}export{se as a};
|
package/dist/chunk-OLSW4AN4.js
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import{f as F,h as V,i as re,j as ae,l as $}from"./chunk-T4VTJNDZ.js";import{B as ie,ba as R,e as U,t as _,w as G,x as ne,y as oe,z as M}from"./chunk-EDQ57S4P.js";import{h as L}from"./chunk-VREXSMC6.js";import{b as K}from"./chunk-6DTXIZY2.js";import{R as te,Sb as D,Xb as x,t as C}from"./chunk-7YOXSMHG.js";import{createHash as Te,randomUUID as Ne}from"crypto";import{z as f}from"zod";async function at(e,t){!Number.isFinite(t)||t<=0||await K({op:R.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>L({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function se(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await K({op:R.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let o of n)!Number.isFinite(o.units)||o.units<=0||await L(o)})}var E="connected_usage",Re="mastra_scheduler",Ie=f.object({toolName:f.string().trim().min(1).max(200).optional(),operationKind:f.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:f.enum(["success","error","partial"]).optional(),requestId:f.string().trim().min(1).max(200).optional(),occurrenceId:f.string().trim().min(1).max(200).optional(),scheduleId:f.string().trim().min(1).max(200).optional(),nangoOperationId:f.string().trim().min(1).max(200).optional(),startedAt:f.string().datetime({offset:!0}).optional(),completedAt:f.string().datetime({offset:!0}).optional()}).strict(),Ae=f.object({identity:f.string().trim().min(1).max(320),idempotencyKey:f.string().trim().min(1).max(500),ratePolicyVersion:f.string().trim().min(1).max(100),connectionId:f.string().trim().min(1).max(500),providerConfigKey:f.string().trim().min(1).max(200),functionRuns:f.number().int().safe().nonnegative(),proxyRequests:f.number().int().safe().nonnegative(),computeMilliseconds:f.number().int().safe().nonnegative(),sourceSurface:f.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Re),metadata:Ie.optional()}).strict(),Pe=f.object({identity:f.string().trim().min(1).max(320),ratePolicyVersion:f.string().trim().min(1).max(100)}).strict(),h=class extends Error{constructor(n,o,a,i={}){super(o);this.code=n;this.httpStatus=a;this.details=i;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function z(e){return Te("sha256").update(e).digest("hex")}function xe(e){return`connected-usage:${z(e)}`}function Ue(e){return`connected-usage-debit:${z(e)}`}function Me(e){return e.trim().toLowerCase()}async function ce(e){let t=await te(Me(e));if(!t)throw new h("user_not_found","billing identity was not found",404);return t}function Oe(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return z(JSON.stringify({ratePolicyVersion:_,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function A(e){let t=JSON.stringify(e);if(t.length>4e3)throw new h("invalid_request","connected usage metadata is too large",400);return t}function W(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==_||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new h("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function H(e){let t=await C().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function de(e,t){return(await C().execute({sql:`SELECT 1
|
|
2
|
-
FROM billing_events
|
|
3
|
-
WHERE user_id = ?
|
|
4
|
-
AND billing_class = ?
|
|
5
|
-
AND status = 'cost_pending'
|
|
6
|
-
${t?"AND idempotency_key <> ?":""}
|
|
7
|
-
LIMIT 1`,args:t?[e,E,t]:[e,E]})).rows.length>0}function ke(e){let t="connected-usage:",n=e.startsWith(t)?e.slice(t.length):"";return/^[a-f0-9]{64}$/.test(n)?`connected-usage-debit:${n}`:null}async function qe(e){let t=await C().execute({sql:`SELECT id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata
|
|
8
|
-
FROM billing_events
|
|
9
|
-
WHERE user_id = ? AND billing_class = ? AND status = 'cost_pending'
|
|
10
|
-
ORDER BY created_at ASC
|
|
11
|
-
LIMIT 10`,args:[e.id,E]});for(let n of t.rows){let o=n,a=String(o.idempotency_key??""),i=ke(a),s;try{s=W(o)}catch{return}if(!i||Number(o.user_id)!==e.id||o.billing_class!==E||Number(o.amount_mc)!==s.charge.amountMc||s.charge.amountMc>0&&!(await D(e.id,s.charge.amountMc,R.CONNECTED_USAGE,`connected-usage:${s.providerConfigKey}`,i)).ok)return;let r={...s};delete r.reason,await C().execute({sql:`UPDATE billing_events
|
|
12
|
-
SET status = 'settled', metadata = ?, settled_at = datetime('now')
|
|
13
|
-
WHERE idempotency_key = ? AND user_id = ? AND billing_class = ? AND status = 'cost_pending'`,args:[A(r),a,e.id,E]})}}function ue(e){if(e!==_)throw new h("rate_policy_mismatch","connected usage receipt rate policy does not match the active policy",409,{expectedRatePolicyVersion:_})}async function Y(e){let t=Pe.safeParse(e);if(!t.success)throw new h("invalid_request","invalid connected usage preflight request",400);ue(t.data.ratePolicyVersion);let n=await ce(t.data.identity);if(await qe(n),await de(n.id))throw new h("pending_cost","a previous connected usage receipt has unresolved cost",409);let o=await x(n.id,n.balance_mc);if(o<M)throw new h("insufficient_credits","the wallet cannot cover the minimum connected function charge",402,{balanceMc:o,requiredMc:M});return{allowed:!0,ratePolicyVersion:_,minimumMc:M,minimumCredits:G,balanceMc:o}}function B(e,t,n,o,a){let i=W(e);if(Number(e.user_id)!==t||e.billing_class!==E||e.source_surface!==n||Number(e.amount_mc)!==a||i.fingerprint!==o)throw new h("idempotency_key_conflict","the idempotency key is already bound to different connected usage",409);return i}function le(e,t){return[{key:"function_runs",label:"Connected function runs",quantity:e.functionRuns,quantityUnit:"function_run",rateCredits:G,rateUnit:"function_run",amountMc:t.functionRunMc,credits:t.functionRunMc/U},{key:"proxy_requests",label:"Connected proxy requests",quantity:e.proxyRequests,quantityUnit:"proxy_request",rateCredits:ne,rateUnit:"proxy_request",amountMc:t.proxyRequestMc,credits:t.proxyRequestMc/U},{key:"compute",label:"Connected function compute",quantity:e.computeMilliseconds,quantityUnit:"millisecond",rateCredits:oe,rateUnit:"compute_second",amountMc:t.computeMc,credits:t.computeMc/U}]}function J(e,t,n,o){let a=e.status==="cost_pending"?t.reason:void 0;return{id:e.id,status:e.status==="settled"?"settled":"cost_pending",ratePolicyVersion:_,connectionId:t.connectionId,providerConfigKey:t.providerConfigKey,amountMc:t.charge.amountMc,credits:t.charge.credits,functionRunMc:t.charge.functionRunMc,proxyRequestMc:t.charge.proxyRequestMc,computeMc:t.charge.computeMc,usage:t.usage,costBreakdown:le(t.usage,t.charge),balanceMc:n,...a?{reason:a}:{},duplicate:o}}async function j(e){let t=Ae.safeParse(e);if(!t.success)throw new h("invalid_request","invalid connected usage settlement request",400,{fields:[...new Set(t.error.issues.map(v=>v.path.join(".")).filter(Boolean))]});let n=t.data;ue(n.ratePolicyVersion);let o=await ce(n.identity),a={functionRuns:n.functionRuns,proxyRequests:n.proxyRequests,computeMilliseconds:n.computeMilliseconds},i=ie(a),s=n.metadata??{},r=Oe({connectionId:n.connectionId,providerConfigKey:n.providerConfigKey,usage:a,sourceSurface:n.sourceSurface,metadata:s}),u={contract:"connected_usage_receipt",ratePolicyVersion:_,fingerprint:r,connectionId:n.connectionId,providerConfigKey:n.providerConfigKey,usage:a,charge:i,safeMetadata:s},p=xe(n.idempotencyKey);if(await de(o.id,p))throw new h("pending_cost","a previous connected usage receipt has unresolved cost",409);let d=await C().execute({sql:`INSERT OR IGNORE INTO billing_events
|
|
14
|
-
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
15
|
-
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,args:[Ne(),o.id,p,E,n.sourceSurface,i.amountMc,A(u)]}),g=await H(p);if(!g)throw new Error("connected usage receipt insert completed without a readable event");let y=B(g,o.id,n.sourceSurface,r,i.amountMc),b=d.rowsAffected===0;if(g.status==="settled")return J(g,y,await x(o.id,o.balance_mc),!0);if(!["settling","cost_pending"].includes(g.status))throw new h("idempotency_key_conflict","the connected usage receipt has an incompatible status",409);if(g.status==="cost_pending"&&await C().execute({sql:"UPDATE billing_events SET status = 'settling', metadata = ? WHERE idempotency_key = ? AND status = 'cost_pending'",args:[A(u),p]}),i.amountMc===0)await C().execute({sql:"UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE idempotency_key = ? AND status = 'settling'",args:[p]});else{let v=await D(o.id,i.amountMc,R.CONNECTED_USAGE,`connected-usage:${n.providerConfigKey}`,Ue(n.idempotencyKey));if(!v.ok){let Se={...u,reason:"insufficient_credits"};if(await C().execute({sql:"UPDATE billing_events SET status = 'cost_pending', metadata = ? WHERE idempotency_key = ? AND status = 'settling'",args:[A(Se),p]}),g=await H(p),!g)throw new Error("connected usage pending receipt was lost");return y=B(g,o.id,n.sourceSurface,r,i.amountMc),J(g,y,v.balance_mc,b||v.duplicate)}await C().execute({sql:"UPDATE billing_events SET status = 'settled', metadata = ?, settled_at = datetime('now') WHERE idempotency_key = ? AND status IN ('settling', 'cost_pending')",args:[A(u),p]}),v.duplicate||await se(o.id,a)}if(g=await H(p),!g)throw new Error("connected usage settlement completed without an event receipt");return y=B(g,o.id,n.sourceSurface,r,i.amountMc),J(g,y,await x(o.id,o.balance_mc),b)}async function pt(e,t=100){let n=await C().execute({sql:`SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata, created_at
|
|
16
|
-
FROM billing_events
|
|
17
|
-
WHERE user_id = ? AND billing_class = ? AND status IN ('settled', 'cost_pending')
|
|
18
|
-
ORDER BY created_at DESC
|
|
19
|
-
LIMIT ?`,args:[e,E,Math.max(1,Math.min(250,Math.round(t)))]}),o=[];for(let a of n.rows){let i=a;try{let s=W(i);o.push({id:i.id,createdAt:String(i.created_at??""),status:i.status==="settled"?"settled":"cost_pending",connectionId:s.connectionId,providerConfigKey:s.providerConfigKey,amountMc:s.charge.amountMc,credits:s.charge.credits,usage:s.usage,costBreakdown:le(s.usage,s.charge),metadata:s.safeMetadata})}catch{}}return o}import{createHash as I,randomUUID as ge}from"crypto";import{Client as De,StreamableHTTPClientTransport as Ke}from"@modelcontextprotocol/client";function Q(e){return e.annotations?.readOnlyHint===!0?"read":e.annotations?.readOnlyHint===!1?"action":/^(get|list|search|query|read|download|fetch|find|lookup|inspect|describe|check)-/i.test(e.name)?"read":"action"}function me(e){return new Map(Object.entries(e??{}).map(([t,n])=>[t,[...new Set(n.map(o=>o.trim()).filter(Boolean))].sort()]))}function O(e){let t=me(e.requiredPermissionsByTool),n=me(e.requiredFeaturesByTool),o=new Set(e.grantedPermissions??[]),a=new Set(e.enabledFeatures??[]),i=new Set(e.adminBlockedTools??[]),s=new Map;for(let r of e.tools){let u=Q(r),p=t.get(r.name)??[],d=n.get(r.name)??[],g=p.filter(v=>!o.has(v)),y=d.filter(v=>!a.has(v)),b=e.connectionActive?i.has(r.name)?"admin_blocked":u==="action"&&!e.actionsEnabled?"actions_disabled":p.length>0&&e.permissionVerification==="unavailable"?"permission_verification_unavailable":g.length>0?"missing_permission":y.length>0?"missing_app_feature":null:"inactive_connection";s.set(r.name,{name:r.name,classification:u,requiredPermissions:p,requiredFeatures:d,available:b===null,blockedReason:b,missingPermissions:g,missingFeatures:y})}return s}function Z(e,t){let n=e.get(t);if(!n)throw new Error("connected_tool_policy_missing");return n}var Le="https://api.nango.dev/mcp",Ge=2e4,Fe=24e4,pe=100,Ve=20,$e=360*60*1e3,He=new Set(["stripe_revenue","salesforce","pipedrive"]),Be="2026-07-16.1",Je="f037859ec80c2ea2fac6e4a67c1a59f3200dd22d3642c97789b9bb3236723d3e",m=class extends Error{status;code;retryable;retryAfterMs;constructor(t,n,o,a=n===429||n>=500,i=null){super(t),this.name="MainNangoTransportError",this.status=n,this.code=o,this.retryable=a,this.retryAfterMs=i}};function k(e){return typeof e=="string"&&He.has(e.normalize("NFKC").trim().toLowerCase())}function fe(e){if(k(e))throw new m("This provider integration is not currently available.",409,"provider_unavailable",!1)}function ze(e){if(e instanceof Error)return e.message;if(typeof e=="string")return e;try{return JSON.stringify(e)}catch{return""}}function ye(e){let t=e.match(/retry[_ -]?after[_ -]?ms[^0-9]{0,20}(\d{1,7})/i);if(t)return Math.min(Math.max(Number(t[1]),1e3),6e4);let n=e.match(/retry[_ -]?after[^0-9]{0,20}(\d{1,4})/i);return n?Math.min(Math.max(Number(n[1]),1)*1e3,6e4):null}function _e(e){if(e instanceof m)return e;let t=ze(e),n=e&&typeof e=="object"?e:null;return n?.code===-32001||n?.name==="RequestTimeout"||/Request timed out|RequestTimeout|Maximum total timeout exceeded/i.test(t)?new m("The connected service operation timed out before returning a result. Retry failed items or use a smaller batch.",503,"mcp_request_timeout",!0):/SERVICE_DISABLED|accessNotConfigured/i.test(t)?new m("The provider API required by this connection is disabled. Enable it in the provider project, wait for propagation, then retry.",409,"provider_api_disabled",!1):/PERMISSION_DENIED|insufficientPermissions|insufficient permission/i.test(t)?new m("The connected account does not have permission to run this provider operation.",403,"missing_provider_permission",!1):/RESOURCE_EXHAUSTED|rate.?limit|status["']?\s*:\s*429/i.test(t)?new m("The connected provider is temporarily rate limited.",429,"upstream_rate_limited",!0,ye(t)):/UNAUTHENTICATED|invalid_grant|status["']?\s*:\s*401/i.test(t)?new m("The service connection requires reconnection.",409,"connection_inactive",!1):new m("The service connection is temporarily unavailable.",503,"connection_transport_unavailable")}async function We(e,t,n){return e.callTool({name:t,arguments:n},{timeout:Fe})}function l(e){return e&&typeof e=="object"&&!Array.isArray(e)?e:null}function c(e){return typeof e=="string"&&e.trim()?e.trim():null}function X(e,t){if(Array.isArray(e))return e;let n=l(e);if(Array.isArray(n?.[t]))return n[t];if(Array.isArray(n?.data))return n.data;let o=l(n?.data);return Array.isArray(o?.[t])?o[t]:[]}function he(){let e=process.env.NANGO_SECRET_KEY?.trim();if(!e)throw new m("Nango is not configured.",503,"nango_not_configured");return e}function we(){try{return new URL(process.env.NANGO_MCP_URL?.trim()||Le)}catch{throw new m("Nango MCP URL is invalid.",503,"nango_invalid_url")}}async function N(e,t={}){let n=new URL(e,`${we().origin}/`);for(let[s,r]of Object.entries(t.query??{}))r!==void 0&&n.searchParams.set(s,String(r));let o;try{o=await fetch(n,{method:t.method??"GET",headers:{authorization:`Bearer ${he()}`,accept:"application/json",...t.body?{"content-type":"application/json"}:{},...t.headers},body:t.body?JSON.stringify(t.body):void 0,signal:AbortSignal.timeout(Ge)})}catch{throw new m("Nango is temporarily unavailable.",503,"connection_transport_unavailable")}let a=await o.text(),i={};try{i=a?JSON.parse(a):{}}catch{throw new m("Nango returned an invalid response.",502,"nango_invalid_response")}if(!o.ok){let s=o.status===400||o.status===403||o.status===404||o.status===409||o.status===429||o.status===503?o.status:502;throw new m(s===429?"The connected provider is temporarily rate limited.":"Nango request failed.",s,s===429?"upstream_rate_limited":s===404?"connection_not_found":"connection_transport_unavailable",s===429||s>=500,s===429?ye(`retry-after: ${o.headers.get("retry-after")??""}`):null)}return i}function P(e){let t=c(e);return t?t.normalize("NFKC").toLowerCase():null}function Ye(e,t){let n=l(e),o=l(n?.tags),a=l(n?.end_user),i=P(t);if(!i)return!1;let s=[P(o?.end_user_email),P(a?.email)].filter(u=>!!u);if(s.some(u=>u!==i))return!1;let r=[P(o?.end_user_id),P(a?.id)].filter(u=>!!u&&u.includes("@"));return r.some(u=>u!==i)?!1:s.length>0?s.includes(i):r.includes(i)}function je(e){let t=l(e),n=c(t?.connection_id),o=c(t?.provider_config_key);if(!n||!o)return null;let i=(Array.isArray(t?.errors)?t.errors:[]).some(s=>c(l(s)?.type)==="auth");return{upstreamConnectionId:n,providerConfigKey:o,provider:c(t?.provider)??o,label:null,lifecycleStatus:i?"needs_reauth":"connected",reconnectRequired:i,createdAt:c(t?.created_at)??c(t?.created),updatedAt:c(t?.updated_at)}}var Qe=new Set(["facebook","github-getting-started","google-analytics","google-calendar","google-drive","google-mail","google-search-console","linkedin","meta-marketing-api","slack","twitter-v2","xero","youtube","zoom"]);function Ze(e){let t=c(e);return t||(typeof e=="number"&&Number.isFinite(e)?String(e):null)}function T(e){let t=c(e)?.toLowerCase()??null;return t&&/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(t)?t:null}function w(e){return e.id||e.email||e.name?e:null}async function S(e){return N(`/proxy${e.path}`,{query:e.query,headers:{"connection-id":e.upstreamConnectionId,"provider-config-key":e.providerConfigKey,...e.baseUrlOverride?{"base-url-override":e.baseUrlOverride}:{}}})}var Xe=[{method:"GET",pattern:/^\/gmail\/v1\/users\/me\/messages$/},{method:"GET",pattern:/^\/gmail\/v1\/users\/me\/messages\/[A-Za-z0-9_-]+$/},{method:"GET",pattern:/^\/gmail\/v1\/users\/me\/messages\/[A-Za-z0-9_-]+\/attachments\/[A-Za-z0-9_-]+$/},{method:"GET",pattern:/^\/gmail\/v1\/users\/me\/labels$/},{method:"POST",pattern:/^\/gmail\/v1\/users\/me\/messages\/batchModify$/},{method:"POST",pattern:/^\/gmail\/v1\/users\/me\/messages\/batchDelete$/},{method:"POST",pattern:/^\/gmail\/v1\/users\/me\/messages\/send$/},{method:"POST",pattern:/^\/gmail\/v1\/users\/me\/messages\/[A-Za-z0-9_-]+\/(?:trash|untrash)$/}],et=new Set(["q","maxResults","pageToken","format","metadataHeaders","fields"]);async function bt(e){let t=await q(e.identity,e.connectionId);if(t.providerConfigKey!=="google-mail")throw new m("Service connection not found.",404,"connection_not_found",!1);if(!Xe.some(u=>u.method===e.method&&u.pattern.test(e.path)))throw new m("The Gmail provider operation is not allowed.",403,"tool_not_allowed",!1);if(Object.keys(e.query??{}).some(u=>!et.has(u)))throw new m("The Gmail provider query is not allowed.",400,"invalid_request",!1);let n=e.method==="POST";if(n&&!t.actionsEnabled)throw new m("Actions are disabled for this service connection.",403,"actions_disabled",!1);await Y({identity:e.identity,ratePolicyVersion:_});let o=e.requestId?.trim()||ge(),a=`main-gmail-proxy:${I("sha256").update(e.identity.toLowerCase()).update("\0").update(t.id).update("\0").update(e.method).update("\0").update(e.path).update("\0").update(o).digest("hex")}`,i=performance.now(),s,r;try{s=await N(`/proxy${e.path}`,{method:e.method,query:e.query,body:e.body,headers:{"connection-id":t.upstreamConnectionId,"provider-config-key":"google-mail","base-url-override":"https://gmail.googleapis.com"}})}catch(u){r=u}if(await j({identity:e.identity,idempotencyKey:a,ratePolicyVersion:_,connectionId:t.id,providerConfigKey:t.providerConfigKey,functionRuns:0,proxyRequests:1,computeMilliseconds:Math.max(0,Math.round(performance.now()-i)),sourceSurface:"main_mcp",metadata:{operationKind:n?"action":"read",outcome:r?"error":"success",requestId:o.length<=200?o:I("sha256").update(o).digest("hex")}}),r)throw _e(r);return s}async function tt(e){let t=(n,o,a)=>S({...e,path:n,query:o,baseUrlOverride:a});try{if(e.providerConfigKey==="google-mail"){let n=l(await t("/gmail/v1/users/me/profile",void 0,"https://gmail.googleapis.com")),o=T(n?.emailAddress);return w({id:o,email:o,name:null})}if(e.providerConfigKey==="google-drive"){let n=l(await t("/drive/v3/about",{fields:"user(permissionId,displayName,emailAddress)"},"https://www.googleapis.com")),o=l(n?.user);return w({id:c(o?.permissionId),email:T(o?.emailAddress),name:c(o?.displayName)})}if(e.providerConfigKey==="google-calendar"){let n=l(await t("/calendar/v3/calendars/primary",void 0,"https://www.googleapis.com")),o=c(n?.id);return w({id:o,email:T(o),name:c(n?.summary)})}if(e.providerConfigKey==="youtube"){let n=l(await t("/youtube/v3/channels",{part:"snippet",mine:"true",maxResults:1},"https://www.googleapis.com")),o=Array.isArray(n?.items)?l(n.items[0]):null;return w({id:c(o?.id),email:null,name:c(l(o?.snippet)?.title)})}}catch{}try{let n=l(await t("/v1/userinfo",void 0,"https://openidconnect.googleapis.com"));return w({id:c(n?.sub),email:n?.email_verified===!1?null:T(n?.email),name:c(n?.name)})}catch{return null}}async function nt(e){if(!Qe.has(e.providerConfigKey))return null;if(e.providerConfigKey.startsWith("google-")||e.providerConfigKey==="youtube")return tt(e);try{if(e.providerConfigKey==="github-getting-started"){let t=l(await S({...e,path:"/user"}));return w({id:Ze(t?.id),email:T(t?.email),name:c(t?.name)??c(t?.login)})}if(e.providerConfigKey==="slack"){let t=l(await S({...e,path:"/api/auth.test"}));return w({id:c(t?.user_id),email:null,name:[c(t?.user),c(t?.team)].filter(Boolean).join(" \xB7 ")||null})}if(e.providerConfigKey==="zoom"){let t=l(await S({...e,path:"/v2/users/me"})),n=[c(t?.first_name),c(t?.last_name)].filter(Boolean).join(" ");return w({id:c(t?.id),email:T(t?.email),name:c(t?.display_name)??(n||null)})}if(e.providerConfigKey==="linkedin"){let t=l(await S({...e,path:"/v2/userinfo"}));return w({id:c(t?.sub),email:t?.email_verified===!1?null:T(t?.email),name:c(t?.name)})}if(e.providerConfigKey==="meta-marketing-api"||e.providerConfigKey==="facebook"){let t=l(await S({...e,path:"/me",query:{fields:"id,name,email"}}));return w({id:c(t?.id),email:T(t?.email),name:c(t?.name)})}if(e.providerConfigKey==="twitter-v2"){let t=l(await S({...e,path:"/2/users/me",query:{"user.fields":"id,name,username"}})),n=l(t?.data);return w({id:c(n?.id),email:null,name:c(n?.username)??c(n?.name)})}if(e.providerConfigKey==="xero"){let t=await S({...e,path:"/connections"}),n=Array.isArray(t)?l(t[0]):null;return w({id:c(n?.tenantId),email:null,name:c(n?.tenantName)})}}catch{return null}return null}async function Et(e,t={}){let n=[];for(let r=0;r<Ve;r+=1){let u=await N("/connections",{query:{limit:pe,page:r}}),p=X(u,"connections");if(n.push(...p),p.length<pe)break}let o=n.filter(r=>Ye(r,e)).map(je).filter(r=>r!==null),i=(await F(e,o)).filter(r=>!k(r.providerConfigKey));if(t.resolveProviderIdentity===!1)return i;let s=new Map(o.map(r=>[`${r.providerConfigKey}\0${r.upstreamConnectionId}`,r]));return await Promise.all(i.map(async r=>{if(r.lifecycleStatus!=="connected"||!r.upstreamConnectionId)return;let u=s.get(`${r.providerConfigKey}\0${r.upstreamConnectionId}`);if(!u)return;let p=u.updatedAt??u.createdAt??r.lifecycleStatus,d=!!(r.providerAccountId||r.providerAccountEmail||r.providerAccountName),g=r.providerIdentityCheckedAt?Date.parse(`${r.providerIdentityCheckedAt.replace(" ","T")}Z`):Number.NaN,y=!d&&Number.isFinite(g)&&Date.now()-g<$e;if(r.providerIdentitySourceUpdatedAt===p&&(d||y))return;let b=await nt({upstreamConnectionId:r.upstreamConnectionId,providerConfigKey:r.providerConfigKey});await ae({connectionId:r.id,providerAccountId:b?.id,providerAccountEmail:b?.email,providerAccountName:b?.name,sourceUpdatedAt:p})})),(await F(e,o)).filter(r=>!k(r.providerConfigKey))}async function St(){let[e,t]=await Promise.all([N("/integrations"),N("/providers")]),n=new Map;for(let o of X(t,"providers")){let a=l(o),i=c(a?.name);a&&i&&n.set(i,a)}return X(e,"integrations").flatMap(o=>{let a=l(o),i=c(a?.unique_key)??c(a?.id),s=c(a?.provider);if(!i||!s||k(i))return[];let r=n.get(s);return[{id:i,provider:s,displayName:c(a?.display_name)??c(r?.display_name)??i,authMode:c(r?.auth_mode),categories:Array.isArray(r?.categories)?r.categories:[],docsUrl:c(r?.docs),logoUrl:c(a?.logo),safeDefaultAllowedTools:[],actionTools:[],setupStatus:"ready"}]})}function Ce(e){let n={linkedin:"openid profile email w_member_social","github-getting-started":"repo workflow read:org read:user user:email","google-drive":"openid profile email https://www.googleapis.com/auth/drive","google-search-console":"openid profile email https://www.googleapis.com/auth/webmasters.readonly https://www.googleapis.com/auth/webmasters"}[e];return n?{[e]:{connection_config:{oauth_scopes_override:n}}}:void 0}function ve(e){let t=l(e),n=l(t?.data)??t,o=c(n?.connect_link);if(!o)throw new m("Nango did not return a connect link.",502,"nango_missing_connect_link");return{connectLink:o,expiresAt:c(n?.expires_at)}}async function Tt(e,t){fe(t);let n=Ce(t);return ve(await N("/connect/sessions",{method:"POST",body:{tags:{end_user_id:e,end_user_email:e},allowed_integrations:[t],...n?{integrations_config_defaults:n}:{}}}))}async function Nt(e,t){let n=await q(e,t,!0),o=Ce(n.providerConfigKey);return ve(await N("/connect/sessions/reconnect",{method:"POST",body:{connection_id:n.upstreamConnectionId,integration_id:n.providerConfigKey,tags:{end_user_id:e,end_user_email:e},...o?{integrations_config_defaults:o}:{}}}))}async function Rt(e,t){let n=await be(e,t);try{await N(`/connections/${encodeURIComponent(n.upstreamConnectionId)}`,{method:"DELETE",query:{provider_config_key:n.providerConfigKey}})}catch(o){if(!(o instanceof m)||o.status!==404)throw o}}async function be(e,t){let n=await V(e,t);if(!n||n.transport!=="nango"||!n.upstreamConnectionId)throw new m("Service connection not found.",404,"connection_not_found",!1);return n}async function q(e,t,n=!1){let o=await be(e,t);if(fe(o.providerConfigKey),o.lifecycleStatus!=="connected"&&!(n&&o.lifecycleStatus==="needs_reauth"))throw new m("Service connection requires reconnection.",409,"connection_inactive",!1);return o}async function ee(e,t){let n=new Ke(we(),{requestInit:{headers:{Authorization:`Bearer ${he()}`,"connection-id":e.upstreamConnectionId,"provider-config-key":e.providerConfigKey}}}),o=new De({name:"mcp-scraper-integrations",version:"1.0.0"},{capabilities:{}});try{await o.connect(n);let a=await t(o);return await $({connectionId:e.id,operationalStatus:"available",evidenceSource:"nango_mcp"}),a}catch(a){let i=_e(a);throw await $({connectionId:e.id,operationalStatus:i.code==="connection_transport_unavailable"?"unavailable":"degraded",failureCode:i.code,retryable:i.retryable,evidenceSource:"nango_mcp"}).catch(()=>{}),i}finally{await o.close().catch(()=>{})}}async function It(e,t){let n=await q(e,t);(n.provider??n.providerConfigKey)==="google-search-console"&&await ee(n,async a=>{if(!(await a.listTools()).tools.some(s=>s.name==="list-sites"))throw new m("The Search Console health-check tool is unavailable.",404,"live_tool_missing",!1);await a.callTool({name:"list-sites",arguments:{}})})}async function Ee(e,t){let n=await q(e,t),a=(await ee(n,d=>d.listTools())).tools.flatMap(d=>{let g=l(d.inputSchema);return!d.name||!g?[]:[{name:d.name,...d.title?{title:d.title}:{},...d.description?{description:d.description}:{},inputSchema:g,...l(d.outputSchema)?{outputSchema:l(d.outputSchema)}:{},...l(d.annotations)?{annotations:l(d.annotations)}:{}}]}),i=O({tools:a,connectionActive:n.lifecycleStatus==="connected",actionsEnabled:n.actionsEnabled}),s=[...i.values()].filter(d=>d.classification==="read").map(d=>d.name),r=[...i.values()].filter(d=>d.classification==="action").map(d=>d.name),u=I("sha256").update(JSON.stringify(a.map(d=>({name:d.name,inputSchema:d.inputSchema})))).digest("hex");return await re(n.id,s,r,u),{connection:await V(e,n.id)??{...n,readTools:s,actionTools:r,toolRevision:u},tools:a}}async function At(e){let{connection:t,tools:n}=await Ee(e.identity,e.connectionId),o=n.find(y=>y.name===e.tool);if(!o)throw new m("The requested provider tool is unavailable.",404,"live_tool_missing",!1);let a=Z(O({tools:n,connectionActive:t.lifecycleStatus==="connected",actionsEnabled:t.actionsEnabled}),o.name);if(a.classification!==e.classification)throw new m("The provider tool classification does not match this operation.",403,"tool_not_allowed",!1);if(!a.available&&a.blockedReason==="actions_disabled")throw new m("Actions are disabled for this service connection.",403,"actions_disabled",!1);if(!a.available)throw new m("The provider tool is not authorized for this connection.",403,"tool_not_allowed",!1);await Y({identity:e.identity,ratePolicyVersion:_});let i=e.requestId?.trim()||ge(),s=`main-nango:${I("sha256").update(e.identity.toLowerCase()).update("\0").update(t.id).update("\0").update(e.tool).update("\0").update(i).digest("hex")}`,r=new Date,u=performance.now(),p,d;try{p=await ee(t,y=>We(y,e.tool,e.input))}catch(y){d=y}let g=new Date;if(await j({identity:e.identity,idempotencyKey:s,ratePolicyVersion:_,connectionId:t.id,providerConfigKey:t.providerConfigKey,functionRuns:1,proxyRequests:0,computeMilliseconds:Math.max(0,Math.round(performance.now()-u)),sourceSurface:"main_mcp",metadata:{toolName:e.tool,operationKind:e.operationKind??e.classification,outcome:d?"error":"partial",requestId:i.length<=200?i:I("sha256").update(i).digest("hex"),startedAt:r.toISOString(),completedAt:g.toISOString()}}),d)throw d;return p}async function Pt(e,t,n){let{connection:o,tools:a}=await Ee(e,t),i=a.find(d=>d.name===n);if(!i)throw new m("The requested provider tool is unavailable.",404,"live_tool_missing",!1);let s=Z(O({tools:a,connectionActive:o.lifecycleStatus==="connected",actionsEnabled:o.actionsEnabled}),i.name),r=Q(i),u=s.available,p={name:i.name,...i.title?{title:i.title}:{},...i.description?{description:i.description}:{},inputSchema:i.inputSchema,...i.outputSchema?{outputSchema:i.outputSchema}:{},...i.annotations?{annotations:i.annotations}:{}};return{...p,classification:r,callable:u,blockedReason:s.blockedReason,requiredPermissions:s.requiredPermissions,missingPermissions:s.missingPermissions,requiredFeatures:s.requiredFeatures,missingFeatures:s.missingFeatures,transport:"nango",providerConfigKey:o.providerConfigKey,providerContractVersion:Be,providerContractHash:Je,protocolVersion:null,schemaSource:"live_tools_list",schemaHash:I("sha256").update(JSON.stringify(p)).digest("hex"),fetchedAt:new Date().toISOString()}}export{at as a,Ae as b,Pe as c,h as d,Y as e,j as f,pt as g,m as h,k as i,fe as j,bt as k,Et as l,St as m,Tt as n,Nt as o,Rt as p,It as q,Ee as r,At as s,Pt as t};
|
package/dist/chunk-RLFCJGHA.js
DELETED
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
import{t as r}from"./chunk-7YOXSMHG.js";import{createHash as b,randomUUID as O}from"crypto";var p="mcp-scraper/lead-list-enrichment.requested",A=new Set(["complete","partial","empty","failed","cancelled"]),U=new Set(["complete","failed","cancelled"]),T=10,g=360,S=60,h=null,m=null;function o(e,t,s){let n=e.trim();if(!n||n.length>s)throw new Error(`${t} is required and must be ${s} characters or fewer`);return n}function d(e,t){if(!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer`);return e}function u(e){if(e==null)return null;try{return JSON.parse(String(e))}catch{return null}}function R(e){let t=new WeakSet,s=n=>{if(n===null||typeof n=="string"||typeof n=="boolean")return n;if(typeof n=="number"){if(!Number.isFinite(n))throw new Error("canonical fingerprint input must contain finite numbers");return Object.is(n,-0)?0:n}if(Array.isArray(n))return n.map(s);if(typeof n=="object"){if(t.has(n))throw new Error("canonical fingerprint input must not be cyclic");t.add(n);let i={};for(let _ of Object.keys(n).sort()){let c=n[_];c!==void 0&&(i[_]=s(c))}return t.delete(n),i}throw new Error(`canonical fingerprint input contains unsupported ${typeof n}`)};return JSON.stringify(s(e))}function y(e){return b("sha256").update(R(e)).digest("hex")}function I(e){return b("sha256").update(R(e)).digest("hex")}function j(){return{mapsAttempts:0,pageAttempts:0,pageSuccesses:0,serpSearches:0}}function E(e){let t=u(e.input_json);if(!t)throw new Error(`lead list enrichment job ${String(e.id??"")} has invalid durable input`);return{id:String(e.id),userId:Number(e.user_id),idempotencyKey:String(e.idempotency_key),requestFingerprint:String(e.request_fingerprint),input:t,status:String(e.status),totalRows:Number(e.total_rows??0),doneRows:Number(e.done_rows??0),failedRows:Number(e.failed_rows??0),cancelledRows:Number(e.cancelled_rows??0),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},concurrencyLockId:e.concurrency_lock_id==null?null:String(e.concurrency_lock_id),debitKey:String(e.debit_key),holdMc:Number(e.hold_mc??0),billedMc:e.billed_mc==null?null:Number(e.billed_mc),result:u(e.result_json),csvArtifact:u(e.csv_artifact_json),xlsxArtifact:u(e.xlsx_artifact_json),warnings:u(e.warnings_json)??[],cancelRequestedAt:e.cancel_requested_at==null?null:String(e.cancel_requested_at),errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:e.retryable==null?null:Number(e.retryable)===1,createdAt:String(e.created_at??""),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function w(e){return{jobId:String(e.job_id),ordinal:Number(e.ordinal),inputDigest:String(e.input_digest),status:String(e.status),result:u(e.result_json),usage:{mapsAttempts:Number(e.maps_attempts??0),pageAttempts:Number(e.page_attempts??0),pageSuccesses:Number(e.page_successes??0),serpSearches:Number(e.serp_searches??0)},errorCode:e.error_code==null?null:String(e.error_code),errorMessage:e.error_message==null?null:String(e.error_message),retryable:Number(e.retryable??0)===1,durationMs:Number(e.duration_ms??0),startedAt:e.started_at==null?null:String(e.started_at),completedAt:e.completed_at==null?null:String(e.completed_at),updatedAt:String(e.updated_at??"")}}function M(e){return{id:String(e.id),jobId:String(e.job_id),eventName:p,status:String(e.status),attempts:Number(e.attempts??0),claimToken:e.locked_by==null?null:String(e.locked_by),lockedUntil:e.locked_until==null?null:String(e.locked_until),nextAttemptAt:e.next_attempt_at==null?null:String(e.next_attempt_at),lastError:e.last_error==null?null:String(e.last_error),createdAt:String(e.created_at??""),dispatchedAt:e.dispatched_at==null?null:String(e.dispatched_at),updatedAt:String(e.updated_at??"")}}async function a(){let e=r();return h!==e&&(h=e,m=null),m||(m=(async()=>{await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_jobs (
|
|
2
|
-
id TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id), idempotency_key TEXT NOT NULL,
|
|
3
|
-
request_fingerprint TEXT NOT NULL, input_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'initializing',
|
|
4
|
-
total_rows INTEGER NOT NULL DEFAULT 0, done_rows INTEGER NOT NULL DEFAULT 0,
|
|
5
|
-
failed_rows INTEGER NOT NULL DEFAULT 0, cancelled_rows INTEGER NOT NULL DEFAULT 0,
|
|
6
|
-
maps_attempts INTEGER NOT NULL DEFAULT 0, page_attempts INTEGER NOT NULL DEFAULT 0,
|
|
7
|
-
page_successes INTEGER NOT NULL DEFAULT 0, serp_searches INTEGER NOT NULL DEFAULT 0,
|
|
8
|
-
concurrency_lock_id TEXT, debit_key TEXT NOT NULL,
|
|
9
|
-
hold_mc INTEGER NOT NULL DEFAULT 0, billed_mc INTEGER, result_json TEXT,
|
|
10
|
-
csv_artifact_json TEXT, xlsx_artifact_json TEXT, warnings_json TEXT NOT NULL DEFAULT '[]',
|
|
11
|
-
cancel_requested_at TEXT, error_code TEXT, error_message TEXT, retryable INTEGER,
|
|
12
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')), started_at TEXT, completed_at TEXT,
|
|
13
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE(user_id, idempotency_key)
|
|
14
|
-
)`);try{await e.execute("ALTER TABLE lead_list_enrichment_jobs ADD COLUMN serp_searches INTEGER NOT NULL DEFAULT 0")}catch{}await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_jobs_user_created ON lead_list_enrichment_jobs(user_id, created_at DESC)"),await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_jobs_status ON lead_list_enrichment_jobs(status, updated_at)"),await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_rows (
|
|
15
|
-
job_id TEXT NOT NULL REFERENCES lead_list_enrichment_jobs(id) ON DELETE CASCADE,
|
|
16
|
-
ordinal INTEGER NOT NULL, input_digest TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending',
|
|
17
|
-
result_json TEXT, maps_attempts INTEGER NOT NULL DEFAULT 0, page_attempts INTEGER NOT NULL DEFAULT 0,
|
|
18
|
-
page_successes INTEGER NOT NULL DEFAULT 0, serp_searches INTEGER NOT NULL DEFAULT 0,
|
|
19
|
-
error_code TEXT, error_message TEXT,
|
|
20
|
-
retryable INTEGER NOT NULL DEFAULT 0, duration_ms INTEGER NOT NULL DEFAULT 0,
|
|
21
|
-
started_at TEXT, completed_at TEXT, updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
22
|
-
PRIMARY KEY(job_id, ordinal)
|
|
23
|
-
)`);try{await e.execute("ALTER TABLE lead_list_enrichment_rows ADD COLUMN serp_searches INTEGER NOT NULL DEFAULT 0")}catch{}await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_rows_job_status ON lead_list_enrichment_rows(job_id, status, ordinal)"),await e.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_outbox (
|
|
24
|
-
id TEXT PRIMARY KEY, job_id TEXT NOT NULL UNIQUE REFERENCES lead_list_enrichment_jobs(id) ON DELETE CASCADE,
|
|
25
|
-
event_name TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'held', attempts INTEGER NOT NULL DEFAULT 0,
|
|
26
|
-
locked_by TEXT, locked_until TEXT, next_attempt_at TEXT, last_error TEXT,
|
|
27
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')), dispatched_at TEXT,
|
|
28
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
29
|
-
)`),await e.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_outbox_dispatch ON lead_list_enrichment_outbox(status, next_attempt_at, locked_until, created_at)")})()),m}async function l(e,t){await a();let s=t==null?await r().execute({sql:"SELECT * FROM lead_list_enrichment_jobs WHERE id = ? LIMIT 1",args:[e]}):await r().execute({sql:"SELECT * FROM lead_list_enrichment_jobs WHERE id = ? AND user_id = ? LIMIT 1",args:[e,t]});return s.rows[0]?E(s.rows[0]):null}async function C(e,t){await a();let s=await r().execute({sql:"SELECT * FROM lead_list_enrichment_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",args:[e,t]});return s.rows[0]?E(s.rows[0]):null}async function F(e){await a();let t=o(e.id,"job id",200),s=o(e.idempotencyKey,"idempotency key",200),n=e.requestFingerprint??y(e.input);o(n,"request fingerprint",200);let i=o(e.debitKey,"debit key",500),_=d(e.holdMc,"holdMc");if(e.input.rows.length<1||e.input.rows.length>250)throw new Error("lead list job must contain 1 to 250 rows");let c=[{sql:`INSERT OR IGNORE INTO lead_list_enrichment_jobs (
|
|
30
|
-
id,user_id,idempotency_key,request_fingerprint,input_json,total_rows,concurrency_lock_id,debit_key,hold_mc
|
|
31
|
-
) VALUES (?,?,?,?,?,?,?,?,?)`,args:[t,e.userId,s,n,JSON.stringify(e.input),e.input.rows.length,e.concurrencyLockId??null,i,_]},{sql:`INSERT OR IGNORE INTO lead_list_enrichment_outbox(id,job_id,event_name,status)
|
|
32
|
-
SELECT 'lead-list-enrichment:' || id,id,?,'held' FROM lead_list_enrichment_jobs
|
|
33
|
-
WHERE user_id=? AND idempotency_key=? AND request_fingerprint=?`,args:[p,e.userId,s,n]}];e.input.rows.forEach((D,x)=>c.push({sql:`INSERT OR IGNORE INTO lead_list_enrichment_rows(job_id,ordinal,input_digest)
|
|
34
|
-
SELECT id,?,? FROM lead_list_enrichment_jobs
|
|
35
|
-
WHERE user_id=? AND idempotency_key=? AND request_fingerprint=?`,args:[x,I(D),e.userId,s,n]}));let f=await r().batch(c,"write"),L=await C(e.userId,s);if(!L)throw new Error("lead list enrichment job could not be created");return{job:L,created:Number(f[0]?.rowsAffected??0)>0,conflict:L.requestFingerprint!==n}}async function H(e,t){return await a(),await r().execute({sql:"UPDATE lead_list_enrichment_jobs SET concurrency_lock_id=?, updated_at=datetime('now') WHERE id=? AND status='initializing'",args:[t,e]}),l(e)}async function W(e){return await a(),await r().batch([{sql:"UPDATE lead_list_enrichment_jobs SET status='queued',updated_at=datetime('now') WHERE id=? AND status='initializing'",args:[e]},{sql:`UPDATE lead_list_enrichment_outbox SET status='pending',next_attempt_at=NULL,updated_at=datetime('now')
|
|
36
|
-
WHERE job_id=? AND status='held' AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='queued')`,args:[e,e]}],"write"),l(e)}async function X(e){await a(),await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status='running',started_at=COALESCE(started_at,datetime('now')),
|
|
37
|
-
updated_at=datetime('now') WHERE id=? AND status IN ('queued','running') AND cancel_requested_at IS NULL`,args:[e]});let t=await l(e);return t?.status==="running"?t:null}async function P(e,t){return await a(),d(t,"row ordinal"),await r().execute({sql:`UPDATE lead_list_enrichment_rows SET status='running',started_at=COALESCE(started_at,datetime('now')),
|
|
38
|
-
updated_at=datetime('now') WHERE job_id=? AND ordinal=? AND status IN ('pending','running')
|
|
39
|
-
AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='running' AND cancel_requested_at IS NULL)`,args:[e,t,e]}),N(e,t)}async function N(e,t){await a();let s=await r().execute({sql:"SELECT * FROM lead_list_enrichment_rows WHERE job_id=? AND ordinal=? LIMIT 1",args:[e,t]});return s.rows[0]?w(s.rows[0]):null}async function J(e){return await a(),(await r().execute({sql:"SELECT * FROM lead_list_enrichment_rows WHERE job_id=? ORDER BY ordinal ASC",args:[e]})).rows.map(s=>w(s))}async function G(e){await a();let t=e.result.usage??j(),s=e.result.rowStatus==="failed"?"failed":"complete",n=await N(e.jobId,e.ordinal);return n&&U.has(n.status)?n:(await r().batch([{sql:`UPDATE lead_list_enrichment_rows SET status=?,result_json=?,maps_attempts=?,page_attempts=?,page_successes=?,serp_searches=?,
|
|
40
|
-
error_code=?,error_message=?,retryable=?,duration_ms=?,completed_at=datetime('now'),updated_at=datetime('now')
|
|
41
|
-
WHERE job_id=? AND ordinal=? AND status IN ('pending','running')
|
|
42
|
-
AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='running')`,args:[s,JSON.stringify(e.result),d(t.mapsAttempts,"mapsAttempts"),d(t.pageAttempts,"pageAttempts"),d(t.pageSuccesses,"pageSuccesses"),d(t.serpSearches,"serpSearches"),e.result.error?.code??null,e.result.error?.message??null,e.result.error?1:0,d(e.durationMs??0,"durationMs"),e.jobId,e.ordinal,e.jobId]},{sql:`UPDATE lead_list_enrichment_jobs SET
|
|
43
|
-
done_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status IN ('complete','failed','cancelled')),
|
|
44
|
-
failed_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='failed'),
|
|
45
|
-
cancelled_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='cancelled'),
|
|
46
|
-
maps_attempts=(SELECT COALESCE(SUM(maps_attempts),0) FROM lead_list_enrichment_rows WHERE job_id=?),
|
|
47
|
-
page_attempts=(SELECT COALESCE(SUM(page_attempts),0) FROM lead_list_enrichment_rows WHERE job_id=?),
|
|
48
|
-
page_successes=(SELECT COALESCE(SUM(page_successes),0) FROM lead_list_enrichment_rows WHERE job_id=?),
|
|
49
|
-
serp_searches=(SELECT COALESCE(SUM(serp_searches),0) FROM lead_list_enrichment_rows WHERE job_id=?),
|
|
50
|
-
updated_at=datetime('now') WHERE id=? AND status='running'`,args:[e.jobId,e.jobId,e.jobId,e.jobId,e.jobId,e.jobId,e.jobId,e.jobId]}],"write"),N(e.jobId,e.ordinal))}async function v(e,t){await a();let s=t==null?"":" AND user_id=?",n=[e];return t!=null&&n.push(t),await r().batch([{sql:`UPDATE lead_list_enrichment_jobs SET cancel_requested_at=COALESCE(cancel_requested_at,datetime('now')),
|
|
51
|
-
status=CASE WHEN status IN ('initializing','queued') THEN 'settling' ELSE status END,updated_at=datetime('now')
|
|
52
|
-
WHERE id=?${s} AND status IN ('initializing','queued','running')`,args:n},{sql:`UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
|
|
53
|
-
last_error='Lead list enrichment was cancelled before execution.',updated_at=datetime('now')
|
|
54
|
-
WHERE job_id=? AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND cancel_requested_at IS NOT NULL AND status='settling')`,args:[e,e]}],"write"),l(e,t)}async function $(e){await a();let t=await r().batch([{sql:`UPDATE lead_list_enrichment_rows SET status='cancelled',completed_at=datetime('now'),updated_at=datetime('now')
|
|
55
|
-
WHERE job_id=? AND status='pending' AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND cancel_requested_at IS NOT NULL)`,args:[e,e]},{sql:`UPDATE lead_list_enrichment_jobs SET
|
|
56
|
-
done_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status IN ('complete','failed','cancelled')),
|
|
57
|
-
failed_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='failed'),
|
|
58
|
-
cancelled_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='cancelled'),
|
|
59
|
-
status=CASE WHEN status='running' THEN 'settling' ELSE status END,updated_at=datetime('now')
|
|
60
|
-
WHERE id=? AND cancel_requested_at IS NOT NULL`,args:[e,e,e,e]}],"write");return Number(t[0]?.rowsAffected??0)}async function B(e){return await a(),await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status='settling',result_json=?,csv_artifact_json=?,xlsx_artifact_json=?,
|
|
61
|
-
warnings_json=?,updated_at=datetime('now') WHERE id=? AND status IN ('running','settling')`,args:[JSON.stringify(e.result),e.csvArtifact==null?null:JSON.stringify(e.csvArtifact),e.xlsxArtifact==null?null:JSON.stringify(e.xlsxArtifact),JSON.stringify(e.warnings??e.result.warnings),e.jobId]}),l(e.jobId)}async function K(e){await a();let t=await l(e.jobId);if(!t)throw new Error("lead list enrichment job not found");let s=d(e.billedMc,"billedMc");if(s>t.holdMc)throw new Error("billedMc cannot exceed the lead list enrichment hold");if(A.has(t.status)){if(t.status===e.status&&t.billedMc===s)return t;throw new Error("lead list enrichment job already has a conflicting terminal result")}await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status=?,billed_mc=?,result_json=COALESCE(?,result_json),
|
|
62
|
-
csv_artifact_json=CASE WHEN ?=1 THEN ? ELSE csv_artifact_json END,
|
|
63
|
-
xlsx_artifact_json=CASE WHEN ?=1 THEN ? ELSE xlsx_artifact_json END,
|
|
64
|
-
warnings_json=CASE WHEN ?=1 THEN ? ELSE warnings_json END,error_code=NULL,error_message=NULL,retryable=NULL,
|
|
65
|
-
completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status NOT IN ('complete','partial','empty','failed','cancelled')`,args:[e.status,s,e.result===void 0?null:JSON.stringify(e.result),e.csvArtifact===void 0?0:1,e.csvArtifact==null?null:JSON.stringify(e.csvArtifact),e.xlsxArtifact===void 0?0:1,e.xlsxArtifact==null?null:JSON.stringify(e.xlsxArtifact),e.warnings===void 0?0:1,JSON.stringify(e.warnings??[]),e.jobId]});let n=await l(e.jobId);if(!n||n.status!==e.status||n.billedMc!==s)throw new Error("lead list enrichment job completion conflicted");return n}async function z(e){await a();let t=await l(e.jobId);if(!t)throw new Error("lead list enrichment job not found");if(t.status!=="failed"&&A.has(t.status))throw new Error("completed lead list enrichment job cannot be failed");if(t.status==="failed")return t;let s=e.billedMc==null?null:d(e.billedMc,"billedMc");if(s!=null&&s>t.holdMc)throw new Error("billedMc cannot exceed the lead list enrichment hold");await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET status='failed',billed_mc=COALESCE(?,billed_mc),error_code=?,error_message=?,retryable=?,
|
|
66
|
-
completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status NOT IN ('complete','partial','empty','failed','cancelled')`,args:[s,o(e.errorCode,"error code",200),o(e.errorMessage,"error message",2e3),e.retryable?1:0,e.jobId]});let n=await l(e.jobId);if(!n||n.status!=="failed")throw new Error("lead list enrichment job failure conflicted");return n}async function Y(e,t){await a();let s=await l(e);if(!s)throw new Error("lead list enrichment job not found");let n=d(t,"billedMc");if(n>s.holdMc)throw new Error("billedMc cannot exceed the lead list enrichment hold");await r().execute({sql:`UPDATE lead_list_enrichment_jobs SET billed_mc=COALESCE(billed_mc,?),updated_at=datetime('now')
|
|
67
|
-
WHERE id=? AND status IN ('settling','failed','cancelled')`,args:[n,e]});let i=await l(e);if(!i||i.billedMc!==n)throw new Error("lead list enrichment settlement conflicts with durable amount");return i}async function Q(e=25){return await a(),(await r().execute({sql:`SELECT * FROM lead_list_enrichment_jobs WHERE status='settling' OR (status IN ('failed','cancelled') AND billed_mc IS NULL)
|
|
68
|
-
ORDER BY updated_at ASC LIMIT ?`,args:[Math.max(1,Math.min(100,Math.trunc(e)))]})).rows.map(s=>E(s))}async function V(e=25,t=T,s=g){return await a(),(await r().execute({sql:`SELECT j.* FROM lead_list_enrichment_jobs j JOIN lead_list_enrichment_outbox o ON o.job_id=j.id
|
|
69
|
-
WHERE j.status IN ('initializing','queued') AND j.billed_mc IS NULL AND (o.attempts>=? OR j.created_at<=datetime('now',?))
|
|
70
|
-
AND (o.status IN ('held','pending','failed','dispatched') OR (o.status='dispatching' AND o.locked_until<=datetime('now')))
|
|
71
|
-
ORDER BY j.updated_at ASC LIMIT ?`,args:[Math.max(1,Math.trunc(t)),`-${Math.max(15,Math.trunc(s))} minutes`,Math.max(1,Math.min(100,Math.trunc(e)))]})).rows.map(i=>E(i))}async function Z(e,t=T,s=g){await a();let n=await r().batch([{sql:`UPDATE lead_list_enrichment_jobs SET status='failed',billed_mc=CASE
|
|
72
|
-
WHEN EXISTS(SELECT 1 FROM billing_debits d WHERE d.idempotency_key=lead_list_enrichment_jobs.debit_key
|
|
73
|
-
AND d.user_id=lead_list_enrichment_jobs.user_id AND d.status IN ('applied','settled')) THEN NULL ELSE 0 END,
|
|
74
|
-
error_code='lead_list_dispatch_expired',
|
|
75
|
-
error_message='Lead list enrichment did not start before its dispatch deadline; the unused hold will be refunded.',retryable=1,
|
|
76
|
-
completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status IN ('initializing','queued') AND billed_mc IS NULL
|
|
77
|
-
AND EXISTS(SELECT 1 FROM lead_list_enrichment_outbox o WHERE o.job_id=lead_list_enrichment_jobs.id AND
|
|
78
|
-
(o.attempts>=? OR lead_list_enrichment_jobs.created_at<=datetime('now',?)))`,args:[e,Math.max(1,Math.trunc(t)),`-${Math.max(15,Math.trunc(s))} minutes`]},{sql:`UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
|
|
79
|
-
last_error='Lead list enrichment dispatch deadline exhausted.',updated_at=datetime('now') WHERE job_id=?
|
|
80
|
-
AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND error_code='lead_list_dispatch_expired')`,args:[e,e]}],"write");return Number(n[0]?.rowsAffected??0)===1?l(e):null}async function ee(e=25,t=S){return await a(),(await r().execute({sql:`SELECT * FROM lead_list_enrichment_jobs WHERE status='running' AND billed_mc IS NULL
|
|
81
|
-
AND updated_at<=datetime('now',?) ORDER BY updated_at ASC LIMIT ?`,args:[`-${Math.max(15,Math.trunc(t))} minutes`,Math.max(1,Math.min(100,Math.trunc(e)))]})).rows.map(n=>E(n))}async function te(e,t=S){await a();let s=await r().batch([{sql:`UPDATE lead_list_enrichment_jobs SET
|
|
82
|
-
status=CASE WHEN cancel_requested_at IS NULL THEN 'failed' ELSE 'settling' END,
|
|
83
|
-
error_code=CASE WHEN cancel_requested_at IS NULL THEN 'lead_list_worker_stale' ELSE NULL END,
|
|
84
|
-
error_message=CASE WHEN cancel_requested_at IS NULL
|
|
85
|
-
THEN 'Lead list enrichment stopped without a heartbeat; completed row checkpoints were preserved.' ELSE NULL END,
|
|
86
|
-
retryable=CASE WHEN cancel_requested_at IS NULL THEN 1 ELSE NULL END,
|
|
87
|
-
completed_at=CASE WHEN cancel_requested_at IS NULL THEN datetime('now') ELSE NULL END,updated_at=datetime('now')
|
|
88
|
-
WHERE id=? AND status='running' AND billed_mc IS NULL
|
|
89
|
-
AND updated_at<=datetime('now',?)`,args:[e,`-${Math.max(15,Math.trunc(t))} minutes`]},{sql:`UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
|
|
90
|
-
last_error='Lead list enrichment worker heartbeat expired.',updated_at=datetime('now') WHERE job_id=?
|
|
91
|
-
AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND (error_code='lead_list_worker_stale'
|
|
92
|
-
OR (status='settling' AND cancel_requested_at IS NOT NULL)))`,args:[e,e]}],"write");return Number(s[0]?.rowsAffected??0)===1?l(e):null}async function ne(e){await a();let t=o(e.workerId,"outbox worker id",160),s=Math.max(1,Math.min(100,Math.trunc(e.limit??25))),n=Math.max(30,Math.min(3600,Math.trunc(e.leaseSeconds??120))),i=`${t}:${O()}`;return(await r().batch([{sql:`UPDATE lead_list_enrichment_outbox SET status='dispatching',attempts=attempts+1,locked_by=?,
|
|
93
|
-
locked_until=datetime('now',?),updated_at=datetime('now') WHERE id IN (SELECT id FROM lead_list_enrichment_outbox
|
|
94
|
-
WHERE attempts<${T} AND created_at>datetime('now','-${g} minutes')
|
|
95
|
-
AND (status='pending' OR (status='failed' AND (next_attempt_at IS NULL OR next_attempt_at<=datetime('now')))
|
|
96
|
-
OR (status='dispatching' AND locked_until<=datetime('now'))) ORDER BY created_at ASC LIMIT ?)`,args:[i,`+${n} seconds`,s]},{sql:"SELECT * FROM lead_list_enrichment_outbox WHERE locked_by=? AND status='dispatching' ORDER BY created_at ASC",args:[i]}],"write"))[1]?.rows.map(c=>M(c))??[]}async function se(e,t){await a();let s=await r().execute({sql:`UPDATE lead_list_enrichment_outbox SET status='dispatched',locked_by=NULL,
|
|
97
|
-
locked_until=NULL,next_attempt_at=NULL,last_error=NULL,dispatched_at=datetime('now'),updated_at=datetime('now')
|
|
98
|
-
WHERE id=? AND status='dispatching' AND locked_by=?`,args:[e,t]});return Number(s.rowsAffected??0)>0}async function re(e){await a();let t=Math.max(1,Math.min(86400,Math.trunc(e.retryAfterSeconds??30))),s=await r().execute({sql:`UPDATE lead_list_enrichment_outbox SET status='failed',locked_by=NULL,
|
|
99
|
-
locked_until=NULL,next_attempt_at=datetime('now',?),last_error=?,updated_at=datetime('now')
|
|
100
|
-
WHERE id=? AND status='dispatching' AND locked_by=?`,args:[`+${t} seconds`,String(e.error).slice(0,2e3),e.id,e.claimToken]});return Number(s.rowsAffected??0)>0}export{T as a,g as b,S as c,R as d,y as e,I as f,a as g,l as h,C as i,F as j,H as k,W as l,X as m,P as n,N as o,J as p,G as q,v as r,$ as s,B as t,K as u,z as v,Y as w,Q as x,V as y,Z as z,ee as A,te as B,ne as C,se as D,re as E};
|
package/dist/chunk-T4VTJNDZ.js
DELETED
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
import{R as p,t as o}from"./chunk-7YOXSMHG.js";import{createHmac as T,timingSafeEqual as y}from"crypto";var N=()=>process.env.NODE_ENV==="production"||process.env.VERCEL==="1";function S(){let e=process.env.SESSION_SECRET?.trim();if(e)return e;if(N())throw new Error("SESSION_SECRET is not set \u2014 add it to your Vercel env vars (see src/api/env.ts)");return"dev-secret-change-me"}var v=()=>S();function m(e,t){if(e.length!==t.length)return!1;try{return y(Buffer.from(e,"hex"),Buffer.from(t,"hex"))}catch{return!1}}function L(e){let t=String(e),n=T("sha256",v()).update(t).digest("hex");return`${t}.${n}`}function I(e){let t=e.lastIndexOf(".");if(t===-1)return null;let n=e.slice(0,t),r=e.slice(t+1),i=T("sha256",v()).update(n).digest("hex");if(!m(r,i))return null;let s=parseInt(n);return isNaN(s)?null:s}import{randomUUID as _}from"crypto";function x(e,t,n){return e.lifecycleStatus==="connected"&&e.actionsEnabled&&t.includes(e.providerConfigKey)&&e.actionTools.includes(n)}var a=null,u=null;function c(){let e=o();return a&&u===e||(u=e,a=(async()=>{let t=e;await t.execute(`
|
|
2
|
-
CREATE TABLE IF NOT EXISTS service_connections (
|
|
3
|
-
id TEXT PRIMARY KEY,
|
|
4
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
5
|
-
provider_config_key TEXT NOT NULL,
|
|
6
|
-
provider TEXT,
|
|
7
|
-
transport TEXT NOT NULL,
|
|
8
|
-
upstream_connection_id TEXT,
|
|
9
|
-
label TEXT,
|
|
10
|
-
user_label TEXT,
|
|
11
|
-
provider_account_id TEXT,
|
|
12
|
-
provider_account_email TEXT,
|
|
13
|
-
provider_account_name TEXT,
|
|
14
|
-
provider_identity_checked_at TEXT,
|
|
15
|
-
provider_identity_source_updated_at TEXT,
|
|
16
|
-
lifecycle_status TEXT NOT NULL DEFAULT 'pending',
|
|
17
|
-
operational_status TEXT NOT NULL DEFAULT 'unknown',
|
|
18
|
-
reconnect_required INTEGER NOT NULL DEFAULT 0,
|
|
19
|
-
actions_enabled INTEGER NOT NULL DEFAULT 0,
|
|
20
|
-
read_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
21
|
-
action_tools_json TEXT NOT NULL DEFAULT '[]',
|
|
22
|
-
tool_revision TEXT,
|
|
23
|
-
last_checked_at TEXT,
|
|
24
|
-
last_successful_call_at TEXT,
|
|
25
|
-
last_failure_at TEXT,
|
|
26
|
-
last_failure_code TEXT,
|
|
27
|
-
last_failure_retryable INTEGER,
|
|
28
|
-
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
29
|
-
vault_name TEXT,
|
|
30
|
-
table_name TEXT,
|
|
31
|
-
legacy_owner_identity TEXT,
|
|
32
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
33
|
-
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
34
|
-
UNIQUE(user_id, provider_config_key, upstream_connection_id)
|
|
35
|
-
)
|
|
36
|
-
`);try{await t.execute("ALTER TABLE service_connections ADD COLUMN user_label TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_id TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_email TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_account_name TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_identity_checked_at TEXT")}catch{}try{await t.execute("ALTER TABLE service_connections ADD COLUMN provider_identity_source_updated_at TEXT")}catch{}await t.execute("CREATE INDEX IF NOT EXISTS service_connections_user_updated ON service_connections(user_id, updated_at DESC)"),await t.execute("CREATE INDEX IF NOT EXISTS service_connections_upstream ON service_connections(provider_config_key, upstream_connection_id)"),await t.execute(`
|
|
37
|
-
CREATE TABLE IF NOT EXISTS service_connection_health_events (
|
|
38
|
-
id TEXT PRIMARY KEY,
|
|
39
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
40
|
-
operational_status TEXT NOT NULL,
|
|
41
|
-
failure_code TEXT,
|
|
42
|
-
retryable INTEGER,
|
|
43
|
-
evidence_source TEXT NOT NULL,
|
|
44
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
45
|
-
)
|
|
46
|
-
`),await t.execute("CREATE INDEX IF NOT EXISTS service_connection_health_events_connection ON service_connection_health_events(connection_id, created_at DESC)"),await t.execute(`
|
|
47
|
-
CREATE TABLE IF NOT EXISTS service_connection_action_audit (
|
|
48
|
-
id TEXT PRIMARY KEY,
|
|
49
|
-
connection_id TEXT NOT NULL REFERENCES service_connections(id) ON DELETE CASCADE,
|
|
50
|
-
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
51
|
-
tool TEXT NOT NULL,
|
|
52
|
-
request_id TEXT NOT NULL,
|
|
53
|
-
status TEXT NOT NULL,
|
|
54
|
-
error_code TEXT,
|
|
55
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
56
|
-
UNIQUE(user_id, request_id)
|
|
57
|
-
)
|
|
58
|
-
`);try{await t.execute("ALTER TABLE service_connection_action_audit ADD COLUMN request_digest TEXT")}catch{}try{await t.execute("ALTER TABLE service_connection_action_audit ADD COLUMN result_json TEXT")}catch{}})().catch(t=>{throw a=null,u=null,t})),a}function f(e){if(typeof e!="string")return[];try{let t=JSON.parse(e);return Array.isArray(t)?t.filter(n=>typeof n=="string"):[]}catch{return[]}}function g(e){return{id:String(e.id),userId:Number(e.user_id),providerConfigKey:String(e.provider_config_key),provider:typeof e.provider=="string"?e.provider:null,transport:e.transport==="remote_mcp"?"remote_mcp":e.transport==="credential_reference"?"credential_reference":"nango",upstreamConnectionId:typeof e.upstream_connection_id=="string"?e.upstream_connection_id:null,label:typeof e.label=="string"?e.label:null,userLabel:typeof e.user_label=="string"?e.user_label:null,providerAccountId:typeof e.provider_account_id=="string"?e.provider_account_id:null,providerAccountEmail:typeof e.provider_account_email=="string"?e.provider_account_email:null,providerAccountName:typeof e.provider_account_name=="string"?e.provider_account_name:null,providerIdentityCheckedAt:typeof e.provider_identity_checked_at=="string"?e.provider_identity_checked_at:null,providerIdentitySourceUpdatedAt:typeof e.provider_identity_source_updated_at=="string"?e.provider_identity_source_updated_at:null,lifecycleStatus:String(e.lifecycle_status),operationalStatus:String(e.operational_status),reconnectRequired:Number(e.reconnect_required)===1,actionsEnabled:Number(e.actions_enabled)===1,readTools:f(e.read_tools_json),actionTools:f(e.action_tools_json),toolRevision:typeof e.tool_revision=="string"?e.tool_revision:null,lastCheckedAt:typeof e.last_checked_at=="string"?e.last_checked_at:null,lastSuccessfulCallAt:typeof e.last_successful_call_at=="string"?e.last_successful_call_at:null,lastFailureAt:typeof e.last_failure_at=="string"?e.last_failure_at:null,lastFailureCode:typeof e.last_failure_code=="string"?e.last_failure_code:null,lastFailureRetryable:e.last_failure_retryable===null||e.last_failure_retryable===void 0?null:Number(e.last_failure_retryable)===1,consecutiveFailures:Number(e.consecutive_failures??0),vaultName:typeof e.vault_name=="string"?e.vault_name:null,tableName:typeof e.table_name=="string"?e.table_name:null,createdAt:String(e.created_at),updatedAt:String(e.updated_at)}}async function E(e){let t=await p(e.normalize("NFKC").trim().toLowerCase());if(!t)throw new Error("service_connection_user_not_found");return Number(t.id)}async function D(e,t){await c();let n=o(),r=await E(e);for(let i of t)await n.execute({sql:`
|
|
59
|
-
INSERT INTO service_connections (
|
|
60
|
-
id, user_id, provider_config_key, provider, transport, upstream_connection_id,
|
|
61
|
-
label, lifecycle_status, reconnect_required, actions_enabled, legacy_owner_identity, created_at, updated_at
|
|
62
|
-
) VALUES (?, ?, ?, ?, 'nango', ?, ?, ?, ?, 1, ?, COALESCE(?, datetime('now')), COALESCE(?, datetime('now')))
|
|
63
|
-
ON CONFLICT(user_id, provider_config_key, upstream_connection_id) DO UPDATE SET
|
|
64
|
-
provider = excluded.provider,
|
|
65
|
-
label = COALESCE(excluded.label, service_connections.label),
|
|
66
|
-
lifecycle_status = excluded.lifecycle_status,
|
|
67
|
-
reconnect_required = excluded.reconnect_required,
|
|
68
|
-
updated_at = excluded.updated_at
|
|
69
|
-
`,args:[_(),r,i.providerConfigKey,i.provider,i.upstreamConnectionId,i.label,i.lifecycleStatus,i.reconnectRequired?1:0,e,i.createdAt,i.updatedAt]});return A(e,"nango")}async function A(e,t){await c();let n=await E(e);return(await o().execute({sql:`SELECT * FROM service_connections WHERE user_id = ?${t?" AND transport = ?":""} AND lifecycle_status <> 'disconnected' ORDER BY updated_at DESC`,args:t?[n,t]:[n]})).rows.map(i=>g(i))}async function d(e,t){await c();let n=await E(e),i=(await o().execute({sql:"SELECT * FROM service_connections WHERE id = ? AND user_id = ? LIMIT 1",args:[t,n]})).rows[0];return i?g(i):null}async function w(e,t,n,r){await c(),await o().execute({sql:"UPDATE service_connections SET read_tools_json = ?, action_tools_json = ?, tool_revision = ?, updated_at = datetime('now') WHERE id = ?",args:[JSON.stringify(t),JSON.stringify(n),r,e]})}async function U(e){await c(),await o().execute({sql:`
|
|
70
|
-
UPDATE service_connections SET
|
|
71
|
-
provider_account_id = ?, provider_account_email = ?, provider_account_name = ?,
|
|
72
|
-
provider_identity_checked_at = datetime('now'), provider_identity_source_updated_at = ?,
|
|
73
|
-
updated_at = datetime('now')
|
|
74
|
-
WHERE id = ?
|
|
75
|
-
`,args:[e.providerAccountId??null,e.providerAccountEmail?.toLowerCase()??null,e.providerAccountName??null,e.sourceUpdatedAt,e.connectionId]})}async function h(e,t,n){let r=await d(e,t);if(!r||r.lifecycleStatus==="disconnected")throw new Error("service_connection_not_found");return await o().execute({sql:"UPDATE service_connections SET user_label = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[n,t,r.userId]}),n}async function X(e){await c();let t=new Date().toISOString();await o().execute({sql:`
|
|
76
|
-
UPDATE service_connections SET
|
|
77
|
-
operational_status = ?, last_checked_at = ?,
|
|
78
|
-
last_successful_call_at = CASE WHEN ? = 'available' THEN ? ELSE last_successful_call_at END,
|
|
79
|
-
last_failure_at = CASE WHEN ? IN ('degraded', 'unavailable') THEN ? ELSE NULL END,
|
|
80
|
-
last_failure_code = ?, last_failure_retryable = ?,
|
|
81
|
-
consecutive_failures = CASE WHEN ? = 'available' THEN 0 ELSE consecutive_failures + 1 END,
|
|
82
|
-
updated_at = datetime('now')
|
|
83
|
-
WHERE id = ?
|
|
84
|
-
`,args:[e.operationalStatus,t,e.operationalStatus,t,e.operationalStatus,t,e.failureCode??null,e.retryable==null?null:e.retryable?1:0,e.operationalStatus,e.connectionId]}),await o().execute({sql:"INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)",args:[_(),e.connectionId,e.operationalStatus,e.failureCode??null,e.retryable==null?null:e.retryable?1:0,e.evidenceSource]})}async function q(e,t,n){let r=await d(e,t);if(!r||r.lifecycleStatus==="disconnected")throw new Error("service_connection_not_found");return await o().execute({sql:"UPDATE service_connections SET actions_enabled = ?, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[n?1:0,t,r.userId]}),n}async function k(e,t){let n=await d(e,t);if(!n)throw new Error("service_connection_not_found");await o().execute({sql:"UPDATE service_connections SET lifecycle_status = 'disconnected', operational_status = 'unknown', reconnect_required = 0, updated_at = datetime('now') WHERE id = ? AND user_id = ?",args:[t,n.userId]});let r=await o().execute({sql:"SELECT COUNT(*) AS count FROM service_connections WHERE user_id = ? AND transport = 'nango' AND lifecycle_status = 'connected'",args:[n.userId]});return Number(r.rows[0]?.count??0)}async function F(e){await c();let t=await d(e.identity,e.connectionId);if(!t)throw new Error("service_connection_not_found");await o().execute({sql:"UPDATE service_connection_action_audit SET status = ?, error_code = ?, result_json = ? WHERE user_id = ? AND request_id = ? AND connection_id = ?",args:[e.status,e.errorCode??null,e.result===void 0?null:JSON.stringify(e.result),t.userId,e.requestId,t.id]})}async function P(e){await c();let t=await d(e.identity,e.connectionId);if(!t)throw new Error("service_connection_not_found");let n=await o().execute({sql:"INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)",args:[_(),t.id,t.userId,e.tool,e.requestId,e.requestDigest]});if(Number(n.rowsAffected??0)===1)return{claimed:!0};let i=(await o().execute({sql:"SELECT connection_id, tool, status, error_code, request_digest, result_json FROM service_connection_action_audit WHERE user_id = ? AND request_id = ? LIMIT 1",args:[t.userId,e.requestId]})).rows[0],s=String(i?.connection_id??"")!==t.id||String(i?.tool??"")!==e.tool||String(i?.request_digest??"")!==e.requestDigest,l;if(!s&&typeof i?.result_json=="string")try{l=JSON.parse(i.result_json)}catch{l=void 0}return{claimed:!1,status:String(i?.status??"unknown"),errorCode:typeof i?.error_code=="string"?i.error_code:null,conflict:s,...l!==void 0?{result:l}:{}}}export{S as a,L as b,I as c,x as d,c as e,D as f,A as g,d as h,w as i,U as j,h as k,X as l,q as m,k as n,F as o,P as p};
|
package/dist/chunk-TMB56NCA.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{createHash as M}from"crypto";import{mkdir as D,readFile as $,writeFile as O}from"fs/promises";import{createWriteStream as C}from"fs";import{homedir as N}from"os";import{dirname as _,join as x}from"path";import{Transform as K}from"stream";import{pipeline as E}from"stream/promises";var b="9999-12-31T23:59:59.999Z";function L(){return process.env.VERCEL==="1"||process.env.NODE_ENV==="production"}function T(t){return t.hosted??L()}function v(t){return t.localBaseDir?.trim()||process.env.MCP_SCRAPER_OUTPUT_DIR?.trim()||x(N(),"Downloads","mcp-scraper")}function P(t){let e=t.replace(/^\/+/,"").replace(/\/+$/,"");if(!e||e.includes("..")||!/^[a-zA-Z0-9/_-]+$/.test(e))throw new Error("private artifact prefix is invalid");return`${e}/`}function y(t,e){let r=t.trim();if(!/^[a-zA-Z0-9_-]{1,160}$/.test(r))throw new Error(`${e} must contain only letters, numbers, underscores, or hyphens`);return r}function U(t){let e=t.trim();if(!/^[a-zA-Z0-9_.-]{1,200}$/.test(e)||e==="."||e==="..")throw new Error("private artifact key is invalid");return e}function k(t){return(t.trim().replace(/[^a-zA-Z0-9_.-]+/g,"-").replace(/^-+|-+$/g,"")||"artifact").slice(0,160)}function R(t){let e=typeof t=="string"&&/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(t)?`${t.replace(" ","T")}Z`:t,r=e instanceof Date?e.getTime():typeof e=="number"?e:new Date(e).getTime();if(!Number.isSafeInteger(r)||r<=0)throw new Error("private artifact createdAt is invalid");return r}function I(t){if(t.artifactTtlMs!==null&&(!Number.isSafeInteger(t.artifactTtlMs)||t.artifactTtlMs<=0))throw new Error("private artifact TTL must be a positive integer");if(!Number.isSafeInteger(t.downloadTtlMs)||t.downloadTtlMs<=0)throw new Error("private artifact download TTL must be a positive integer");return{prefix:P(t.prefix),artifactTtlMs:t.artifactTtlMs,downloadTtlMs:t.downloadTtlMs}}function W(t,e){let r=P(e);if(!t.startsWith(r))return null;let n=(t.split("/").at(-1)??"").match(/^(\d{13})-/);if(!n)return null;let o=Number(n[1]);return Number.isSafeInteger(o)&&o>0?o:null}function g(t,e){let r=P(e);if(!t.startsWith(r)||t.includes(".."))return null;let n=t.slice(r.length).split("/");if(n.length<2)return null;try{return y(n[0]??"","ownerId")}catch{return null}}function S(t,e){let r=I(e);if(g(t,r.prefix)===null||r.artifactTtlMs===null)return null;let i=W(t,r.prefix);return i===null?null:new Date(i+r.artifactTtlMs)}async function B(t,e,r){let i=r.token?.trim();if(!i)return null;let n=e?Math.min(Date.now()+r.downloadTtlMs,e.getTime()):Date.now()+r.downloadTtlMs;if(n<=Date.now())return null;let{issueSignedToken:o,presignUrl:c}=await import("@vercel/blob"),a=await o({token:i,pathname:t,operations:["get"],validUntil:n}),{presignedUrl:u}=await c(a,{access:"private",operation:"get",pathname:t,validUntil:n});return{url:u,expiresAt:new Date(n).toISOString()}}async function J(t){let e={...t.policy,...I(t.policy)},r=y(t.ownerId,"ownerId"),i=U(t.artifactKey),n=(t.scopeSegments??[]).map((f,l)=>y(f,`scopeSegments[${l}]`)),o=R(t.createdAt),c=`${e.prefix}${r}/${n.length?`${n.join("/")}/`:""}${o}-${i}`,a=Buffer.isBuffer(t.content)?t.content:Buffer.from(t.content),u=a.length,A=M("sha256").update(a).digest("hex"),m=e.artifactTtlMs===null?null:new Date(o+e.artifactTtlMs),d=e.token?.trim()||null,p=c;if(d){let{put:f}=await import("@vercel/blob");p=(await f(c,a,{access:"private",token:d,contentType:t.contentType,addRandomSuffix:!1,allowOverwrite:!0,cacheControlMaxAge:60,multipart:u>100*1024*1024})).pathname}else{if(T(e))throw new Error("private_artifact_blob_not_configured");let f=x(v(e),"blobs",c);await D(_(f),{recursive:!0}),await O(f,a)}let h=await B(p,m,e);return{artifactId:p,filename:k(t.filename),contentType:t.contentType,bytes:u,sha256:A,expiresAt:m?.toISOString()??b,downloadUrl:h?.url??null,downloadUrlExpiresAt:h?.expiresAt??null}}async function Q(t){let e={...t.policy,...I(t.policy)},r=y(t.ownerId,"ownerId"),i=U(t.artifactKey),n=(t.scopeSegments??[]).map((l,w)=>y(l,`scopeSegments[${w}]`)),o=R(t.createdAt),c=`${e.prefix}${r}/${n.length?`${n.join("/")}/`:""}${o}-${i}`,a=e.artifactTtlMs===null?null:new Date(o+e.artifactTtlMs),u=e.token?.trim()||null,A=M("sha256"),m=0,d=new K({transform(l,w,s){m+=l.length,A.update(l),s(null,l)}}),p=c;if(u){let{put:l}=await import("@vercel/blob"),w=E(t.content,d);try{let s=await l(c,d,{access:"private",token:u,contentType:t.contentType,addRandomSuffix:!1,allowOverwrite:!0,cacheControlMaxAge:60,multipart:!0});await w,p=s.pathname}catch(s){throw t.content.destroy(s instanceof Error?s:new Error(String(s))),d.destroy(s instanceof Error?s:new Error(String(s))),await w.catch(()=>{}),s}}else{if(T(e))throw new Error("private_artifact_blob_not_configured");let l=x(v(e),"blobs",c);await D(_(l),{recursive:!0}),await E(t.content,d,C(l))}let h=A.digest("hex"),f=await B(p,a,e);return{artifactId:p,filename:k(t.filename),contentType:t.contentType,bytes:m,sha256:h,expiresAt:a?.toISOString()??b,downloadUrl:f?.url??null,downloadUrlExpiresAt:f?.expiresAt??null}}async function Y(t){if(g(t.artifactId,t.policy.prefix)!==t.ownerId)return null;let e=S(t.artifactId,t.policy);if(t.policy.artifactTtlMs!==null&&(!e||e.getTime()<=Date.now()))return null;let r=await B(t.artifactId,e,t.policy);return r?{downloadUrl:r.url,downloadUrlExpiresAt:r.expiresAt,expiresAt:e?.toISOString()??b}:null}async function z(t){let e=t.getReader(),r=[];try{for(;;){let{done:i,value:n}=await e.read();if(i)break;n&&r.push(Buffer.from(n))}}finally{e.releaseLock()}return Buffer.concat(r)}async function tt(t){if(g(t.artifactId,t.policy.prefix)===null)return null;let e=S(t.artifactId,t.policy);if(t.policy.artifactTtlMs!==null&&(!e||e.getTime()<=Date.now()))return null;if(!Number.isSafeInteger(t.offset)||t.offset<0)throw new Error("private artifact offset is invalid");if(!Number.isSafeInteger(t.maxBytes)||t.maxBytes<=0)throw new Error("private artifact maxBytes is invalid");let r=t.policy.token?.trim()||null,i;if(r){let{get:c}=await import("@vercel/blob"),a=await c(t.artifactId,{access:"private",token:r,useCache:!1});if(!a||a.statusCode!==200)return null;i=await z(a.stream)}else{if(T(t.policy))return null;try{i=await $(x(v(t.policy),"blobs",t.artifactId))}catch{return null}}let n=i.length,o=Math.min(n,t.offset+t.maxBytes);return{text:i.subarray(t.offset,o).toString("utf8"),totalBytes:n,nextOffset:o<n?o:null}}async function et(t){if(g(t.artifactId,t.policy.prefix)===null)return null;let e=S(t.artifactId,t.policy);if(t.policy.artifactTtlMs!==null&&(!e||e.getTime()<=Date.now()))return null;if(!Number.isSafeInteger(t.maxBytes)||t.maxBytes<=0)throw new Error("private artifact maxBytes is invalid");let r=t.policy.token?.trim()||null,i;if(r){let{get:n}=await import("@vercel/blob"),o=await n(t.artifactId,{access:"private",token:r,useCache:!1});if(!o||o.statusCode!==200)return null;i=await z(o.stream)}else{if(T(t.policy))return null;try{i=await $(x(v(t.policy),"blobs",t.artifactId))}catch{return null}}if(i.length>t.maxBytes)throw new Error("private artifact exceeds read limit");return i}export{g as a,J as b,Q as c,Y as d,tt as e,et as f};
|
package/dist/chunk-VXOXB3IB.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
var o="0.95.2";export{o as a};
|
package/dist/chunk-WO3N5FH2.js
DELETED
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import{a as P,c as S}from"./chunk-HE45FFBU.js";var N=20,U=20,H=10,j=20,E=280,q=/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi,D=[".png",".jpg",".jpeg",".gif",".svg",".webp",".ico","@2x","sentry","wix.com","wixpress","example.","godaddy","squarespace","schema.org","u003e","core-js"],W=/\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i,z=/(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i,Y={"facebook.com":"facebook","fb.com":"facebook","instagram.com":"instagram","linkedin.com":"linkedin","twitter.com":"twitter","x.com":"x","youtube.com":"youtube","youtu.be":"youtube","tiktok.com":"tiktok","pinterest.com":"pinterest"},F=/(?:\/shar(?:e|er)|\/intent\/|\/dialog\/|\/plugins\/|\/share\?|\/developers?\/|platform\.)/i;function x(t){return(t??"").replace(/\s+/g," ").trim()}function _(t,e){try{let r=new URL(t,e);return!["http:","https:"].includes(r.protocol)||r.username||r.password?null:r}catch{return null}}function g(t){let e=t;try{e=decodeURIComponent(t)}catch{}let r=e.split("?")[0].trim().toLowerCase().replace(/[),.;:]+$/,"");return r.length>254||r.split("@").length!==2||!/^[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9.-]+\.[a-z]{2,}$/i.test(r)||D.some(n=>r.includes(n))?null:r}function f(t){let e=t.closest("p,li,address,section,article,div").first();return x(e.length?e.text():t.text()).slice(0,E)}function k(t,e,r,n){let s=g(e);!s||W.test(n)||t.has(s)||t.set(s,{value:s,sourceUrl:r,context:x(n).slice(0,E),sourceType:"first_party_page"})}function G(t){t.hash="";for(let e of[...t.searchParams.keys()])/^(?:utm_|fbclid|gclid)/i.test(e)&&t.searchParams.delete(e);return t.href}function J(t){let e=t.toLowerCase().replace(/^www\./,"");return Object.entries(Y).find(([n])=>e===n||e.endsWith(`.${n}`))?.[1]??null}function V(t,e){let r=_(e);if(!r)throw new Error("pageUrl must be a public HTTP(S) URL");let{$:n}=S(t),s=new Map,p=new Map,o=new Map,c=new Map;return n('a[href^="mailto:" i]').each((u,a)=>{let i=n(a);k(s,(i.attr("href")??"").slice(7),r.href,f(i))}),n("body").find("*").addBack().contents().filter((u,a)=>a.type==="text").each((u,a)=>{let i=n(a.parent??a),l=String(a.data??"");for(let d of l.match(q)??[])k(s,d,r.href,f(i))}),n("a[href]").each((u,a)=>{let i=n(a),l=_(i.attr("href")??"",r.href);if(!l)return;let d=x(`${i.text()} ${f(i)}`).slice(0,E),T=J(l.hostname);if(T&&!F.test(`${l.pathname}${l.search}`)){let v=G(l);p.has(v)||p.set(v,{url:v,network:T,sourceUrl:r.href,context:d});return}if(l.origin!==r.origin||!z.test(l.pathname))return;l.hash="";let y=l.href;c.has(y)||c.set(y,{url:y,sourceUrl:r.href,context:d})}),n("form").each((u,a)=>{let i=n(a),l=_(i.attr("action")||r.href,r.href);if(!l||l.origin!==r.origin)return;l.hash="";let d=l.href;o.has(d)||o.set(d,{url:d,sourceUrl:r.href,context:f(i)})}),{emails:[...s.values()].slice(0,N),socialProfiles:[...p.values()].slice(0,U),contactForms:[...o.values()].slice(0,H),candidatePageUrls:[...c.values()].slice(0,j)}}import{createHash as K}from"crypto";var I="3.0.0",w="analytics_setup_site",B="Install X-Ray",R="Install X-Ray and tell me what drives customers",m=`Call ${w} with { "startUrl": "<site URL>", "goal": "${R}" }; on every later call, reuse the returned setupId and revision.`,C=JSON.stringify({mcpServers:{"mcp-scraper":{command:"npx",args:["-y","--package","mcp-scraper@latest","mcp-scraper"],env:{MCP_SCRAPER_API_KEY:"${env:MCP_SCRAPER_API_KEY}"}}}},null,2),Z=JSON.stringify({mcpServers:{"mcp-scraper":{url:"https://mcpscraper.dev/mcp"}}},null,2),O={schema:"https://mcpscraper.dev/schemas/xray-install-contract-v1.json",protocolVersion:I,product:"Thorbit X-Ray",triggerPhrases:[B,"Install X-Ray on this site and tell me what drives customers"],canonicalUrls:{markdown:"https://mcpscraper.dev/agent-install.md",json:"https://mcpscraper.dev/.well-known/xray-install.json",hostedMcp:"https://mcpscraper.dev/mcp"},facade:{tool:w,initialCall:{startUrl:"<site URL>",goal:R},continuationRule:"Treat setupId plus revision as the durable continuation handle. Reuse both after approval, runtime publication, verification, retry, or return."},topologies:{local_stdio:{label:"Local npm MCP process",authority:"The client starts MCP Scraper locally. After the one-time Pixel insertion, X-Ray setup publishes only owner-confirmed runtime configuration.",secretHandling:"Set MCP_SCRAPER_API_KEY in the local environment or client secret field. Never paste it into chat or commit it."},hosted_remote:{label:"Hosted authenticated MCP",authority:"The hosted MCP inspects approved public pages and publishes owner-confirmed X-Ray runtime configuration. It cannot read or write a caller-local path.",secretHandling:"Use the MCP OAuth/connection flow. Never put a credential in a URL, prompt, or config shown in chat."}},hosts:[{id:"codex",label:"Codex",supportedTopologies:["local_stdio","hosted_remote"],mcpConfigMode:"toml",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:`[mcp_servers.mcp-scraper]
|
|
2
|
-
command = "npx"
|
|
3
|
-
args = ["-y", "--package", "mcp-scraper@latest", "mcp-scraper"]
|
|
4
|
-
env_vars = ["MCP_SCRAPER_API_KEY"]`,continuation:m,rollback:"Remove the mcp_servers.mcp-scraper table, then restart Codex. Pixel removal is a separate owner-controlled site action."},{id:"claude-code",label:"Claude Code",supportedTopologies:["local_stdio","hosted_remote"],mcpConfigMode:"command",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:'claude mcp add mcp-scraper --scope user --env MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" -- npx -y --package mcp-scraper@latest mcp-scraper',continuation:m,rollback:"Run: claude mcp remove mcp-scraper -s user. Pixel removal is a separate owner-controlled site action."},{id:"claude-desktop",label:"Claude Desktop",supportedTopologies:["local_stdio"],mcpConfigMode:"desktop_extension",workspaceRead:!1,workspaceWrite:!1,restartRequired:!0,exactConfig:"Install https://mcpscraper.dev/downloads/mcp-scraper.mcpb and enter the API key only in Claude Desktop's protected configuration field.",continuation:m,rollback:"Remove the MCP Scraper extension in Claude Desktop. Pixel removal is a separate owner-controlled site action."},{id:"cursor",label:"Cursor",supportedTopologies:["local_stdio","hosted_remote"],mcpConfigMode:"json",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:C,continuation:m,rollback:"Remove mcp-scraper from .cursor/mcp.json (or the user MCP settings), then restart Cursor."},{id:"windsurf",label:"Windsurf",supportedTopologies:["local_stdio"],mcpConfigMode:"json",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:C,continuation:m,rollback:"Remove mcp-scraper from the Windsurf MCP configuration, then restart Windsurf."},{id:"cline",label:"Cline",supportedTopologies:["local_stdio"],mcpConfigMode:"json",workspaceRead:!0,workspaceWrite:!0,restartRequired:!0,exactConfig:C,continuation:m,rollback:"Remove mcp-scraper from Cline MCP Servers, then restart the extension host."},{id:"user-action-only",label:"User-action-only MCP host",supportedTopologies:["hosted_remote"],mcpConfigMode:"handoff_only",workspaceRead:!1,workspaceWrite:!1,restartRequired:!1,exactConfig:Z,continuation:m,rollback:"Disconnect MCP Scraper from the host. Pixel removal remains an owner-controlled site action."}],tagBootstrap:{requirement:"one_global_pixel_once",ongoingConfiguration:"xray_runtime_manifest",sourceAccessRequiredAfterInstall:!1,supportedInsertionPaths:["manual","tag_manager","cms_plugin","coding_host"]},workflow:["Resolve Thorbit X-Ray entitlement and the target site without consuming MCP Scraper Credits.","Inspect the whole approved public site; enumerate distinct renderers, routes, and Pixel coverage.","Infer the business model and customer outcome, then propose one privacy-safe measurement plan.","If the Pixel is absent, return the canonical one-time tag and exact uncovered surfaces; do not claim ongoing control there.","Publish an approved measurement revision as a digest-bound runtime manifest; source or repository access is not required.","For SaaS, verify the measurement foundation: global tag coverage, ad-source capture, lifecycle events, currency and timezone, consent, and observed outcome events.","Optionally publish a structured post-purchase survey after approved disclosure evidence; reported influence remains separate from deterministic touches.","Compare observed attribution with customer-reported influence only with explicit weights totaling 1,000,000 micros and a named methodology version.","Keep view-through reporting unavailable until event-level provider evidence can join to an eligible X-Ray subject; aggregate impressions never qualify.","Verify route coverage, consent, PageViews, manifest convergence, bounded rule matches, and persisted definition/version evidence.","Require a separate exact-revision approval before any canary or real-visitor experiment activation.","Return first customer-journey answers when sufficient, or exact evidence-readiness thresholds when not.","On \u201Cimprove\u201D, propose one highest-value uncovered outcome, require approval when needed, and reverify."],safetyRules:["Never invent or hand-author tracking code; use the canonical product-authored Pixel snippet returned by analytics_setup_site.","Never ask for or print API keys, OAuth tokens, signed URLs, cookies, or other credentials.","Never call ad platforms directly or silently activate events, definitions, funnels, or delivery destinations.","Never treat a CTA click as a customer, sale, or revenue event without verified outcome evidence.","Never claim full-domain installation while an in-scope route or renderer is missing, excluded without approval, or unverifiable.","Never treat event-name presence as verification; require persisted definition ID and version evidence.","Never accept survey invitation tokens or exchange codes through MCP inputs or prompts.","Never treat customer-reported influence as identity, CRM mutation authority, ad-activation evidence, or a deterministic touch."],terminalStates:["complete","tag_required","input_required","approval_required","runtime_convergence_required","not_ready","error"]};function b(t){return Array.isArray(t)?`[${t.map(b).join(",")}]`:t&&typeof t=="object"?`{${Object.entries(t).sort(([e],[r])=>e.localeCompare(r)).map(([e,r])=>`${JSON.stringify(e)}:${b(r)}`).join(",")}}`:JSON.stringify(t)}function Q(){return b(O)}var L=`sha256:${K("sha256").update(Q(),"utf8").digest("hex")}`,X={...O,protocolDigest:L};function fe(t){let e=X.hosts.find(r=>r.id===t);if(!e)throw new Error(`Unknown X-Ray host: ${t}`);return e}function ge(){return["# Install X-Ray","",`Protocol: ${I} (${L})`,"",`Use ${w} as the single conversational setup facade. Ask for the target site URL if it is missing, then call it with the goal \u201C${R}\u201D.`,"","Continue the same setup with its setupId and revision. Confirm one global Pixel across all public rendering surfaces; present one measurement plan; publish it as a digest-bound runtime manifest after exact-revision approval; verify manifest and rule versions; then return first customer-journey answers or explicit readiness thresholds. For SaaS, verify the provider-neutral measurement foundation. Browser events never prove paid revenue. GitHub or repository access is not required after the tag is installed.","","Do not request pasted credentials, invent tracking code, silently activate measurement, call ad platforms directly, or claim customers/revenue from CTA clicks."].join(`
|
|
5
|
-
`)}function he(t=X){return JSON.parse(JSON.stringify(t))}var ee=new Set(["admin","contact","customerservice","hello","help","info","inquiries","mail","office","reception","sales","service","support","team"]),te=/(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i,re=/\b(?:our team|meet the team|staff|agents?|brokers?|realtors?|people)\b/i,oe=/\b(?:brokerage|company|corporate|front desk|main office|office|organization|reception)\b/i;function M(t){return t.toLowerCase().replace(/[^a-z0-9]+/g," ").trim().split(/\s+/).filter(Boolean)}function A(t,e){try{let r=new URL(t,e);return!["http:","https:"].includes(r.protocol)||r.username||r.password?null:r}catch{return null}}function ne(t,e){let r=t.value.split("@")[0].toLowerCase().replace(/[^a-z0-9]+/g," "),n=M(e.name).filter(a=>a.length>1),s=A(t.sourceUrl)?.pathname.toLowerCase()??"",p=`${t.context} ${s}`.toLowerCase(),o=ee.has(r.replace(/\s+/g,"")),c=n.filter(a=>new RegExp(`(?:^|[^a-z0-9])${a}(?:$|[^a-z0-9])`,"i").test(`${r} ${p}`)).length,u=e.entityType==="person"&&n.length>=2&&c>=Math.min(2,n.length)&&(r.includes(n[0])||s.includes(n.join("-"))||s.includes(n.join("/")));return!o&&u?{scope:"person",reasons:["named_person_page_context"]}:o||oe.test(p)?{scope:"office",reasons:o?["generic_mailbox_prefix"]:["organization_page_context"]}:re.test(p)?{scope:"team",reasons:["team_page_context"]}:e.entityType==="business"&&M(e.organization||e.name).filter(i=>i.length>2).some(i=>p.includes(i))?{scope:"team",reasons:["business_context"]}:{scope:"unknown",reasons:["public_occurrence_without_identity_evidence"]}}function ve(t,e){let r=t.map(o=>({...o,...ne(o,e)})),n=e.entityType==="person"?["person","team"]:["person","team","office"],s=["provided","first_party_page","structured_data","serp_snippet"],p=["person","team","office"];return r.filter(o=>n.includes(o.scope)).sort((o,c)=>s.indexOf(o.sourceType)-s.indexOf(c.sourceType)||p.indexOf(o.scope)-p.indexOf(c.scope))[0]??null}function _e(t,e,r){if(!Number.isInteger(r)||r<1||r>4)throw new Error("maxPages must be an integer from 1 to 4");let n=A(t);if(!n)return[];n.hash="";let s=new Set([n.href]),p=e.map(o=>typeof o=="string"?o:o.url).map(o=>A(o,n.href)).filter(o=>!!(o&&o.origin===n.origin&&te.test(o.pathname)));for(let o of p)o.hash="",s.add(o.href);for(let o of["/contact","/contact-us"])s.add(new URL(o,n.origin).href);return[...s].slice(0,r)}var se=/[A-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi,ie=/\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i,ae=280,ce=10;function h(t){return t.toLowerCase().replace(/^www\./,"").replace(/\.$/,"")}function le(t){let e=h(t);return e==="localhost"||e.endsWith(".localhost")||e.endsWith(".local")||!e.includes(".")&&!e.includes(":")}function $(t){try{let e=new URL(t.trim());return!["http:","https:"].includes(e.protocol)||e.username||e.password||le(e.hostname)||P(e.hostname)?null:e}catch{return null}}function pe(t,e){return`${t} ${e??""}`.replace(/\s+/g," ").trim().slice(0,ae)}function we(t){let e=$(t);return e?(e.hostname=h(e.hostname),`email for ${e.protocol}//${e.host}/`):null}function Re({websiteUrl:t,results:e}){let r=$(t);if(!r)return[];let n=h(r.hostname),s=new Map,p=e.map((o,c)=>({result:o,index:c})).sort((o,c)=>{let u=Number.isFinite(o.result.position)?o.result.position:Number.MAX_SAFE_INTEGER,a=Number.isFinite(c.result.position)?c.result.position:Number.MAX_SAFE_INTEGER;return u-a||o.index-c.index});for(let{result:o}of p){let c=`${o.title} ${o.snippet??""}`,u=pe(o.title,o.snippet);for(let a of c.match(se)??[]){let i=g(a);!i||ie.test(c)||h(i.slice(i.lastIndexOf("@")+1))!==n||s.has(i)||s.set(i,{value:i,sourceUrl:o.url,sourceTitle:o.title,sourcePosition:o.position,context:u,sourceType:"serp_snippet"})}}return[...s.values()].slice(0,ce)}export{V as a,ne as b,ve as c,_e as d,we as e,Re as f,I as g,w as h,R as i,L as j,X as k,fe as l,ge as m,he as n};
|
package/dist/chunk-WO46LTBW.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{g as S,i as b}from"./chunk-7YOXSMHG.js";import{createHash as C}from"crypto";var P=["observed_present","observed_absent","incomplete","not_requested","unknown","unsupported"],A=["provider_transport","invalid_payload","serp_readiness","query_integrity","pagination","optional_feature_parsing","unknown"],v=class extends Error{stage;failureCode;constructor(r,n,t){super(t),this.name="SerpExtractionError",this.stage=r,this.failureCode=n}};function K(e){return e instanceof v?e.stage:null}function p(e){return C("sha256").update(e).digest("hex")}function I(e){return e.normalize("NFKC").toLocaleLowerCase("en-US").split(/[^\p{L}\p{N}]+/u).filter(Boolean).slice(0,64)}function W(e){try{let r=new URL(e);return r.hash="",r.pathname.length>1&&(r.pathname=r.pathname.replace(/\/+$/,"")),r.toString()}catch{return e.trim()}}function Z(e,r){let n=e===2?[1,2]:[1],t=r.filter(o=>o.attempted).map(o=>o.page),s=r.filter(o=>o.captured&&o.resultCount>0).map(o=>o.page),a=r.find(o=>o.failureCode);return{status:n.every(o=>s.includes(o))?"complete":"incomplete",requestedPages:n,attemptedPages:t,capturedPages:s,pageResultCounts:n.map(o=>({page:o,resultCount:r.find(g=>g.page===o)?.resultCount??0})),providerRequestCount:t.length,evidence:r,failureCode:a?.failureCode??null}}function $(e){return e.requested?e.moduleObserved?!e.rawShapeValid||e.rawCount>0&&e.mappedCount===0?"incomplete":e.mappedCount>0?"observed_present":"observed_absent":"unknown":"not_requested"}function X(e){let r=p(e.query),n=e.finalQuery==null?null:p(e.finalQuery),t=e.locationResolutionSource==="unresolved";return{status:n!==null&&n!==r?"query_mismatch":t?"unresolved_location":"matched",queryHash:r,normalizedQueryTokens:I(e.query),canonicalLocation:e.canonicalLocation??null,locationResolutionSource:e.locationResolutionSource,locationAmbiguousCandidates:Math.max(0,Math.trunc(e.locationAmbiguousCandidates??0)),uulePresent:!!e.uule,uuleHash:e.uule?p(e.uule):null,outboundProviderMethod:e.outboundProviderMethod,finalQueryHash:n}}function J(e){return p(JSON.stringify(Object.fromEntries(Object.entries(e).filter(([r,n])=>r!=="requestFingerprint"&&n!==void 0).sort(([r],[n])=>r.localeCompare(n)))))}function l(e){return e&&typeof e=="object"&&!Array.isArray(e)?e:{}}function m(e){return typeof e=="number"&&Number.isFinite(e)&&e>=0?Math.trunc(e):0}function u(e,r){return typeof e=="string"&&e.length<=r?e:null}function ee(e){let r=l(e),n=l(r.pagination),t=l(r.features),s=l(r.queryIntegrity),a=i=>Array.isArray(i)?i.filter(d=>d===1||d===2).slice(0,2):[],o=a(n.requestedPages);if(o.length===0)return null;let g=new Set(P),k=new Set(A);return{pagination:{status:n.status==="complete"?"complete":"incomplete",requestedPages:o,attemptedPages:a(n.attemptedPages),capturedPages:a(n.capturedPages),pageResultCounts:Array.isArray(n.pageResultCounts)?n.pageResultCounts.slice(0,2).map(i=>{let d=l(i);return{page:d.page===2?2:1,resultCount:m(d.resultCount)}}):[],providerRequestCount:m(n.providerRequestCount),failureCode:u(n.failureCode,100)},features:Object.fromEntries(["localPack","forums","videos","aiOverview","whatPeopleSaying","aiMode","paaPreview","entityIds"].map(i=>[i,g.has(String(t[i]))?t[i]:"unknown"])),queryIntegrity:{status:["matched","unresolved_location","query_mismatch"].includes(String(s.status))?s.status:"query_mismatch",queryHash:u(s.queryHash,64)??"",normalizedQueryTokens:Array.isArray(s.normalizedQueryTokens)?s.normalizedQueryTokens.slice(0,64).flatMap(i=>typeof i=="string"?[i.slice(0,100)]:[]):[],canonicalLocation:u(s.canonicalLocation,500),locationResolutionSource:u(s.locationResolutionSource,100)??"unknown",locationAmbiguousCandidates:m(s.locationAmbiguousCandidates),uulePresent:s.uulePresent===!0,uuleHash:u(s.uuleHash,64),outboundProviderMethod:"search",finalQueryHash:u(s.finalQueryHash,64)},failureStage:k.has(String(r.failureStage))?r.failureStage:null}}var T={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},f=/error|message/i,O=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),x=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,q=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),F=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),L=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function R(e){return e.replace(L,"[browser-service]")}var N=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),D=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),U=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),M=new Set(["provider_session_api","client_runtime","extractor","unknown"]),z=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),V=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),H=512,Q=320;function _(e,r){return typeof e=="string"&&r.has(e)?e:null}function j(e){return typeof e!="string"?null:z.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function B(e){return typeof e!="string"||e.length<1||e.length>H||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function y(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function h(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function w(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let n=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return R(n).slice(0,Q)}function E(e){return typeof e=="boolean"?e:null}function G(e){return{browser_provider:_(e.browser_provider,N),provider_session_id:B(e.provider_session_id),provider_session_observed_at:h(e.provider_session_observed_at),provider_lookup_status:_(e.provider_lookup_status,D),provider_status:_(e.provider_status,U),provider_error_code:j(e.provider_error_code),provider_error_message:w(e.provider_error_message),provider_reason_source:_(e.provider_reason_source,M),provider_duration_ms:y(e.provider_duration_ms),provider_bandwidth_bytes:y(e.provider_bandwidth_bytes),provider_captcha:E(e.provider_captcha),provider_checked_at:h(e.provider_checked_at),provider_disconnect_observed:E(e.provider_disconnect_observed),provider_disconnect_message:w(e.provider_disconnect_message)}}function c(e,r=""){if(typeof e=="string"){let n=R(e);return f.test(r)&&(O.test(e)||x.test(e))?n=S("service_unavailable"):f.test(r)&&(n=b(n)),n}if(Array.isArray(e))return e.map(n=>c(n,r));if(e!==null&&typeof e=="object"){let n={};for(let[t,s]of Object.entries(e)){if(q.has(t))continue;let a=T[t]??t;if(F.has(t)){n[a]=null;continue}n[a]=c(s,t)}return n}return e}function te(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return c(r);let n=r,t=c(n);return Object.keys(n).some(a=>V.has(a))?{...t,...G(n)}:t})}function oe(e){let r=e?.diagnostics;if(!r)return e;let n=r.serpCompleteness,t=n?.queryIntegrity;return!r.debug&&!t?e:{...e,diagnostics:{...r,...r.debug?{debug:c(r.debug)}:{},...t?{serpCompleteness:{...n,queryIntegrity:{...t,outboundProviderMethod:"search"}}}:{}}}}export{v as a,K as b,W as c,Z as d,$ as e,X as f,J as g,ee as h,c as i,te as j,oe as k};
|