mcp-scraper 0.96.1 → 0.96.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -1
- package/README.md +1 -1
- package/dist/actions-FKAX2PYT.js +1 -0
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/bundle-7JMACIIR.js +1 -0
- package/dist/chunk-3HXMW2UJ.js +1 -0
- package/dist/{chunk-RN4VG3VH.js → chunk-42SLBGDI.js} +1 -1
- package/dist/{chunk-JHT6A4U6.js → chunk-4KHDV5E7.js} +1 -1
- package/dist/{chunk-FB4KBB5W.js → chunk-7ZR5IO3B.js} +2 -2
- package/dist/{chunk-HUNSEGUN.js → chunk-ANQ4JJVI.js} +43 -29
- package/dist/{chunk-SE4VKLEC.js → chunk-CFZXZKFT.js} +1 -1
- package/dist/{chunk-6BD75VKS.js → chunk-DJT2547J.js} +1 -1
- package/dist/chunk-E7I3OQHQ.js +203 -0
- package/dist/{chunk-HVQZRQRJ.js → chunk-GLCR4KJI.js} +16 -37
- package/dist/chunk-JGGQ3UX5.js +1 -0
- package/dist/chunk-JREMJRSS.js +1 -0
- package/dist/chunk-K3VDGJNK.js +1 -0
- package/dist/{chunk-H6MODA7W.js → chunk-MJOXIDKF.js} +5 -5
- package/dist/chunk-OY24ERPV.js +8 -0
- package/dist/{chunk-7WPCGRD7.js → chunk-P55O4IDC.js} +41 -34
- package/dist/{chunk-ZTMQEDIB.js → chunk-PANT6VEM.js} +1 -1
- package/dist/chunk-PKQHHCO4.js +20 -0
- package/dist/{chunk-RIHI3KH7.js → chunk-QYPSFZK3.js} +1 -1
- package/dist/{chunk-BPKTUCV7.js → chunk-U7BZ5KTE.js} +1 -1
- package/dist/{chunk-SFUOSJ3A.js → chunk-WBJ6Y7WP.js} +2 -2
- package/dist/dispatch-E6VTNHHS.js +1 -0
- package/dist/index.cjs +10 -10
- package/dist/index.js +1 -1
- package/dist/metering-YHZ46GV4.js +1 -0
- package/dist/queue-4XHYUV7J.js +1 -0
- package/dist/reconciliation-43CSC3XD.js +1 -0
- package/dist/repository-UKDA4IWP.js +1 -0
- package/dist/{server-DXZ27KQO.js → server-74VGVXA4.js} +840 -802
- package/dist/settlement-I5IBIK6U.js +1 -0
- package/dist/worker-L5NISZQV.js +1 -0
- package/package.json +1 -2
- package/dist/actions-R4EJDLRO.js +0 -1
- package/dist/bundle-FCXZMIRB.js +0 -1
- package/dist/chunk-5BKSKLHX.js +0 -1
- package/dist/chunk-5LYINW7Z.js +0 -8
- package/dist/chunk-CORWWDHL.js +0 -1
- package/dist/chunk-CVQUHVHX.js +0 -1
- package/dist/chunk-IY6ZUUY7.js +0 -173
- package/dist/chunk-RYX4FJVH.js +0 -1
- package/dist/chunk-UXNR3K6I.js +0 -1
- package/dist/dispatch-P7IY75LH.js +0 -1
- package/dist/metering-ZH2P62IC.js +0 -1
- package/dist/queue-AC4XKD6Z.js +0 -1
- package/dist/reconciliation-UZUZ6B74.js +0 -1
- package/dist/repository-4EC5EXVN.js +0 -1
- package/dist/settlement-6CS4JZTY.js +0 -1
- package/dist/worker-ATRD6VKM.js +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,28 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.96.3] - 2026-10-13
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- Run background website extraction, audits, directory enrichment, lead enrichment, Sourcebook acquisition and video analysis in the existing request runtime. Preserve original jobs, payments, saved phase results and artifacts across bounded continuations without a separate workflow service.
|
|
12
|
+
- Remove Inngest runtime dependencies and endpoints. Deploy the server and scheduler together and retire their old Inngest registrations during cutover; a source change alone does not stop previously registered remote jobs.
|
|
13
|
+
- Make CTO reports describe the configured native job dispatch and stored queue outcomes instead of querying the retired workflow service.
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- Remove one-second database permission polling during browser acquisition while preserving deadline, cancellation, explicit ownership checks and cleanup. Keep PAA client progress checks at least 20 seconds apart.
|
|
18
|
+
- Recover accepted jobs interrupted before execution begins, without replacing their job or payment. Keep uncertain already-started work from being acquired again, and preserve the original settlement when execution permission is lost.
|
|
19
|
+
- Return prompt background-job handoff receipts so short cron and client requests do not wait for long-running video work.
|
|
20
|
+
- Bound manual Memory maintenance by the scheduler's request limit and resume saved progress. Protect built-in workflow routes with the existing scheduler credential and require the native runner for execution.
|
|
21
|
+
|
|
22
|
+
## [0.96.2] - 2026-10-13
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- Stop and drain PAA lease heartbeats before capturing and publishing completion, so a completed browser run can settle and report its saved result without falsely losing ownership.
|
|
27
|
+
- Refund the original customer hold after a confirmed failed direct SERP API attempt. Same-key recovery settles saved failure without another acquisition, while unresolved provider usage costs remain pending.
|
|
28
|
+
|
|
7
29
|
## [0.96.1] - 2026-10-13
|
|
8
30
|
|
|
9
31
|
### Fixed
|
|
@@ -2218,7 +2240,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
2218
2240
|
- Write actions remain unavailable until the account owner explicitly enables them.
|
|
2219
2241
|
- Provider-specific connection data is normalized into one agent-facing contract.
|
|
2220
2242
|
|
|
2221
|
-
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.96.
|
|
2243
|
+
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.96.3...HEAD
|
|
2222
2244
|
[0.95.2]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.95.1...v0.95.2
|
|
2223
2245
|
[0.95.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.95.0...v0.95.1
|
|
2224
2246
|
[0.95.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.94.4...v0.95.0
|
|
@@ -2389,3 +2411,6 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
2389
2411
|
[0.89.5]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.89.4...v0.89.5
|
|
2390
2412
|
[0.96.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.95.2...v0.96.0
|
|
2391
2413
|
[0.96.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.96.0...v0.96.1
|
|
2414
|
+
[0.96.2]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.96.1...v0.96.2
|
|
2415
|
+
|
|
2416
|
+
[0.96.3]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.96.2...v0.96.3
|
package/README.md
CHANGED
|
@@ -177,7 +177,7 @@ Build the branded one-click bundle:
|
|
|
177
177
|
npm run build:mcpb
|
|
178
178
|
```
|
|
179
179
|
|
|
180
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.96.
|
|
180
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.96.3`, SHA-256 `78b3c9f6aaf5ad404315cbd66ec103928b5905a123eb13f633c520e4ecc1e708`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
181
181
|
|
|
182
182
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
183
183
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{e as a,f as b,g as c,h as d,i as e,j as f,k as g,l as h}from"./chunk-CFZXZKFT.js";import"./chunk-WBJ6Y7WP.js";import"./chunk-NALXKTUU.js";import"./chunk-DJT2547J.js";import"./chunk-RDHGQ2WZ.js";import"./chunk-OY24ERPV.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-VVSHZV5S.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-WP3QS4CJ.js";import"./chunk-OILOHQNL.js";export{d as GMAIL_ATTACHMENT_REF_TTL_MS,c as GMAIL_INLINE_BODY_BYTES,b as GMAIL_MAIN_CONTRACT_HASH,a as GMAIL_MAIN_CONTRACT_VERSION,h as GmailService,e as GmailServiceError,g as normalizeGmailMessage,f as parseGmailAddresses};
|
package/dist/bin/api-server.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import{readFileSync as s}from"fs";function c(){try{for(let r of s(".env","utf8").split(`
|
|
3
|
-
`)){let o=r.indexOf("=");if(o<1||r.trimStart().startsWith("#"))continue;let e=r.slice(0,o).trim();process.env[e]||(process.env[e]=r.slice(o+1).trim())}}catch{}}c();async function a(){let[{serve:r},{app:o},{startWorker:e},{migrate:i}]=await Promise.all([import("@hono/node-server"),import("../server-
|
|
3
|
+
`)){let o=r.indexOf("=");if(o<1||r.trimStart().startsWith("#"))continue;let e=r.slice(0,o).trim();process.env[e]||(process.env[e]=r.slice(o+1).trim())}}catch{}}c();async function a(){let[{serve:r},{app:o},{startWorker:e},{migrate:i}]=await Promise.all([import("@hono/node-server"),import("../server-74VGVXA4.js"),import("../worker-L5NISZQV.js"),import("../core-schema-423BSTE6.js")]),n=parseInt(process.env.PORT??"3001");try{if(await i(),process.env.ANALYTICS_DATABASE_URL){let{migrateAnalytics:t}=await import("../migrations-3AKEC5QE.js");await t()}e(),r({fetch:o.fetch,port:n},t=>{console.log(`[server] http://localhost:${t.port}`),console.log(`[server] admin auth: ${process.env.ADMIN_KEY?"configured":"not configured"}`)})}catch(t){console.error("[startup] server preflight failed",t instanceof Error?t.name:"unknown_error"),process.exit(1)}}a();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{k as N,l as j,m as L,n as T,r as U}from"../chunk-4HZXATHB.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-6N5EC7ZO.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-24YDQ3EO.js";import"../chunk-OLYPLX5C.js";import"../chunk-B3JRL45T.js";import{c as _}from"../chunk-OILOHQNL.js";import{a as P}from"../chunk-
|
|
2
|
+
import{k as N,l as j,m as L,n as T,r as U}from"../chunk-4HZXATHB.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-6N5EC7ZO.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-24YDQ3EO.js";import"../chunk-OLYPLX5C.js";import"../chunk-B3JRL45T.js";import{c as _}from"../chunk-OILOHQNL.js";import{a as P}from"../chunk-JGGQ3UX5.js";import{Command as he}from"commander";import{spawn as te}from"child_process";import{mkdir as ke,writeFile as Pe}from"fs/promises";import{basename as Ce,join as X}from"path";function ie(e){return e.apiKey?.trim()||"sk_live_your_key"}function ae(e){return e.packageSpec?.trim()||"mcp-scraper@latest"}function A(e={}){return["-y","--package",ae(e),"mcp-scraper"]}function se(e){let n={MCP_SCRAPER_API_KEY:ie(e)},c=e.browserProfileName?.trim();return c&&(n.BROWSER_AGENT_PROFILE_NAME=c),e.browserProfileSaveChanges===!0&&(n.BROWSER_AGENT_PROFILE_SAVE_CHANGES="true"),n}function q(){return["mcp","remove","mcp-scraper","-s","user"]}function B(){return["mcp","get","mcp-scraper"]}function J(e){let n=e.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];if(!n)return null;let c=e.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1]??"",i=c.length?c.split(/\s+/):[],p={},u=e.split(/^\s*Environment:\s*$/m)[1];if(u)for(let a of u.split(`
|
|
3
3
|
`)){let l=a.match(/^\s{2,}([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);if(!l){if(a.trim().length&&!/^\s{2,}/.test(a))break;continue}p[l[1]]=l[2]}return{command:n,args:i,env:p}}function O(e){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(e.env))n.push("--env",`${c}=${i}`);return n.push("--",e.command,...e.args),n}function G(e={}){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(se(e)))n.push("--env",`${c}=${i}`);return n.push("--","npx",...A(e)),n}function I(e){if(e==="claude-code")return"claude";if(e==="claude"||D.hosts.some(n=>n.id===e))return e;throw new Error('Unknown host "'+e+'". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only')}function ce(e){return K(e==="claude"?"claude-code":e)}function W(e,n={}){let c=I(e),i=ce(c),p="Restart the MCP client so it starts a fresh npx process.",u='MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply',a=`X-Ray install protocol: ${v} (${b})`;return c==="codex"?["# Codex MCP config",a,i.exactConfig,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
4
4
|
`):c==="claude"?["# Claude Code command",a,i.exactConfig,"","# One-command Claude Code setup",u,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
5
5
|
`):c==="claude-desktop"?["# Claude Desktop config",a,i.exactConfig,"","Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,p].join(`
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as e}from"../chunk-
|
|
2
|
+
import{a as e}from"../chunk-42SLBGDI.js";import"../chunk-QYPSFZK3.js";import"../chunk-QER2ELZP.js";import"../chunk-4KHDV5E7.js";import"../chunk-7POHNTVW.js";import"../chunk-MVMWPKRC.js";import"../chunk-63VWBMYR.js";import"../chunk-3F7AFM53.js";import"../chunk-RDHGQ2WZ.js";import"../chunk-2IATUJO6.js";import"../chunk-OY24ERPV.js";import"../chunk-CNSVZTTT.js";import"../chunk-K3VDGJNK.js";import"../chunk-7SR7MEU5.js";import"../chunk-ANQ4JJVI.js";import"../chunk-SEZETSB3.js";import"../chunk-SDQW7IJN.js";import"../chunk-B4ZKPLSP.js";import"../chunk-VVSHZV5S.js";import"../chunk-C7MGJUTU.js";import"../chunk-PKQHHCO4.js";import"../chunk-4ZVO6GX4.js";import"../chunk-WNUXMPVX.js";import"../chunk-7ZR5IO3B.js";import"../chunk-WP3QS4CJ.js";import"../chunk-24YDQ3EO.js";import"../chunk-OLYPLX5C.js";import"../chunk-B3JRL45T.js";import"../chunk-OILOHQNL.js";import"../chunk-JGGQ3UX5.js";import"../chunk-HNNYXMSI.js";var _=["harvest_paa","search_serp","extract_url","diff_page","map_site_urls","map_wayback_snapshots","extract_site","analyze_site_similarity","audit_site","check_site_export","site_export_read","site_export_image","archive_read","youtube_harvest","youtube_transcribe","facebook_page_intel","facebook_ad_search","reddit_thread","reddit_trending","video_frame_analysis","video_frame_analysis_status","facebook_ad_transcribe","google_ads_search","google_ads_page_intel","google_ads_transcribe","facebook_video_transcribe","instagram_profile_content","instagram_media_download","maps_place_intel","maps_search","trustpilot_reviews","g2_reviews","capture_serp_snapshot","capture_serp_page_snapshots"];e({toolsets:new Set(["paa","serp"]),allowedToolNames:_});
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as s}from"../chunk-
|
|
2
|
+
import{a as s}from"../chunk-7ZR5IO3B.js";import"../chunk-WP3QS4CJ.js";import{a as e}from"../chunk-JGGQ3UX5.js";var r=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY,n=process.argv.includes("--help")||process.argv.includes("-h");n&&(process.stdout.write(["Usage: mcp-scraper-install [--no-color]","","Prints the branded MCP Scraper terminal install card and copyable install commands.","mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.",""].join(`
|
|
3
3
|
`)),process.exit(0));process.stdout.write(s({version:e,color:!r,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()}));
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as r}from"../chunk-
|
|
2
|
+
import{a as r}from"../chunk-42SLBGDI.js";import"../chunk-QYPSFZK3.js";import"../chunk-QER2ELZP.js";import"../chunk-4KHDV5E7.js";import"../chunk-7POHNTVW.js";import"../chunk-MVMWPKRC.js";import"../chunk-63VWBMYR.js";import"../chunk-3F7AFM53.js";import"../chunk-RDHGQ2WZ.js";import"../chunk-2IATUJO6.js";import"../chunk-OY24ERPV.js";import"../chunk-CNSVZTTT.js";import"../chunk-K3VDGJNK.js";import"../chunk-7SR7MEU5.js";import"../chunk-ANQ4JJVI.js";import"../chunk-SEZETSB3.js";import"../chunk-SDQW7IJN.js";import"../chunk-B4ZKPLSP.js";import"../chunk-VVSHZV5S.js";import"../chunk-C7MGJUTU.js";import"../chunk-PKQHHCO4.js";import"../chunk-4ZVO6GX4.js";import"../chunk-WNUXMPVX.js";import"../chunk-7ZR5IO3B.js";import"../chunk-WP3QS4CJ.js";import"../chunk-24YDQ3EO.js";import"../chunk-OLYPLX5C.js";import"../chunk-B3JRL45T.js";import"../chunk-OILOHQNL.js";import"../chunk-JGGQ3UX5.js";import"../chunk-HNNYXMSI.js";r();
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{
|
|
2
|
+
import{K as t}from"../chunk-GLCR4KJI.js";import"../chunk-M5QHXNFZ.js";import{d as r}from"../chunk-B4ZKPLSP.js";import"../chunk-VVSHZV5S.js";import"../chunk-C7MGJUTU.js";import"../chunk-PKQHHCO4.js";import"../chunk-4ZVO6GX4.js";import"../chunk-WNUXMPVX.js";import"../chunk-6N5EC7ZO.js";import"../chunk-HNNYXMSI.js";import{Command as s,Option as a}from"commander";var i=new s;i.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>","Seed query").option("-l, --location <location>",'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>","Google country code","us").option("--hl <hl>","Google language code","en").option("-d, --depth <depth>","BFS depth (1-30)","3").option("-m, --max-questions <n>","Max questions to harvest","100").option("-o, --output <dir>","Output directory","./paa-output").option("-f, --format <format>","Output format: json, csv, or both","both").option("--headless","Run browser in headless mode",!1).option("--profile <dir>","Persistent browser profile directory").option("--proxy <url>","Proxy server URL").option("--browser-api-key <key>","Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").addOption(new a("--\u006b\u0065\u0072\u006e\u0065\u006c-api-key <key>").hideHelp()).action(async e=>{try{let o=await t({query:e.query,location:e.location,gl:e.gl,hl:e.hl,depth:parseInt(e.depth,10),maxQuestions:parseInt(e.maxQuestions,10),outputDir:e.output,format:e.format,headless:e.headless,profileDir:e.profile,proxy:e.proxy,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.browserApiKey??e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey??r()});console.log(JSON.stringify({totalQuestions:o.totalQuestions,outputDir:o.stats.seed}))}catch(o){console.error(o instanceof Error?o.message:String(o)),process.exit(1)}});async function n(){await i.parseAsync()}n();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{j as b,k as c}from"./chunk-PANT6VEM.js";import"./chunk-U4WIJZCX.js";import{F as a}from"./chunk-P55O4IDC.js";import"./chunk-KXJCGV3H.js";import"./chunk-JMZIDHYQ.js";import"./chunk-M5QHXNFZ.js";import"./chunk-CNSVZTTT.js";import"./chunk-K3VDGJNK.js";import"./chunk-7SR7MEU5.js";import"./chunk-ANQ4JJVI.js";import"./chunk-SEZETSB3.js";import"./chunk-SDQW7IJN.js";import"./chunk-C7MGJUTU.js";import"./chunk-PKQHHCO4.js";import"./chunk-4ZVO6GX4.js";import"./chunk-WNUXMPVX.js";import"./chunk-OLYPLX5C.js";import"./chunk-B3JRL45T.js";import"./chunk-OILOHQNL.js";import"./chunk-HNNYXMSI.js";export{b as MAX_ANALYZED_LINK_EDGES,a as SITE_EXTRACT_PAGE_CHUNK,c as assembleExtractArtifacts};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{c as j}from"./chunk-U7BZ5KTE.js";import{k as X}from"./chunk-PANT6VEM.js";import{A as F,c as A,d as $,g as b,h as C,o as K,p as U,q as G,r as M,s as D,t as W,z as q}from"./chunk-P55O4IDC.js";import{a as B,b as L,c as H}from"./chunk-QER2ELZP.js";import{b as k,c as S,d as y,e as x,f as O,g as I,i as h,j as m,l as E,p as N}from"./chunk-ANQ4JJVI.js";import{h as v,i as z}from"./chunk-C7MGJUTU.js";import{a as d,b as J,c as T,d as R}from"./chunk-HNNYXMSI.js";import{randomUUID as oe}from"crypto";import{createHash as Q,randomUUID as V}from"crypto";function _(e,i=new Set){if(e===null||typeof e=="string"||typeof e=="boolean"||typeof e=="number"&&Number.isFinite(e))return e;if(typeof e!="object"||!e)throw new d("Operation inputs must be finite JSON data");if(i.has(e))throw new d("Operation inputs cannot be cyclic");if(!Array.isArray(e)&&![Object.prototype,null].includes(Object.getPrototypeOf(e)))throw new d("Operation inputs must be plain JSON data");i.add(e);try{return Array.isArray(e)?e.map(t=>_(t,i)):Object.fromEntries(Object.keys(e).sort().map(t=>[t,_(e[t],i)]))}finally{i.delete(e)}}async function P(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0)throw new d("Authenticated operation principal is required");if(e.jobId!=null&&(!e.jobId.trim()||e.jobId.length>512))throw new d("Operation job ID is invalid");let i=e.tool.trim();if(!i||i.length>100)throw new d("Operation tool is required and bounded");if(e.idempotencyKey!=null&&(!e.idempotencyKey.trim()||e.idempotencyKey.length>512))throw new d("Operation idempotency key is invalid");let t=e.leaseMs??3e5;if(!Number.isSafeInteger(t)||t<1e3||t>36e5)throw new d("Execution lease must be between one second and one hour");let n=JSON.stringify(_(e.normalizedInput));if(Buffer.byteLength(n)>64*1024)throw new d("Operation inputs exceed 64 KiB");let r=e.idempotencyKey?.trim()??V();return E({userId:e.userId,ownerScope:`user:${e.userId}`,tool:i,jobId:e.jobId??null,normalizedFlags:e.normalizedInput,idempotencyKey:r,requestFingerprint:Q("sha256").update(n).digest("hex")},V(),t,e.storeGate)}async function Y(e){let i=JSON.stringify(_(e.normalizedInput));if(!Number.isSafeInteger(e.userId)||e.userId<=0||Buffer.byteLength(i)>64*1024)throw new d("Invalid operation receipt principal/input");let{readOperationIntentResult:t}=await import("./repository-UKDA4IWP.js"),{operationIdentityHash:n}=await import("./contracts-GUXGFCVV.js");return t(e.userId,n(`user:${e.userId}\0${e.tool}\0${e.idempotencyKey}`),Q("sha256").update(i).digest("hex"))}async function we(e){let i=Date.now()+27e4,t=e.idempotencyKey?.trim()??oe();if(!t||t.length>512)return{status:400,body:{error_code:"invalid_request",message:"Idempotency-Key must contain 1 to 512 characters"}};let n=(a,c,p)=>({...a,body:{...a.body,operation:{runId:c,idempotencyKey:t,state:p}}}),r=(a,c,p)=>n({status:c==="pending"?202:503,headers:{"Retry-After":"2"},body:{...v({errorCode:"response_lost",retryable:!0,chargeStatus:p,details:{retry_guidance:"Retry the identical request with the returned idempotencyKey. Pending work is not started again; a saved response replays. Unknown work permits compensation only."}}),message:c==="pending"?"This operation is still running. Retry with the same idempotencyKey.":"The execution outcome is unknown. Retry with the same idempotencyKey for result or compensation recovery."}},a,c),s;try{s=await P({...e,idempotencyKey:t,leaseMs:3e5})}catch(a){return a instanceof d?n({status:400,body:{error_code:"invalid_request",message:a.message,retryable:!1}},null,"not_started"):a instanceof J?n({status:409,body:{error_code:"idempotency_conflict",message:a.message,retryable:!1}},null,"conflict"):r(null,"unknown")}let o=s.kind==="claimed"?s.claim.runId:s.runId;if(s.kind==="replay"){let a=JSON.parse(s.result.json);return!a||typeof a.status!="number"||!a.body||typeof a.body!="object"?r(o,"unknown"):n(a,o,"replayed")}if(s.kind==="pending")return r(o,"pending");if(s.kind==="unknown")try{let a=await y(o,e.userId);if(!a)return r(o,"unknown");let c=await e.recover(a);if(c.recover)return await h(a),n(c.reply,o,"unknown");let p=n(c.reply,o,"recovered_unknown"),l={json:JSON.stringify(p),status:"unknown"};return await O(a,l)?p:r(o,"unknown")}catch{return r(o,"unknown","refund_pending")}if(s.kind!=="claimed")return r(o,"unknown");let u=s.claim,f;try{if(f=await e.prepare(),f.rejection)return await I(u),n(f.rejection,o,"not_started");if(!await k(u))return r(o,"unknown");let a={claim:u,deadlineAt:i,assertActive:async()=>{if(B(i),!await m(u))throw Error("Execution ownership is no longer current")}},c;try{c=await e.execute(a,f.value)}catch{return await h(u),r(o,"unknown","refund_pending")}if(c.recover)return await h(u),n(c.reply,o,"unknown");let p=n(c.reply,o,c.reply.status<400?"succeeded":"failed");try{return await x(u,{json:JSON.stringify(p),status:p.status<400?"succeeded":"failed"})?p:r(o,"unknown")}catch{return await h(u).catch(()=>{}),r(o,"unknown")}}catch{return await I(u).catch(()=>{}),r(o,"unknown")}finally{await f?.release().catch(()=>{})}}var g=3e5;function ee(e){if(!e?.trim()||e.length>512)throw new d("A bounded framework workflow ID is required");return T(`workflow:${e}`)}async function ye(e){if(!Number.isSafeInteger(e.userId)||e.userId<=0)throw new d("Workflow principal is required");let i=ee(e.workflowId),t=await E(e,i,g);if(t.kind==="claimed"){let n;try{n=await k(t.claim)}catch(r){if(!await m(t.claim))throw r;n=!0}if(!n)throw Error("workflow_execution_start_unknown");return t}if(t.kind==="pending"){let n={runId:t.runId,userId:e.userId,token:i};if(await m(n)&&await S(n,g))return{kind:"claimed",claim:n}}return t}var te=e=>`${e.workflow.runId}:${e.phase}`;async function he(e){if(!await S(e.workflow,g))throw Error("workflow_execution_not_current");let i=te(e),t=await P({userId:e.workflow.userId,tool:e.tool,jobId:e.jobId,normalizedInput:e.normalizedInput,idempotencyKey:i,leaseMs:g});if(t.kind==="replay"){if(t.result.status!=="succeeded")throw Error(`operation_phase_unknown:${t.runId}`);return JSON.parse(t.result.json)}if(t.kind!=="claimed")throw Error(`operation_phase_${t.kind}:${t.runId}`);let n=t.claim;try{await e.prepare()}catch(r){throw await I(n).catch(()=>{}),r}try{let r;try{r=await k(n)}catch(a){if(!await m(n))throw a;r=!0}if(!r)throw Error("operation_phase_start_unknown");let s=Date.now()+g,o=async()=>{if(Date.now()>=s)throw new L(e.phase);if(!await m(e.workflow)||!await m(n))throw Error("operation_phase_claim_not_current")};await o();let u=await H(s,()=>e.execute({claim:n,deadlineAt:s,assertActive:o}),e.phase);await o();let f=JSON.stringify(u);if(f==null||!await x(n,{status:"succeeded",json:f}))throw Error("operation_phase_publication_unknown");return u}catch(r){await h(n).catch(()=>{});let s=await y(n.runId,n.userId).catch(()=>null);throw s&&await O(s,{status:"unknown",json:JSON.stringify({phase:e.phase,outcome:"unknown"})}).catch(()=>{}),r}}async function be(e,i){let t=await Y({userId:e.workflow.userId,tool:e.tool,idempotencyKey:te(e),normalizedInput:e.normalizedInput});if(!t||t.result.status!=="succeeded"||R({value:JSON.parse(t.result.json)})!==R({value:i}))throw Error("operation_phase_cache_receipt_unknown")}async function ge(e,i,t){let n={runId:e,userId:i,token:ee(t)};if(await m(n))return{claim:n,recovery:!1};let r=await y(e,i);return r?{claim:r,recovery:!0}:null}async function re(e,i,t){let n={json:JSON.stringify(i),status:e.recovery?"unknown":t};if(!(e.recovery?await O(e.claim,n,e.settlementGate):await x(e.claim,n)))throw Error("workflow_result_publication_unknown")}async function je(e){if(!await b(e))return{kind:"missing",body:{error:"job not found"}};let t=await M(e);if(!t)return{kind:"conflict",body:{error:"Only failed or expired recovery jobs can be re-finalized.",status:(await b(e))?.status??"missing"}};let n=t.executionGeneration;if(!await D(e,n))return{kind:"conflict",body:{error:"job state changed during re-finalization claim"}};let r;try{r=await X(t,{})}catch(l){throw await F(e,l instanceof Error?l.message:String(l),void 0,n).catch(()=>!1),l}let s=A(t),o=$(t,s.creditTruncated),u=o==="failed"?`No pages were extracted successfully (${t.failedUrls} failed, ${t.remainingUrls} remaining).`:o==="partial"?`${t.successfulUrls} pages succeeded; ${t.failedUrls} failed and ${t.remainingUrls} remain.${s.creditTruncated?` Credit availability limited this request from ${s.requestedMaxPages} to ${s.effectiveMaxPages} pages.`:""}`:null,f;try{f=await q(e,r,o,u,!1,null,n)}catch(l){let w=await b(e).catch(()=>null);if(!w||w.id!==e||w.executionGeneration!==n||w.status!==o||JSON.stringify(w.artifacts)!==JSON.stringify(r)||w.error!==(u?z(u).slice(0,2e3):null)||w.publicError!==null)throw l;f=!0}if(!f)return{kind:"conflict",body:{error:"job state changed during re-finalization"}};let a=await b(e);if(!a||a.executionGeneration!==n||a.status!==o)return{kind:"conflict",body:{error:"job state changed before re-finalization settlement"}};let c="already_settled_or_refunded";if(a.billedMc==null&&a.userId!=null){let l=await j(a);if(!l)throw Error("extract_settlement_acknowledgment_unknown");c=`settled: billed ${l.netChargeMc} mc, refunded ${l.refundMc} mc${l.expiredRefundMc?`, expired refund ${l.expiredRefundMc} mc`:""}`}let p=r.find(l=>l.contentType==="application/zip"||l.filename?.endsWith(".zip"));return{kind:"complete",body:{ok:!0,jobId:e,artifacts:r.length,settlement:c,bundleUrl:p?.url??null,bundleBytes:p?.bytes??null}}}async function Pe(e=25){let i=await U(e);for(let r of i)await G(r.id).catch(()=>!1);let t=await C(e),n={checked:new Set([...i,...t].map(r=>r.id)).size,settled:0,failed:0};for(let r of t){if(r.userId==null){await W(r.id,"Settlement abandoned because the legacy job has no user owner.",r.executionGeneration??0),n.failed+=1;continue}try{if(!await j(r))throw Error("extract_settlement_acknowledgment_unknown");if(r.executionGeneration===0&&await N(`extract-site:${r.id}`,r.userId)){let o=await y(`extract-site:${r.id}`,r.userId);o&&await re({claim:o,recovery:!0},{jobId:r.id,status:r.status,artifacts:r.artifacts??[]},"unknown")}n.settled+=1}catch(s){n.failed+=1;let o=s instanceof Error?s.message:String(s);await K(r.id,o,r.executionGeneration??0).catch(()=>{}),console.warn("[site-extract/reconcile]",r.id,o)}}return n}export{P as a,we as b,g as c,ee as d,ye as e,he as f,be as g,ge as h,re as i,je as j,Pe as k};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{$ as E,B as R,D as S,La as C,Ma as P,Y as v,_ as h,ha as T,ja as w,ka as _,la as y,x as u,y as f,z as g}from"./chunk-
|
|
1
|
+
import{$ as E,B as R,D as S,La as C,Ma as P,Y as v,_ as h,ha as T,ja as w,ka as _,la as y,x as u,y as f,z as g}from"./chunk-QYPSFZK3.js";import{a as x}from"./chunk-7ZR5IO3B.js";import{a as c}from"./chunk-JGGQ3UX5.js";import{readFileSync as L}from"fs";import{homedir as B}from"os";import{join as U}from"path";import{serveStdio as Y}from"@modelcontextprotocol/server/stdio";import{McpServer as J}from"@modelcontextprotocol/server";var I=new Map(_.map(r=>[r.upstreamName,r.id]));function N(r){let s=r.split(`
|
|
2
2
|
`).filter(t=>t.startsWith("data:")).map(t=>t.slice(5).trim()).filter(Boolean);if(!s.length)return JSON.parse(r);for(let t of s){let e=JSON.parse(t);if(e.result||e.error)return e}throw new Error("hosted MCP returned no JSON-RPC result")}function l(r){return{content:[{type:"text",text:JSON.stringify({ok:!1,error:r})}],isError:!0}}var p=class{baseUrl;apiKey;constructor(s,t){this.baseUrl=s.replace(/\/$/,""),this.apiKey=t}async callMemoryTool(s,t){let e=I.get(s);if(!e)return l(`unknown memory tool: ${s}`);try{let n=await fetch(`${this.baseUrl}/mcp`,{method:"POST",headers:{"content-type":"application/json",accept:"application/json, text/event-stream","x-api-key":this.apiKey},body:JSON.stringify({jsonrpc:"2.0",id:`memory:${e}`,method:"tools/call",params:{name:e,arguments:t}})}),i=await n.text();if(!n.ok)return l(`hosted memory ${e} failed (HTTP ${n.status})`);let o=N(i);return o.error?l(o.error.message??`hosted memory ${e} failed`):o.result??l(`hosted memory ${e} returned no result`)}catch(n){return l(n instanceof Error?n.message:`hosted memory ${e} call failed`)}}};function M(r,s){if(s.length===0)throw new Error("Restricted MCP tool allowlist must contain at least one tool name");let t=new Set;for(let o of s){if(typeof o!="string"||o.trim()!==o||o.length===0)throw new Error("Restricted MCP tool allowlist contains a malformed tool name");if(t.has(o))throw new Error(`Restricted MCP tool allowlist contains duplicate tool name: ${o}`);t.add(o)}let e=new Set,n=r,i=n.registerTool.bind(r);return n.registerTool=(o,a,m)=>{if(t.has(o))return e.add(o),i(o,a,m)},{assertComplete(){let o=[...t].filter(a=>!e.has(a));if(o.length>0)throw new Error(`Restricted MCP tool allowlist contains unknown or unavailable tool names: ${o.join(", ")}`)},registeredToolNames(){return[...e]}}}import{Client as b,StreamableHTTPClientTransport as k}from"@modelcontextprotocol/client";var $=new Set(["artifact_read","artifact_describe","artifact_download","report_artifact_read"]),A=()=>({content:[{type:"text",text:"Retained file storage is unavailable. Preserve the artifactId and retry retrieval; do not repeat acquisition."}],isError:!0});function O(r,s,t={}){return async(e,n)=>{if(!$.has(e))return A();let i=new b({name:"mcp-scraper-artifact-client",version:c},{versionNegotiation:{mode:"auto"}}),o=AbortSignal.timeout(3e4);try{let a=new k(new URL(`${r.replace(/\/$/,"")}/mcp`),{requestInit:{signal:o,headers:{Authorization:`Bearer ${s}`,"x-api-key":s,...t.localNetworkAccess?{"x-mcp-scraper-local-network":"1"}:{}}}});return await i.connect(a,{signal:o,timeout:3e4}),await i.callTool({name:e,arguments:n},{signal:o,timeout:3e4})}catch{return A()}finally{await i.close().catch(()=>{})}}}var G=new Set(["paa","serp","browser-agent","scheduled-results","memory"]);function K(){let s=[process.env.MCP_SCRAPER_KEY_PATH?.trim(),U(B(),".mcp-scraper-key")].filter(Boolean);for(let t of s)try{let e=L(t,"utf8").trim();if(e)return e}catch{}}function j(r,s={}){let t=s.toolsets??G,e=process.env.MCP_SCRAPER_BASE_URL?.trim()||process.env.MCP_BASE_URL?.trim()||"https://mcpscraper.dev",n=R(),i=S({deploymentProfile:n,transportProfile:"stdio",baseUrl:e,explicitlyEnabled:process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK==="1"}),o=process.env.BROWSER_AGENT_CONSOLE_URL?.trim()||e,a=new J({name:"mcp-scraper",version:c},{instructions:f,cacheHints:{"server/discover":{ttlMs:3e5,cacheScope:"private"},"tools/list":{ttlMs:3e5,cacheScope:"private"},"prompts/list":{ttlMs:3e5,cacheScope:"private"},"resources/list":{ttlMs:3e5,cacheScope:"private"},"resources/templates/list":{ttlMs:3e5,cacheScope:"private"},"resources/read":{ttlMs:6e4,cacheScope:"private"}}});g(a);let m=s.allowedToolNames?M(a,s.allowedToolNames):void 0,d=s.httpExecutor??new T(e,r,{localNetworkAccess:i,analyticsIdentity:()=>E(a.server.getClientVersion()?.name)});return t.has("paa")&&h(a,d,{ownerId:v(r),deploymentProfile:n,transportProfile:"stdio",baseUrl:e,localNetworkAccess:i,taskHandleSecret:r,remoteArtifactAccess:O(e,r,{localNetworkAccess:i})}),t.has("serp")&&u(a,d,{exposeDevelopmentDiagnostics:n==="development"||n==="test"}),t.has("browser-agent")&&w(a,{baseUrl:e,apiKey:r,consoleBaseUrl:o}),t.has("scheduled-results")&&P(a,new C(e,r)),t.has("memory")&&y(a,new p(e,r)),m?.assertComplete(),a}function Re(r={}){let s=process.argv.includes("--stdio")||process.env.MCP_SCRAPER_FORCE_STDIO==="1",t=!!(process.stdin.isTTY&&process.stdout.isTTY),e=process.argv.includes("--help")||process.argv.includes("-h");if(!s&&(t||e)){let i=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY;process.stdout.write(x({version:c,color:!i,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()})),process.exit(0)}let n=(process.env.MCP_SCRAPER_API_KEY??process.env.MCP_SCRAPER_KEY??process.env.MCP_API_KEY??K())?.trim();n||(process.stderr.write(`MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required
|
|
3
3
|
`),process.exit(1)),Y(()=>j(n,r),{legacy:"serve",onerror(i){process.stderr.write(`${i.message}
|
|
4
4
|
`)}})}export{Re as a};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{c as b}from"./chunk-
|
|
1
|
+
import{c as b}from"./chunk-OY24ERPV.js";import{g as y,i as f}from"./chunk-C7MGJUTU.js";var h={\u006b\u0065\u0072\u006e\u0065\u006c:"browserRuntime"},E=/error|message/i,x=(new RegExp("(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*(?:dashboard|on\u006b\u0065\u0072\u006e\u0065\u006c)", "i")),T=/\b(?:ENOENT|ENOTDIR|EACCES|EPERM|EISDIR|EMFILE|ENFILE)\b|\b(?:mkdir|open|stat|lstat|readFile|writeFile)\s+['"]?(?:\/|[A-Za-z]:\\)|(?:^|[\s'"])(?:\/Users|\/home|\/tmp|\/var|\/private|\/opt|\/srv|\/root)\//i,N=new Set(["\u006b\u0065\u0072\u006e\u0065\u006c_session_id","browser_session_id","\u006b\u0065\u0072\u006e\u0065\u006cSessionId","browserSessionId","sessionId","\u006b\u0065\u0072\u006e\u0065\u006c_delete_started","session_cleanup_started","\u006b\u0065\u0072\u006e\u0065\u006cDeleteStarted","sessionCleanupStarted","\u006b\u0065\u0072\u006e\u0065\u006c_delete_succeeded","session_cleanup_succeeded","\u006b\u0065\u0072\u006e\u0065\u006cDeleteSucceeded","sessionCleanupSucceeded","\u006b\u0065\u0072\u006e\u0065\u006c_delete_error","session_cleanup_error","\u006b\u0065\u0072\u006e\u0065\u006cDeleteError","sessionCleanupError","\u006b\u0065\u0072\u006e\u0065\u006cProxyId","proxyId","requestedProxyIdSuffix","retrievedProxyIdSuffix","provider","providerName","service","serviceName","vendor","vendorName","organization","organizationId","accountId","browser_provider","provider_session_id","provider_session_observed_at","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message","browserProvider","providerSessionId","providerLookupStatus","providerStatus","providerErrorCode","providerErrorMessage","providerReasonSource","providerDurationMs","providerBandwidthBytes","providerCaptcha","providerCheckedAt","providerDetail","providerDisconnectObserved","providerDisconnectMessage"]),O=new Set(["hosted_url","hostedUrl","live_view_url","liveViewUrl","cdp_ws_url","cdpWsUrl","browser_live_view_url"]),L=(new RegExp("\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bon\u006b\u0065\u0072\u006e\u0065\u006c\\.com[^\\s\"'<>]*", "gi"));function k(e){return e.replace(L,"[browser-service]")}var D=new Set(["\u0062\u0072\u0069\u0067\u0068\u0074_\u0064\u0061\u0074\u0061_browser_api","\u006b\u0065\u0072\u006e\u0065\u006c","local"]),j=new Set(["pending","complete","not_found","unavailable_credentials","temporary_error","invalid_payload","not_applicable"]),q=new Set(["active","running","complete","completed","failed","closed","stopped","terminated","timed_out"]),F=new Set(["provider_session_api","client_runtime","extractor","unknown"]),U=new Set(["browser_disconnected","worker_disconnect","job_killed","cdp_cmd_timeout","network_inactivity_timeout","session_timeout","client_timeout","navigate_domains_limit","cdp_error","browser_session_interrupted","provider_detail_pending","provider_lookup_unavailable","provider_reported_no_error","provider_session_not_found","provider_invalid_json","provider_invalid_payload","provider_lookup_timeout","provider_lookup_network_error","unknown"]),M=new Set(["browser_provider","provider_session_id","provider_lookup_status","provider_status","provider_error_code","provider_error_message","provider_reason_source","provider_duration_ms","provider_bandwidth_bytes","provider_captcha","provider_checked_at","provider_detail","provider_detail_json","provider_disconnect_observed","provider_disconnect_message"]),$=512,z=320;function p(e,r){return typeof e=="string"&&r.has(e)?e:null}function H(e){return typeof e!="string"?null:U.has(e)||/^provider_http_(?:429|5\d\d)$/.test(e)?e:null}function Q(e){return typeof e!="string"||e.length<1||e.length>$||/^(?:authorization|bearer|password|secret|token|api[_-]?key|account)/i.test(e)?null:/^[A-Za-z0-9._:-]+$/.test(e)&&!e.includes("://")?e:null}function R(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=0?e:null}function A(e){if(typeof e!="string"||e.length>40)return null;let r=Date.parse(e);return Number.isFinite(r)&&new Date(r).toISOString()===e?e:null}function C(e){if(typeof e!="string")return null;let r=e.replace(/[\u0000-\u001f\u007f]+/g," ").replace(/\s+/g," ").trim();if(!r||r.startsWith("{")||r.startsWith("["))return null;let t=r.replace(/\b(?:wss?|https?):\/\/\S+/gi,"[redacted endpoint]").replace(/\b(?:authorization|proxy-authorization)\s*:\s*(?:Bearer\s+)?\S+/gi,"[redacted credential]").replace(/\bBearer\s+\S+/gi,"[redacted credential]").replace(/\b(?:api[_-]?key|password|secret|token|account[_-]?id)\s*[=:]\s*[^\s,;]+/gi,"[redacted credential]").replace(/\b(?:account|organization)\s+(?:id\s+)?[A-Za-z0-9_-]{4,}/gi,"[redacted account]");return k(t).slice(0,z)}function w(e){return typeof e=="boolean"?e:null}function V(e){return{browser_provider:p(e.browser_provider,D),provider_session_id:Q(e.provider_session_id),provider_session_observed_at:A(e.provider_session_observed_at),provider_lookup_status:p(e.provider_lookup_status,j),provider_status:p(e.provider_status,q),provider_error_code:H(e.provider_error_code),provider_error_message:C(e.provider_error_message),provider_reason_source:p(e.provider_reason_source,F),provider_duration_ms:R(e.provider_duration_ms),provider_bandwidth_bytes:R(e.provider_bandwidth_bytes),provider_captcha:w(e.provider_captcha),provider_checked_at:A(e.provider_checked_at),provider_disconnect_observed:w(e.provider_disconnect_observed),provider_disconnect_message:C(e.provider_disconnect_message)}}function l(e,r=""){if(typeof e=="string"){let t=k(e);return E.test(r)&&(x.test(e)||T.test(e))?t=y("service_unavailable"):E.test(r)&&(t=f(t)),t}if(Array.isArray(e))return e.map(t=>l(t,r));if(e!==null&&typeof e=="object"){let t={};for(let[n,o]of Object.entries(e)){if(N.has(n))continue;let i=h[n]??n;if(O.has(n)){t[i]=null;continue}t[i]=l(o,n)}return t}return e}function Y(e){return e.map(r=>{if(r===null||typeof r!="object"||Array.isArray(r))return l(r);let t=r,n=l(t);return Object.keys(t).some(i=>M.has(i))?{...n,...V(t)}:n})}function X(e){let r=e?.diagnostics;if(!r)return e;let t=r.serpCompleteness,n=t?.queryIntegrity;return!r.debug&&!n?e:{...e,diagnostics:{...r,...r.debug?{debug:l(r.debug)}:{},...n?{serpCompleteness:{...t,queryIntegrity:{...n,outboundProviderMethod:"search"}}}:{}}}}var ee="strftime('%Y-%m-%dT%H:%M:%fZ','now')",_="julianday('now')";var oe=20,m="(json_extract(options,'$.executionOwner')='direct' AND json_extract(options,'$.serpOnly') IS NOT 1)",se=`NOT COALESCE(${m},0) AND json_extract(options,'$.syncExecutionProtocol') IS NOT 1 AND json_extract(options,'$.queueExecutionProtocol') IS NULL`,ie=`${m} AND julianday(json_extract(options,'$.paaLeaseExpiresAt')) > ${_}`,ae=`${m} AND (
|
|
2
2
|
(json_extract(options,'$.paaLeaseExpiresAt') IS NOT NULL AND COALESCE(julianday(json_extract(options,'$.paaLeaseExpiresAt')),0) <= ${_})
|
|
3
3
|
OR (json_extract(options,'$.paaLeaseExpiresAt') IS NULL AND julianday(started_at) <= julianday('now','-15 minutes'))
|
|
4
4
|
)`;function ue(e){return{jobId:e.id,userId:e.user_id,leaseToken:typeof e.options.paaLeaseToken=="string"?e.options.paaLeaseToken:null,recoveryCount:Number(e.options.paaRecoveryCount??0),resultJson:e.result==null?null:JSON.stringify(e.result)}}function de(e,r){return!r||r.jobId!==e||!Number.isSafeInteger(r.userId)||r.userId<=0||!Number.isSafeInteger(r.recoveryCount)||r.recoveryCount<0||r.leaseToken!==null&&(typeof r.leaseToken!="string"||r.leaseToken.length===0)?{sql:"0",args:[]}:{sql:`user_id=? AND json_extract(options,'$.paaLeaseToken') IS ?
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{a
|
|
1
|
+
import{a}from"./chunk-WP3QS4CJ.js";var _={message:"Background extraction and video jobs now use the existing runtime and preserve saved progress across bounded continuations. Browser acquisition no longer checks database permission every second. PAA progress checks remain at least 20 seconds apart."};var c={reset:"\x1B[0m",cyan:"\x1B[36m",lime:"\x1B[32m",amber:"\x1B[33m",red:"\x1B[31m",muted:"\x1B[90m",bold:"\x1B[1m"};function r(t,e,n){return n?`${c[e]}${t}${c.reset}`:t}function o(t,e,n){let s=t.padEnd(9," ");return` ${r(s,"muted",n)} ${e.join(r(" . ","muted",n))}`}function b(t){let e=t.color??!0,n=t.apiKeyConfigured?"$MCP_SCRAPER_API_KEY":"sk_live_your_key",s=String.raw`
|
|
2
2
|
__ __ ____ ____
|
|
3
3
|
| \/ |/ ___| _ \
|
|
4
4
|
| |\/| | | | |_) |
|
|
@@ -12,5 +12,5 @@ import{a as s}from"./chunk-WP3QS4CJ.js";var _={message:"PAA progress checks now
|
|
|
12
12
|
|____/ \____|_| \_\/_/ \_\_| |_____|_| \_\
|
|
13
13
|
`,i=[`MCP_SCRAPER_API_KEY=${n} npx -y -p mcp-scraper@latest \\`," mcp-scraper-cli agent install claude --apply"].join(`
|
|
14
14
|
`),l=["[mcp_servers.mcp-scraper]",'command = "npx"','args = ["-y", "-p", "mcp-scraper@latest", "mcp-scraper"]',`env = { MCP_SCRAPER_API_KEY = "${n}" }`].join(`
|
|
15
|
-
`);return[r(`mcp-scraper v${t.version}`,"bold",e),r("> mcp-scraper-install","muted",e),r(
|
|
15
|
+
`);return[r(`mcp-scraper v${t.version}`,"bold",e),r("> mcp-scraper-install","muted",e),r(s,"amber",e),`${r("MCP Scraper Agent","cyan",e)} . v${t.version} . mcpscraper.dev`,"1/1 install surfaces ready",r(`Newest in v${t.version}: ${_.message}`,"lime",e),"",`${r("Tools","cyan",e)} ${r(`(${a.inventory.totalTools} MCP tools)`,"muted",e)}`,o("search",["harvest_paa","search_serp","maps_search","maps_place_intel"],e),o("extract",["extract_url","map_site_urls","extract_site","audit_site","directory_workflow"],e),o("build",["create_editorial_reading_room","rank_tracker_workflow","portable HTML"],e),o("media",["youtube_harvest","youtube_transcribe","facebook_reels_inventory","facebook_ad_search","facebook_page_intel","facebook_ad_transcribe","facebook_video_transcribe","tiktok_video_transcribe","instagram_profile_content","instagram_media_download","reddit_thread"],e),o("browser",["serp_identity_create","serp_identity_list","browser_open","browser_profile_connect","browser_profile_list","browser_close","browser_screenshot","browser_read","browser_locate","browser_replay_mark","browser_replay_annotate"],e),o("connect",["list_service_connections","describe_service_connection_tool","import_service_connection_to_memory","export_connected_service_data","renew_connected_data_download","read_service_connection","call_service_connection_action"],e),o("commons",["commons_search_entities","commons_get_entity_linkset","commons_prepare_entity","commons_submit_entity","commons_prepare_publication","commons_claim_publication","commons_publish_editorial","commons_get_publication"],e),o("account",["credits_info","reports","MCP resources"],e),o("memory",["memory-put","memory-get","memory-search","list-vaults","record-fact","list-scheduled-actions"],e),`${r("Workflows","cyan",e)} ${r("(MCP + CLI + API)","muted",e)}`,o("route",["workflow_list","workflow_suggest","workflow_run","workflow_step","workflow_status","workflow_artifact_read"],e),o("seo",["directory","agent-packet","competitive audit","map/serp comparison","PAA/AIO briefs","scheduled runs"],e),"",r("Usage tips:","amber",e),"Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.","Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.","Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install","Hosted browser sessions use direct/no-proxy egress by default.","Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.","Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.","Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.","Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.","Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.","For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.","If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.","",`${r("Ready.","lime",e)} Install the combined MCP server with one command:`,"",r("Setup doctor","amber",e),"npx -y -p mcp-scraper@latest mcp-scraper-cli doctor","",r("Hosted profile setup","amber",e),'In your MCP client, call browser_profile_connect with email="seo@example.com" and domain="chatgpt.com".',"Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.","",r("Claude Code one-command setup","amber",e),i,"Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list","",r("Codex config","amber",e),l,"",r("Claude Desktop Extension","amber",e),"Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb","",r("Safety note:","muted",e),"mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.","Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.",""].join(`
|
|
16
16
|
`)}export{b as a};
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import{c as
|
|
2
|
-
JOIN operation_runs r ON r.id = e.run_id WHERE e.run_id = ? AND r.user_id = ?`,args:[t,n]})).rows[0]}async function R(e,t,n,r,o=
|
|
3
|
-
WHERE run_id = ? AND phase = 'claimed' AND expires_at_ms <= ${
|
|
1
|
+
import{a as A}from"./chunk-PKQHHCO4.js";import{c as w}from"./chunk-4ZVO6GX4.js";import{a as u}from"./chunk-WNUXMPVX.js";import{b as E,c as f,d as S,e as m,f as l}from"./chunk-HNNYXMSI.js";import{randomUUID as D}from"crypto";var a="(CAST(strftime('%s', 'now') AS INTEGER) * 1000)",N=e=>{if(!Number.isSafeInteger(e)||e<1e3||e>36e5)throw Error("Execution lease must be between one second and one hour")};async function d(e,t,n){return(await e.execute({sql:`SELECT e.*, r.status AS run_status, r.tool, r.job_id, ${a} AS now_ms FROM operation_executions e
|
|
2
|
+
JOIN operation_runs r ON r.id = e.run_id WHERE e.run_id = ? AND r.user_id = ?`,args:[t,n]})).rows[0]}async function R(e,t,n,r,o=u(),s){N(r),s&&l({userId:t,tool:s.tool,jobId:s.jobId},s);let i=await d(o,e,t);return i?(i.phase==="claimed"&&Number(i.expires_at_ms)<=Number(i.now_ms)&&(await o.execute({sql:`UPDATE operation_executions SET claim_token = ?, expires_at_ms = ${a} + ?
|
|
3
|
+
WHERE run_id = ? AND phase = 'claimed' AND expires_at_ms <= ${a}
|
|
4
4
|
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = 'running' AND (? = 1 OR json_extract(normalized_flags_json,'$._storeGateRequired') IS NOT 1)${s?" AND tool = ? AND job_id = ?":""})
|
|
5
|
-
${s?`AND (${s.sql})`:""}`,args:[n,r,e,e,t,s?1:0,...s?[s.tool,s.jobId,...s.args]:[]]}),i=await
|
|
6
|
-
AND phase = 'effects_started' AND expires_at_ms <= ${
|
|
5
|
+
${s?`AND (${s.sql})`:""}`,args:[n,r,e,e,t,s?1:0,...s?[s.tool,s.jobId,...s.args]:[]]}),i=await d(o,e,t)),i?.phase==="effects_started"&&Number(i.expires_at_ms)<=Number(i.now_ms)&&(await o.batch([{sql:`UPDATE operation_executions SET phase = 'unknown' WHERE run_id = ?
|
|
6
|
+
AND phase = 'effects_started' AND expires_at_ms <= ${a}
|
|
7
7
|
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ?)`,args:[e,e,t]},{sql:`UPDATE operation_runs SET status = 'unknown', failure_class = 'execution_expired_after_effects',
|
|
8
8
|
ended_at = datetime('now'), updated_at = datetime('now') WHERE id = ? AND user_id = ? AND status = 'running'
|
|
9
|
-
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND phase = 'unknown')`,args:[e,t,e]}],"write"),i=await
|
|
10
|
-
AND phase = 'claimed' AND expires_at_ms > ${
|
|
9
|
+
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND phase = 'unknown')`,args:[e,t,e]}],"write"),i=await d(o,e,t)),i?.phase==="published"&&i.run_status===i.result_status?{kind:"replay",runId:e,result:{json:String(i.result_json),status:i.result_status}}:i?.phase==="unknown"||i?.run_status!=="running"?{kind:"unknown",runId:e}:i.phase==="claimed"&&i.claim_token===n&&Number(i.expires_at_ms)>Number(i.now_ms)?{kind:"claimed",claim:{runId:e,userId:t,token:n}}:{kind:"pending",runId:e}):{kind:"unknown",runId:e}}async function I(e,t){return t&&l({userId:e.userId,tool:t.tool,jobId:t.jobId},t),(await u().execute({sql:`UPDATE operation_executions SET phase = 'effects_started' WHERE run_id = ? AND claim_token = ?
|
|
10
|
+
AND phase = 'claimed' AND expires_at_ms > ${a}
|
|
11
11
|
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = 'running' AND (? = 1 OR json_extract(normalized_flags_json,'$._storeGateRequired') IS NOT 1)${t?" AND tool = ? AND job_id = ?":""})
|
|
12
|
-
${t?`AND (${t.sql})`:""}`,args:[e.runId,e.token,e.runId,e.userId,t?1:0,...t?[t.tool,t.jobId,...t.args]:[]]})).rowsAffected===1}async function
|
|
13
|
-
AND phase IN ('claimed', 'effects_started') AND expires_at_ms > ${
|
|
14
|
-
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = 'running')`,args:[t,e.runId,e.token,e.runId,e.userId]})).rowsAffected===1}async function
|
|
15
|
-
WHERE run_id = ? AND phase IN ('effects_started', 'unknown') AND expires_at_ms <= ${
|
|
12
|
+
${t?`AND (${t.sql})`:""}`,args:[e.runId,e.token,e.runId,e.userId,t?1:0,...t?[t.tool,t.jobId,...t.args]:[]]})).rowsAffected===1}async function j(e,t){return N(t),(await u().execute({sql:`UPDATE operation_executions SET expires_at_ms = ${a} + ? WHERE run_id = ? AND claim_token = ?
|
|
13
|
+
AND phase IN ('claimed', 'effects_started') AND expires_at_ms > ${a}
|
|
14
|
+
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = 'running')`,args:[t,e.runId,e.token,e.runId,e.userId]})).rowsAffected===1}async function q(e,t,n=3e5){if(N(n),!Number.isSafeInteger(t)||t<=0)throw Error("Authenticated recovery principal is required");let r=u(),o=D(),s;try{await r.batch([{sql:`UPDATE operation_executions SET phase = 'unknown', claim_token = ?, expires_at_ms = ${a} + ?
|
|
15
|
+
WHERE run_id = ? AND phase IN ('effects_started', 'unknown') AND expires_at_ms <= ${a}
|
|
16
16
|
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status IN ('running', 'unknown'))`,args:[o,n,e,e,t]},{sql:`UPDATE operation_runs SET status = 'unknown', failure_class = 'execution_expired_after_effects',
|
|
17
17
|
ended_at = datetime('now'), updated_at = datetime('now') WHERE id = ? AND user_id = ? AND status = 'running'
|
|
18
|
-
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND phase = 'unknown' AND claim_token = ?)`,args:[e,t,e,o]}],"write")}catch(_){s=_}let i;try{i=await
|
|
19
|
-
WHERE run_id = ? AND claim_token = ? AND phase = ? AND expires_at_ms > ${
|
|
18
|
+
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND phase = 'unknown' AND claim_token = ?)`,args:[e,t,e,o]}],"write")}catch(_){s=_}let i;try{i=await d(r,e,t)}catch(_){throw s??_}if(i?.phase==="unknown"&&i.run_status==="unknown"&&i.claim_token===o&&Number(i.expires_at_ms)>Number(i.now_ms))return{runId:e,userId:t,token:o};if(s)throw s;return null}async function x(e,t,n,r){r&&l({userId:e.userId,tool:r.tool,jobId:r.jobId},r);let o=n?"unknown":"effects_started",s=n?"unknown":"running";if(JSON.parse(t.json),Buffer.byteLength(t.json)>16*1024*1024)throw Error("Operation result exceeds 16 MiB");let i=u(),_;try{await i.batch([{sql:`UPDATE operation_executions SET phase = 'published', result_json = ?, result_status = ?
|
|
19
|
+
WHERE run_id = ? AND claim_token = ? AND phase = ? AND expires_at_ms > ${a}
|
|
20
20
|
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = ?)
|
|
21
21
|
AND (? = 0 OR NOT EXISTS (
|
|
22
22
|
SELECT 1 FROM operation_debit_plans p JOIN billing_debits d ON d.idempotency_key = p.debit_key
|
|
@@ -29,29 +29,43 @@ import{c as S}from"./chunk-4ZVO6GX4.js";import{a}from"./chunk-WNUXMPVX.js";impor
|
|
|
29
29
|
))${r?` AND (${r.sql}) AND EXISTS(SELECT 1 FROM operation_runs g WHERE g.id=operation_executions.run_id AND g.tool=? AND g.job_id=?)`:""}`,args:[t.json,t.status,e.runId,e.token,o,e.runId,e.userId,s,n?1:0,...r?[...r.args,r.tool,r.jobId]:[]]},{sql:`UPDATE operation_runs SET status = ?, ended_at = datetime('now'), updated_at = datetime('now')
|
|
30
30
|
WHERE id = ? AND user_id = ? AND status = ?
|
|
31
31
|
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND claim_token = ?
|
|
32
|
-
AND phase = 'published' AND result_json = ? AND result_status = ?)`,args:[t.status,e.runId,e.userId,s,e.runId,e.token,t.json,t.status]}],"write")}catch(
|
|
32
|
+
AND phase = 'published' AND result_json = ? AND result_status = ?)`,args:[t.status,e.runId,e.userId,s,e.runId,e.token,t.json,t.status]}],"write")}catch(c){_=c}let p=await d(i,e.runId,e.userId);if(p?.phase==="published"&&p.claim_token===e.token&&p.result_json===t.json&&p.result_status===t.status&&p.run_status===t.status)return!0;if(_)throw _;return!1}async function $(e,t){return x(e,t,!1)}async function H(e,t,n){if(t.status!=="unknown")throw Error("Recovery cannot publish a known execution outcome");return x(e,t,!0,n)}async function W(e){await u().execute({sql:`UPDATE operation_executions SET expires_at_ms = ${a}
|
|
33
33
|
WHERE run_id = ? AND claim_token = ? AND phase = 'claimed'
|
|
34
|
-
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = 'running')`,args:[e.runId,e.token,e.runId,e.userId]})}async function
|
|
35
|
-
|
|
34
|
+
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status = 'running')`,args:[e.runId,e.token,e.runId,e.userId]})}async function F(e){let t=u();await A(t);let n;try{return(await t.execute({sql:`UPDATE operation_executions SET phase = 'claimed', expires_at_ms = ${a}
|
|
35
|
+
WHERE run_id = ? AND claim_token = ? AND phase = 'effects_started' AND expires_at_ms > ${a}
|
|
36
|
+
AND EXISTS (SELECT 1 FROM operation_runs root
|
|
37
|
+
WHERE root.id = operation_executions.run_id AND root.user_id = ? AND root.status = 'running'
|
|
38
|
+
AND root.job_id IS NOT NULL
|
|
39
|
+
AND NOT EXISTS (SELECT 1 FROM operation_runs child
|
|
40
|
+
LEFT JOIN operation_executions execution ON execution.run_id = child.id
|
|
41
|
+
WHERE child.user_id = root.user_id AND child.job_id = root.job_id AND child.id <> root.id
|
|
42
|
+
AND (child.status = 'unknown' OR execution.phase IN ('effects_started', 'unknown')
|
|
43
|
+
OR (child.status = 'running' AND execution.run_id IS NULL)))
|
|
44
|
+
AND NOT EXISTS (SELECT 1 FROM operation_physical_attempts physical
|
|
45
|
+
WHERE physical.user_id = root.user_id AND physical.state IN ('prepared', 'effects_started', 'unknown')
|
|
46
|
+
AND (physical.parent_run_id = root.id OR EXISTS (SELECT 1 FROM operation_runs parent
|
|
47
|
+
WHERE parent.id = physical.parent_run_id AND parent.user_id = root.user_id
|
|
48
|
+
AND parent.job_id = root.job_id))))`,args:[e.runId,e.token,e.userId]})).rowsAffected===1}catch(o){n=o}let r=await d(t,e.runId,e.userId).catch(()=>null);if(r?.phase==="claimed"&&r.run_status==="running"&&r.claim_token===e.token&&Number(r.expires_at_ms)<=Number(r.now_ms))return!0;throw n}async function L(e){await u().batch([{sql:`UPDATE operation_executions SET phase = 'unknown', expires_at_ms = ${a}
|
|
49
|
+
WHERE run_id = ? AND claim_token = ? AND phase IN ('effects_started', 'unknown') AND expires_at_ms > ${a}
|
|
36
50
|
AND EXISTS (SELECT 1 FROM operation_runs WHERE id = ? AND user_id = ? AND status IN ('running', 'unknown'))`,args:[e.runId,e.token,e.runId,e.userId]},{sql:`UPDATE operation_runs SET status = 'unknown', failure_class = 'execution_outcome_unavailable',
|
|
37
51
|
ended_at = datetime('now'), updated_at = datetime('now') WHERE id = ? AND user_id = ? AND status IN ('running', 'unknown')
|
|
38
|
-
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND phase = 'unknown' AND claim_token = ?)`,args:[e.runId,e.userId,e.runId,e.token]}],"write")}async function
|
|
52
|
+
AND EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ? AND phase = 'unknown' AND claim_token = ?)`,args:[e.runId,e.userId,e.runId,e.token]}],"write")}async function P(e){let t=await d(u(),e.runId,e.userId);return t?.phase==="effects_started"&&t.run_status==="running"&&t.claim_token===e.token&&Number(t.expires_at_ms)>Number(t.now_ms)}function h(e){let t=m(e.tool,"tool"),n=m(e.ownerScope,"ownerScope"),r=e.idempotencyKey?.trim()||null,o=e.requestFingerprint?.trim()||null;if(r==null!=(o==null))throw Error("idempotencyKey and requestFingerprint must be supplied together");let s=r?f(`${n}\0${t}\0${r}`):null,i=e.id??D(),_=[i,e.userId??null,f(n),t,s,o,e.jobId??null,e.requestId??w()?.requestId??null,e.deploymentCommit??null,S(e.normalizedFlags),e.startedAt??new Date().toISOString()];return{id:i,scopeHash:s,requestFingerprint:o,statement:{sql:`INSERT${s?" OR IGNORE":""} INTO operation_runs
|
|
39
53
|
(id, user_id, owner_scope_hash, tool, idempotency_scope_hash, request_fingerprint,
|
|
40
54
|
job_id, request_id, deployment_commit, normalized_flags_json, started_at)
|
|
41
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:_}}}async function O(e,t){let n=(await e.execute({sql:t.scopeHash?"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?":"SELECT id, request_fingerprint FROM operation_runs WHERE id = ?",args:[t.scopeHash??t.id]})).rows[0];if(!n)throw Error("Operation run insert did not produce an idempotency receipt");if(t.scopeHash&&String(n.request_fingerprint)!==t.requestFingerprint)throw new E;return{id:String(n.id),replayed:String(n.id)!==t.id}}async function
|
|
42
|
-
SELECT id, ?, ${
|
|
43
|
-
WHERE id = ? AND user_id = ? AND status = 'running' AND changes() = 1`,args:[t,n,s.id,e.userId]}],"write")}catch(
|
|
55
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,args:_}}}async function O(e,t){let n=(await e.execute({sql:t.scopeHash?"SELECT id, request_fingerprint FROM operation_runs WHERE idempotency_scope_hash = ?":"SELECT id, request_fingerprint FROM operation_runs WHERE id = ?",args:[t.scopeHash??t.id]})).rows[0];if(!n)throw Error("Operation run insert did not produce an idempotency receipt");if(t.scopeHash&&String(n.request_fingerprint)!==t.requestFingerprint)throw new E;return{id:String(n.id),replayed:String(n.id)!==t.id}}async function M(e){let t=u(),n=h(e);return await t.execute(n.statement),n.scopeHash?O(t,n):{id:n.id,replayed:!1}}async function U(e,t,n,r){N(n),r&&l(e,r);let o=u(),s=h(r?{...e,normalizedFlags:{...e.normalizedFlags,_storeGateRequired:!0}}:e),i=r?{sql:s.statement.sql.replace("VALUES (","SELECT ").replace(/\)$/,"")+` WHERE (${r.sql})`,args:[...s.statement.args,...r.args]}:s.statement,_;try{await o.batch([i,{sql:`INSERT INTO operation_executions (run_id, claim_token, expires_at_ms, phase)
|
|
56
|
+
SELECT id, ?, ${a} + ?, 'claimed' FROM operation_runs
|
|
57
|
+
WHERE id = ? AND user_id = ? AND status = 'running' AND changes() = 1`,args:[t,n,s.id,e.userId]}],"write")}catch(c){_=c}let p;try{p=await O(o,s)}catch(c){if(c instanceof E)throw c;if(r&&!_)return{kind:"unknown",runId:s.id};throw _??c}return R(p.id,e.userId,t,n,o,r)}async function X(e){let t=u(),n=e.endedAt??new Date().toISOString(),r=e.failureClass==null?null:e.failureClass.slice(0,200);if((await t.execute({sql:`UPDATE operation_runs SET status = ?, failure_class = ?, ended_at = ?, updated_at = ?
|
|
44
58
|
WHERE id = ? AND status = 'running'
|
|
45
|
-
AND NOT EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ?)`,args:[e.status,r,n,n,e.runId,e.runId]})).rowsAffected===1)return;let s=(await t.execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]})).rows[0];if(!s)throw Error(`Unknown operation run: ${e.runId}`);if(s.status==="running")throw Error("Claimed operation runs require fenced result publication");let i=s.failure_class==null?null:String(s.failure_class);if(String(s.status)!==e.status||i!==r)throw Error(`Operation run ${e.runId} is already terminal as ${String(s.status)}`)}function
|
|
46
|
-
WHERE x.run_id=? AND x.claim_token=? AND x.phase=? AND x.expires_at_ms > ${
|
|
47
|
-
AND r.user_id=? AND r.status=? AND r.tool=? AND r.job_id=?)`,args:[e.runId,e.token,n,e.userId,r,t.tool,t.jobId]}}async function
|
|
59
|
+
AND NOT EXISTS (SELECT 1 FROM operation_executions WHERE run_id = ?)`,args:[e.status,r,n,n,e.runId,e.runId]})).rowsAffected===1)return;let s=(await t.execute({sql:"SELECT status, failure_class FROM operation_runs WHERE id = ?",args:[e.runId]})).rows[0];if(!s)throw Error(`Unknown operation run: ${e.runId}`);if(s.status==="running")throw Error("Claimed operation runs require fenced result publication");let i=s.failure_class==null?null:String(s.failure_class);if(String(s.status)!==e.status||i!==r)throw Error(`Operation run ${e.runId} is already terminal as ${String(s.status)}`)}function v(e,t){if(!Number.isSafeInteger(e.userId)||e.userId<=0||!e.runId||!e.token||!t.tool||!t.jobId)throw Error("Invalid operation write fence");let n=t.recovery?"unknown":"effects_started",r=t.recovery?"unknown":"running";return{sql:`EXISTS (SELECT 1 FROM operation_executions x JOIN operation_runs r ON r.id=x.run_id
|
|
60
|
+
WHERE x.run_id=? AND x.claim_token=? AND x.phase=? AND x.expires_at_ms > ${a}
|
|
61
|
+
AND r.user_id=? AND r.status=? AND r.tool=? AND r.job_id=?)`,args:[e.runId,e.token,n,e.userId,r,t.tool,t.jobId]}}async function G(e,t,n){let o=(await u().execute({sql:`SELECT r.id,r.request_fingerprint,r.status,x.phase,x.result_json,x.result_status
|
|
48
62
|
FROM operation_runs r LEFT JOIN operation_executions x ON x.run_id=r.id
|
|
49
|
-
WHERE r.user_id=? AND r.idempotency_scope_hash=?`,args:[e,t]})).rows[0];if(!o)return null;if(String(o.request_fingerprint)!==n)throw new E;return o.phase!=="published"||o.status!==o.result_status?null:{runId:String(o.id),result:{json:String(o.result_json),status:o.result_status}}}async function
|
|
50
|
-
WHERE r.job_id=? AND r.user_id=? AND r.tool=? AND (x.phase='unknown' OR r.status='unknown') LIMIT 1`,args:[e,t,n]})).rows.length>0}function
|
|
63
|
+
WHERE r.user_id=? AND r.idempotency_scope_hash=?`,args:[e,t]})).rows[0];if(!o)return null;if(String(o.request_fingerprint)!==n)throw new E;return o.phase!=="published"||o.status!==o.result_status?null:{runId:String(o.id),result:{json:String(o.result_json),status:o.result_status}}}async function z(e,t){return!!await d(u(),e,t)}async function J(e,t,n){return(await u().execute({sql:`SELECT 1 FROM operation_runs r JOIN operation_executions x ON x.run_id=r.id
|
|
64
|
+
WHERE r.job_id=? AND r.user_id=? AND r.tool=? AND (x.phase='unknown' OR r.status='unknown') LIMIT 1`,args:[e,t,n]})).rows.length>0}function V(e,t,n,r){if(![e,t].every(s=>/^[a-z_.]+$/.test(s))||!/^[a-z_]+$/.test(n))throw Error("Invalid operation group fence");if(r!=null&&(!r.trim()||r.length>512))throw Error("Invalid operation group exclusion");let o=r==null?"":`AND x.run_id <> '${r.replace(/'/g,"''")}'`;return`NOT EXISTS (SELECT 1 FROM operation_runs r JOIN operation_executions x ON x.run_id=r.id
|
|
51
65
|
WHERE r.job_id=${e} AND r.user_id=${t} AND r.tool='${n}'
|
|
52
|
-
AND x.phase IN ('claimed','effects_started') AND x.expires_at_ms > ${
|
|
53
|
-
e.phase,e.expires_at_ms,${
|
|
66
|
+
AND x.phase IN ('claimed','effects_started') AND x.expires_at_ms > ${a} AND r.status='running' ${o})`}async function K(e,t){l({userId:e.userId,tool:t.tool,jobId:t.jobId},t);try{return await I(e,t)}catch(n){let r=await d(u(),e.runId,e.userId).catch(()=>null);if(r?.phase==="effects_started"&&r.claim_token===e.token&&r.run_status==="running"&&r.tool===t.tool&&r.job_id===t.jobId&&Number(r.expires_at_ms)>Number(r.now_ms))return!0;throw n}}async function B(e,t,n,r){if(!Number.isSafeInteger(t)||t<=0||!e||!n||!r)return null;let o=(await u().execute({sql:`SELECT r.id,r.status,r.normalized_flags_json,r.request_fingerprint,r.started_at,
|
|
67
|
+
e.phase,e.expires_at_ms,${a} AS now_ms FROM operation_runs r JOIN operation_executions e ON e.run_id=r.id
|
|
54
68
|
WHERE r.id=? AND r.user_id=? AND r.tool=? AND r.job_id=?
|
|
55
|
-
AND json_extract(r.normalized_flags_json,'$._storeGateRequired') IS 1`,args:[e,t,n,r]})).rows[0];return o?{runId:String(o.id),status:String(o.status),phase:String(o.phase),flags:JSON.parse(String(o.normalized_flags_json)),fingerprint:String(o.request_fingerprint),startedAt:String(o.started_at),expiresAtMs:Number(o.expires_at_ms),nowMs:Number(o.now_ms)}:null}function
|
|
69
|
+
AND json_extract(r.normalized_flags_json,'$._storeGateRequired') IS 1`,args:[e,t,n,r]})).rows[0];return o?{runId:String(o.id),status:String(o.status),phase:String(o.phase),flags:JSON.parse(String(o.normalized_flags_json)),fingerprint:String(o.request_fingerprint),startedAt:String(o.started_at),expiresAtMs:Number(o.expires_at_ms),nowMs:Number(o.now_ms)}:null}function Q(e,t,n){if(![e,t].every(r=>/^[a-z_.]+$/.test(r))||!/^[a-z_]+$/.test(n))throw Error("Invalid unresolved operation fence");return`NOT EXISTS (SELECT 1 FROM operation_runs r JOIN operation_executions e ON e.run_id=r.id
|
|
56
70
|
WHERE r.job_id=${e} AND r.user_id=${t} AND r.tool='${n}'
|
|
57
|
-
AND (e.phase IN ('effects_started','unknown') OR r.status='unknown'))`}export{R as a,b,
|
|
71
|
+
AND (e.phase IN ('effects_started','unknown') OR r.status='unknown'))`}export{R as a,I as b,j as c,q as d,$ as e,H as f,W as g,F as h,L as i,P as j,M as k,U as l,X as m,v as n,G as o,z as p,J as q,V as r,K as s,B as t,Q as u};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{S as K}from"./chunk-
|
|
1
|
+
import{S as K}from"./chunk-WBJ6Y7WP.js";import{a as X}from"./chunk-NALXKTUU.js";import{h as J,i as H}from"./chunk-RDHGQ2WZ.js";import{a as f}from"./chunk-WNUXMPVX.js";import{createHash as U,createHmac as re,timingSafeEqual as he}from"crypto";import{createHash as fe,randomUUID as Y}from"crypto";var ge=1440*60*1e3,F=5e3,h=class extends Error{constructor(n,i,o=400){super(i);this.code=n;this.status=o;this.name="GmailSelectionError"}code;status},k=null,P=null;function $(e){return e===null||typeof e!="object"?JSON.stringify(e):Array.isArray(e)?`[${e.map($).join(",")}]`:`{${Object.entries(e).sort(([t],[n])=>t.localeCompare(n)).map(([t,n])=>`${JSON.stringify(t)}:${$(n)}`).join(",")}}`}function b(e){return fe("sha256").update($(e)).digest("hex")}async function x(){let e=f();return k&&P===e||(P=e,k=(async()=>{await e.execute(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS gmail_selections (
|
|
3
3
|
selection_id TEXT PRIMARY KEY,
|
|
4
4
|
owner_identity TEXT NOT NULL,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{a as _,b as N,c as I}from"./chunk-
|
|
1
|
+
import{a as _,b as N,c as I}from"./chunk-OY24ERPV.js";import{a as u}from"./chunk-WNUXMPVX.js";var y="2026-06-26 21:58:03";var k="+3 months";function D(t){return{sql:`INSERT INTO credit_lots (user_id, amount_mc, remaining_mc, source, expires_at)
|
|
2
2
|
SELECT ?, legacy.balance_mc, legacy.balance_mc, 'migration', datetime('now', ?)
|
|
3
3
|
FROM (
|
|
4
4
|
SELECT COALESCE(SUM(amount_mc), 0) AS balance_mc
|