mcp-scraper 0.34.0 → 0.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/bin/api-server.cjs +82 -56
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +29 -30
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-V36LS5YV.js → chunk-4ZB3X6BQ.js} +4 -3
- package/dist/chunk-4ZB3X6BQ.js.map +1 -0
- package/dist/{chunk-SUOHUXQS.js → chunk-5PZ6N2QM.js} +2 -2
- package/dist/{chunk-J32XSMJI.js → chunk-6OPHG76G.js} +16 -19
- package/dist/chunk-6OPHG76G.js.map +1 -0
- package/dist/{chunk-2EDFOQD7.js → chunk-7AYRWAEK.js} +2 -2
- package/dist/{chunk-2EDFOQD7.js.map → chunk-7AYRWAEK.js.map} +1 -1
- package/dist/chunk-7N2KYL4U.js +7 -0
- package/dist/chunk-7N2KYL4U.js.map +1 -0
- package/dist/{chunk-JBSGBSGT.js → chunk-BWXLTWF7.js} +21 -16
- package/dist/chunk-BWXLTWF7.js.map +1 -0
- package/dist/{chunk-4KQVIYIH.js → chunk-XVVNKASZ.js} +2 -2
- package/dist/index.js +2 -2
- package/dist/{server-LKVPEQFE.js → server-IWDHTES2.js} +60 -36
- package/dist/server-IWDHTES2.js.map +1 -0
- package/dist/{site-extract-repository-JHMVHENZ.js → site-extract-repository-OLVWMOU2.js} +3 -3
- package/dist/{worker-ZZHSYI3Y.js → worker-645BZPEK.js} +4 -4
- package/docs/mcp-tool-craft-lint.generated.md +171 -49
- package/docs/mcp-tool-manifest.generated.json +11 -4
- package/docs/specs/query-fanout-transport-contract-fix.md +37 -0
- package/package.json +1 -1
- package/dist/chunk-J32XSMJI.js.map +0 -1
- package/dist/chunk-JBSGBSGT.js.map +0 -1
- package/dist/chunk-NFMIHEYE.js +0 -7
- package/dist/chunk-NFMIHEYE.js.map +0 -1
- package/dist/chunk-V36LS5YV.js.map +0 -1
- package/dist/server-LKVPEQFE.js.map +0 -1
- /package/dist/{chunk-SUOHUXQS.js.map → chunk-5PZ6N2QM.js.map} +0 -0
- /package/dist/{chunk-4KQVIYIH.js.map → chunk-XVVNKASZ.js.map} +0 -0
- /package/dist/{site-extract-repository-JHMVHENZ.js.map → site-extract-repository-OLVWMOU2.js.map} +0 -0
- /package/dist/{worker-ZZHSYI3Y.js.map → worker-645BZPEK.js.map} +0 -0
package/README.md
CHANGED
|
@@ -88,7 +88,7 @@ Build the branded one-click bundle:
|
|
|
88
88
|
npm run build:mcpb
|
|
89
89
|
```
|
|
90
90
|
|
|
91
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
91
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.34.1`, SHA-256 `b217a7273ae7ee406003619de7f4bf95b4c8ad241c7f723244187c1bb557d947`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
|
|
92
92
|
|
|
93
93
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
94
94
|
|
|
@@ -213,7 +213,7 @@ Google Search Console exposes eight bounded reads and eight gated property and s
|
|
|
213
213
|
- `browser_replay_download` — download and save the replay MP4 under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`.
|
|
214
214
|
- `browser_replay_mark` — while recording, locate a DOM target and return a replay-timed annotation object.
|
|
215
215
|
- `browser_replay_annotate` — download a replay MP4, render timed boxes, circles, underlines, arrows, and labels using annotation objects from `browser_replay_mark` or exact bounds from `browser_locate`, and save a new annotated MP4 under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`.
|
|
216
|
-
- `browser_capture_fanout` — capture ChatGPT/Claude AI-search fan-out from an open logged-in hosted session.
|
|
216
|
+
- `browser_capture_fanout` — capture ChatGPT/Claude AI-search fan-out from an open logged-in hosted session. Every client receives the complete structured capture inline. Installed stdio/MCPB clients can use `export=true` for durable `fanout.json`, query/source/citation/domain/snippet CSVs, TSV, and `report.html` under `MCP_SCRAPER_OUTPUT_DIR/fanout`; hosted OAuth clients receive `exports: null` and use the inline data.
|
|
217
217
|
- `browser_close`
|
|
218
218
|
- `browser_list_sessions`
|
|
219
219
|
|
|
@@ -227,7 +227,7 @@ The `mcp-scraper` server (and the MCPB bundle, which runs it) exposes both secti
|
|
|
227
227
|
|
|
228
228
|
All MCP tools expose output schemas and return `structuredContent` with the IDs, URLs, CSV paths, transcripts, browser session handles, replay paths, artifacts, recipe fields, or blueprint fields needed by the next step. Browser Agent tools keep a JSON text block for older clients, but structured data is the primary contract. All tools carry MCP annotations; file-writing tools such as replay downloads and annotations state their filesystem side effects.
|
|
229
229
|
|
|
230
|
-
The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. Both the `mcp-scraper` stdio server and the hosted endpoint at `https://mcpscraper.dev/mcp` expose the same
|
|
230
|
+
The canonical tool inventory is generated at `docs/mcp-tool-manifest.generated.json`. Both the `mcp-scraper` stdio server and the hosted endpoint at `https://mcpscraper.dev/mcp` expose the same 167 tools: 78 scraper, browser, workflow, billing, and connected-service tools plus 89 durable-memory tools. Release verification compares the exact local and remote tool-name sets, not only the count.
|
|
231
231
|
|
|
232
232
|
For contract parity, stdio and MCPB memory calls invoke the matching public tool on the hosted MCP Scraper `/mcp` endpoint. The hosted aggregate runtime owns MCP Scraper-specific billing, scheduling, credential, and in-process cutover policy; its internal `/memory/mcp-call` bridge is a fallback to the standalone memory service, not the public stdio execution path. Direct `mcp-memory` OAuth and stdio clients continue to use `memory.mcpscraper.dev` and must be verified as a separate dependent release surface.
|
|
233
233
|
|
|
@@ -245,7 +245,7 @@ The `mcp-scraper` NPX stdio server also exposes saved reports as MCP resources:
|
|
|
245
245
|
- `BROWSER_AGENT_PROFILE_NAME` is optional and sets the default saved hosted browser profile for `mcp-scraper` stdio sessions. Aliases: `BROWSER_SERVICE_PROFILE_NAME`, `KERNEL_BROWSER_PROFILE_NAME`, `KERNEL_PROFILE_NAME`.
|
|
246
246
|
- `BROWSER_AGENT_PROFILE_SAVE_CHANGES=true` is optional hosted setup behavior. It persists cookies and storage back to the named profile when `browser_close` deletes the hosted browser session. Aliases: `BROWSER_SERVICE_PROFILE_SAVE_CHANGES`, `KERNEL_BROWSER_PROFILE_SAVE_CHANGES`, `KERNEL_PROFILE_SAVE_CHANGES`.
|
|
247
247
|
|
|
248
|
-
Every web intelligence tool call made through `mcp-scraper` saves a full Markdown report to disk by default and returns the file path in the MCP response. The hosted `/mcp` endpoint returns reports inline only and never writes files. Browser replay downloads are saved by `browser_replay_download` under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`. AI fan-out
|
|
248
|
+
Every web intelligence tool call made through `mcp-scraper` saves a full Markdown report to disk by default and returns the file path in the MCP response. The hosted `/mcp` endpoint returns reports inline only and never writes files. Browser replay downloads are saved by `browser_replay_download` under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`. AI fan-out captures are always returned inline; only installed stdio/MCPB clients write optional `export=true` files under `MCP_SCRAPER_OUTPUT_DIR/fanout`, returning relative paths. Hosted clients always receive `exports: null`.
|
|
249
249
|
|
|
250
250
|
## Updating Existing Installs
|
|
251
251
|
|
package/dist/bin/api-server.cjs
CHANGED
|
@@ -9166,7 +9166,11 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
|
|
|
9166
9166
|
topup_url: topupUrl
|
|
9167
9167
|
};
|
|
9168
9168
|
}
|
|
9169
|
-
|
|
9169
|
+
function mediaTranscribeChargeMc(durationSecs) {
|
|
9170
|
+
const mins = Math.max(1, Math.ceil((durationSecs || 60) / 60));
|
|
9171
|
+
return MEDIA_TRANSCRIBE_FLAT_MC + MEDIA_TRANSCRIBE_PER_MIN_MC * mins;
|
|
9172
|
+
}
|
|
9173
|
+
var MC_COSTS, MC_PER_BROWSER_MS, BROWSER_OPEN_MIN_BALANCE_MC, MC_PER_CREDIT, BILLING_MC_PER_USD_NUMERATOR, BILLING_MC_PER_USD_DENOMINATOR, BILLING_BASIS_POINTS, USD_NANOS, SCHEDULED_RUN_BASE_CREDITS, SCHEDULED_RUN_BASE_MC, SCHEDULED_AGENT_LLM_MARKUP_BPS, SCHEDULED_AGENT_LLM_MARKUP, SCHEDULED_RUN_DEFAULT_MAX_CREDITS, CONNECTED_USAGE_RATE_POLICY_VERSION, CONNECTED_ACTIVE_CONNECTION_MONTHLY_CREDITS, CONNECTED_ACTIVE_CONNECTION_MONTHLY_MC, CONNECTED_FUNCTION_RUN_CREDITS, CONNECTED_PROXY_REQUEST_CREDITS, CONNECTED_COMPUTE_CREDITS_PER_SECOND, CONNECTED_FUNCTION_RUN_MC, CONNECTED_PROXY_REQUEST_MC, CONNECTED_COMPUTE_MC_PER_SECOND, CONNECTED_USAGE_RATE_POLICY, SCHEDULED_ACTIONS_RATE_POLICY, CREDIT_COST_CATALOG, CONCURRENCY_PRICE_ID, SUBSCRIPTION_TIERS, SUBSCRIPTION_TIER_BY_KEY, PRIVATE_OFFERS, CONCURRENCY_SLOT_PRICE_USD, CONCURRENCY_SLOT_PRICE_CURRENCY, CONCURRENCY_SLOT_PRICE_INTERVAL, CONCURRENCY_SLOT_PRICE_LABEL, CONCURRENCY_SLOT_TERMINAL_COMMAND, MEMORY_FREE_QUOTA_BYTES, MEMORY_PLANS, MEMORY_PLAN_QUOTA, MEMORY_MARGIN_MULTIPLE, MEMORY_PRO_COST_BUDGET_USD, LedgerOperation, MEMORY_AI_MARGIN_MULTIPLE, SITE_AUDIT_LLM_MARGIN_MULTIPLE, VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE, MEDIA_TRANSCRIBE_FLAT_MC, MEDIA_TRANSCRIBE_PER_MIN_MC, MEDIA_TRANSCRIBE_HOLD_MC, MC_PER_USD;
|
|
9170
9174
|
var init_rates = __esm({
|
|
9171
9175
|
"src/api/rates.ts"() {
|
|
9172
9176
|
"use strict";
|
|
@@ -9327,9 +9331,9 @@ var init_rates = __esm({
|
|
|
9327
9331
|
key: "yt_transcription",
|
|
9328
9332
|
label: "YouTube transcription",
|
|
9329
9333
|
aliases: ["youtube_transcribe", "youtube transcript", "transcription", "yt_transcription"],
|
|
9330
|
-
credits: mcToCredits(
|
|
9331
|
-
unit: "
|
|
9332
|
-
notes: "
|
|
9334
|
+
credits: mcToCredits(2e4),
|
|
9335
|
+
unit: "base + 2/min of video",
|
|
9336
|
+
notes: "Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."
|
|
9333
9337
|
},
|
|
9334
9338
|
{
|
|
9335
9339
|
key: "fb_ad",
|
|
@@ -9390,17 +9394,17 @@ var init_rates = __esm({
|
|
|
9390
9394
|
key: "google_ads_transcribe",
|
|
9391
9395
|
label: "Google ad video transcription",
|
|
9392
9396
|
aliases: ["google_ads_transcribe", "google ad transcript"],
|
|
9393
|
-
credits: mcToCredits(
|
|
9394
|
-
unit: "
|
|
9395
|
-
notes: "
|
|
9397
|
+
credits: mcToCredits(2e4),
|
|
9398
|
+
unit: "base + 2/min of video",
|
|
9399
|
+
notes: "Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."
|
|
9396
9400
|
},
|
|
9397
9401
|
{
|
|
9398
9402
|
key: "fb_transcribe",
|
|
9399
9403
|
label: "Facebook video / ad transcription",
|
|
9400
9404
|
aliases: ["facebook_transcribe", "facebook_video_transcribe", "fb_transcribe", "fb ad transcript"],
|
|
9401
|
-
credits: mcToCredits(
|
|
9402
|
-
unit: "
|
|
9403
|
-
notes: "
|
|
9405
|
+
credits: mcToCredits(2e4),
|
|
9406
|
+
unit: "base + 2/min of video",
|
|
9407
|
+
notes: "Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."
|
|
9404
9408
|
},
|
|
9405
9409
|
{
|
|
9406
9410
|
key: "instagram_profile",
|
|
@@ -9422,9 +9426,9 @@ var init_rates = __esm({
|
|
|
9422
9426
|
key: "instagram_transcribe",
|
|
9423
9427
|
label: "Instagram media transcription",
|
|
9424
9428
|
aliases: ["instagram transcript", "instagram reel transcript", "ig transcribe"],
|
|
9425
|
-
credits: mcToCredits(
|
|
9426
|
-
unit: "
|
|
9427
|
-
notes: "Whisper transcription of selected Instagram audio/video media
|
|
9429
|
+
credits: mcToCredits(2e4),
|
|
9430
|
+
unit: "base + 2/min of video",
|
|
9431
|
+
notes: "Whisper transcription of selected Instagram audio/video media. Flat 200-credit base plus 2 credits per minute of video, reconciled to the real video length after transcription (e.g. a 10-minute video is about 220 credits)."
|
|
9428
9432
|
},
|
|
9429
9433
|
{
|
|
9430
9434
|
key: "browser_minute",
|
|
@@ -9603,8 +9607,9 @@ var init_rates = __esm({
|
|
|
9603
9607
|
MEMORY_AI_MARGIN_MULTIPLE = 3;
|
|
9604
9608
|
SITE_AUDIT_LLM_MARGIN_MULTIPLE = 1.5;
|
|
9605
9609
|
VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE = 1.5;
|
|
9606
|
-
|
|
9607
|
-
|
|
9610
|
+
MEDIA_TRANSCRIBE_FLAT_MC = 2e4;
|
|
9611
|
+
MEDIA_TRANSCRIBE_PER_MIN_MC = 200;
|
|
9612
|
+
MEDIA_TRANSCRIBE_HOLD_MC = MEDIA_TRANSCRIBE_FLAT_MC + MEDIA_TRANSCRIBE_PER_MIN_MC * 30;
|
|
9608
9613
|
MC_PER_USD = SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].credits_mc / SUBSCRIPTION_TIERS["price_1TmiHRS8aAcsk3TGwmSNfNIa"].monthly_usd;
|
|
9609
9614
|
}
|
|
9610
9615
|
});
|
|
@@ -16955,8 +16960,7 @@ var init_youtube_routes = __esm({
|
|
|
16955
16960
|
const { videoId } = parsed.data;
|
|
16956
16961
|
const id = videoId.trim();
|
|
16957
16962
|
const user = c.get("user");
|
|
16958
|
-
const
|
|
16959
|
-
const holdMc = MC_COSTS.yt_transcription * holdMins;
|
|
16963
|
+
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
16960
16964
|
const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
|
|
16961
16965
|
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
16962
16966
|
metadata: { videoId: id }
|
|
@@ -16972,8 +16976,7 @@ var init_youtube_routes = __esm({
|
|
|
16972
16976
|
const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
|
|
16973
16977
|
const durationSecs = Number.isFinite(result.durationMs) ? result.durationMs / 1e3 : 0;
|
|
16974
16978
|
const videoSecs = Math.max(chunkSecs, durationSecs) || 60;
|
|
16975
|
-
const
|
|
16976
|
-
const actualMc = MC_COSTS.yt_transcription * actualMins;
|
|
16979
|
+
const actualMc = mediaTranscribeChargeMc(videoSecs);
|
|
16977
16980
|
const diff = holdMc - actualMc;
|
|
16978
16981
|
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.TRANSCRIPTION_REFUND, "overestimate refund");
|
|
16979
16982
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.TRANSCRIPTION, id);
|
|
@@ -18211,7 +18214,6 @@ var init_facebook_ad_routes = __esm({
|
|
|
18211
18214
|
import_zod19 = require("zod");
|
|
18212
18215
|
init_db();
|
|
18213
18216
|
init_rates();
|
|
18214
|
-
init_cost_rates();
|
|
18215
18217
|
init_BrowserDriver();
|
|
18216
18218
|
init_FacebookAdExtractor();
|
|
18217
18219
|
init_FacebookAdGraphql();
|
|
@@ -18369,7 +18371,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
18369
18371
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
18370
18372
|
debited = true;
|
|
18371
18373
|
const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
|
|
18372
|
-
const settleMc =
|
|
18374
|
+
const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
|
|
18373
18375
|
const diff = holdMc - settleMc;
|
|
18374
18376
|
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
18375
18377
|
else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
|
|
@@ -18420,7 +18422,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
18420
18422
|
const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
|
|
18421
18423
|
const kernelMs = Date.now() - kernelStartMs;
|
|
18422
18424
|
const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
|
|
18423
|
-
const settleMc =
|
|
18425
|
+
const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
|
|
18424
18426
|
const diff = holdMc - settleMc;
|
|
18425
18427
|
if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
18426
18428
|
else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
|
|
@@ -18925,7 +18927,7 @@ var init_google_ads_routes = __esm({
|
|
|
18925
18927
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
18926
18928
|
debited = true;
|
|
18927
18929
|
const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
|
|
18928
|
-
const settleMc =
|
|
18930
|
+
const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
|
|
18929
18931
|
const diff = holdMc - settleMc;
|
|
18930
18932
|
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
18931
18933
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
|
|
@@ -19600,7 +19602,7 @@ var init_instagram_routes = __esm({
|
|
|
19600
19602
|
if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
|
|
19601
19603
|
transcriptDebited = true;
|
|
19602
19604
|
transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
|
|
19603
|
-
const settleMc =
|
|
19605
|
+
const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
|
|
19604
19606
|
const diff = holdMc - settleMc;
|
|
19605
19607
|
if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
|
|
19606
19608
|
else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
|
|
@@ -31109,7 +31111,7 @@ var PACKAGE_VERSION;
|
|
|
31109
31111
|
var init_version = __esm({
|
|
31110
31112
|
"src/version.ts"() {
|
|
31111
31113
|
"use strict";
|
|
31112
|
-
PACKAGE_VERSION = "0.34.
|
|
31114
|
+
PACKAGE_VERSION = "0.34.1";
|
|
31113
31115
|
}
|
|
31114
31116
|
});
|
|
31115
31117
|
|
|
@@ -34557,7 +34559,7 @@ var init_browser_agent_tool_schemas = __esm({
|
|
|
34557
34559
|
wait_ms: import_zod37.z.number().int().min(0).max(18e4).optional().describe("How long to wait for the answer stream to finish. Defaults to 90000 when a prompt is sent, 8000 for passive capture."),
|
|
34558
34560
|
first_party_domain: import_zod37.z.string().optional().describe("The brand/site being researched, e.g. example.com \u2014 sources on this domain are tagged First-party/vendor."),
|
|
34559
34561
|
reset: import_zod37.z.boolean().default(false).describe("Clear any previously buffered stream for this page before capturing."),
|
|
34560
|
-
export: import_zod37.z.boolean().default(false).describe("
|
|
34562
|
+
export: import_zod37.z.boolean().default(false).describe("When using the installed local MCP server, write JSON/CSV/TSV/HTML exports to MCP_SCRAPER_OUTPUT_DIR/fanout. Hosted clients such as ChatGPT always receive the complete structured result inline and leave exports null.")
|
|
34561
34563
|
};
|
|
34562
34564
|
FanoutSourceOutput = import_zod37.z.object({
|
|
34563
34565
|
url: import_zod37.z.string(),
|
|
@@ -34598,6 +34600,7 @@ var init_browser_agent_tool_schemas = __esm({
|
|
|
34598
34600
|
byCategory: import_zod37.z.record(import_zod37.z.number().int())
|
|
34599
34601
|
}).describe("Objective aggregates: top sourced sites by frequency, citation order, and URL-category counts."),
|
|
34600
34602
|
first_party_domain: import_zod37.z.string().nullable(),
|
|
34603
|
+
export_error: import_zod37.z.string().nullable().optional().describe("Non-fatal local export failure. The inline capture remains complete when this is present."),
|
|
34601
34604
|
exports: import_zod37.z.object({
|
|
34602
34605
|
relativeTo: import_zod37.z.string(),
|
|
34603
34606
|
dir: import_zod37.z.string(),
|
|
@@ -35167,6 +35170,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
35167
35170
|
const baseUrl = opts.baseUrl.replace(/\/$/, "");
|
|
35168
35171
|
const consoleBase = (opts.consoleBaseUrl ?? opts.baseUrl).replace(/\/$/, "");
|
|
35169
35172
|
const timeoutMs = opts.timeoutMs ?? 9e4;
|
|
35173
|
+
const savesReportsLocally = opts.savesReportsLocally !== false;
|
|
35170
35174
|
async function req(method, path6, body, requestTimeoutMs = timeoutMs) {
|
|
35171
35175
|
try {
|
|
35172
35176
|
const res = await fetch(`${baseUrl}${path6}`, {
|
|
@@ -35818,13 +35822,13 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
35818
35822
|
"query_fanout_workflow",
|
|
35819
35823
|
{
|
|
35820
35824
|
title: "Capture AI Search Fan-Out",
|
|
35821
|
-
description: "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites.
|
|
35825
|
+
description: "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Complete structured data is always returned inline for analysis. export=true additionally writes JSON/CSV/TSV/HTML only from an installed local MCP server; hosted OAuth/HTTP clients receive exports=null and use the inline data. A local export failure does not discard a successful capture. WRITE NOTE: passing prompt submits a real message in the user's logged-in account \u2014 only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview \u2014 use harvest_paa for that.",
|
|
35822
35826
|
inputSchema: BrowserCaptureFanoutInputSchema,
|
|
35823
35827
|
outputSchema: recordOutputSchema("query_fanout_workflow", BrowserCaptureFanoutOutputSchema),
|
|
35824
35828
|
annotations: annotations("Capture AI Search Fan-Out")
|
|
35825
35829
|
},
|
|
35826
35830
|
async (input) => {
|
|
35827
|
-
const emit = (result, exports3) => structuredResult({
|
|
35831
|
+
const emit = (result, exports3, exportError2 = null) => structuredResult({
|
|
35828
35832
|
ok: true,
|
|
35829
35833
|
tool: "query_fanout_workflow",
|
|
35830
35834
|
session_id: input.session_id,
|
|
@@ -35841,6 +35845,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
35841
35845
|
aggregates: result.aggregates,
|
|
35842
35846
|
first_party_domain: result.firstPartyDomain,
|
|
35843
35847
|
exports: exports3 ?? null,
|
|
35848
|
+
export_error: exportError2,
|
|
35844
35849
|
...result.debug ? { debug: result.debug } : {}
|
|
35845
35850
|
});
|
|
35846
35851
|
const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
|
|
@@ -35848,26 +35853,20 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
35848
35853
|
wait_ms: input.wait_ms,
|
|
35849
35854
|
first_party_domain: input.first_party_domain,
|
|
35850
35855
|
reset: input.reset,
|
|
35851
|
-
export:
|
|
35856
|
+
export: false
|
|
35852
35857
|
}, Math.max(timeoutMs, (input.wait_ms ?? (input.prompt ? 9e4 : 8e3)) + 3e4));
|
|
35853
35858
|
if (!res.ok) return errorResult("query_fanout_workflow", res.data, input.session_id);
|
|
35854
35859
|
const hosted = res.data?.result ?? res.data;
|
|
35855
|
-
let exports2 =
|
|
35856
|
-
|
|
35860
|
+
let exports2 = null;
|
|
35861
|
+
let exportError = null;
|
|
35862
|
+
if (input.export && savesReportsLocally && isEnrichedFanout(hosted)) {
|
|
35857
35863
|
try {
|
|
35858
35864
|
exports2 = exportFanout(hosted);
|
|
35859
35865
|
} catch (err) {
|
|
35860
|
-
|
|
35861
|
-
"query_fanout_workflow",
|
|
35862
|
-
{
|
|
35863
|
-
error: `Fan-out captured but local export failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
35864
|
-
result: hosted
|
|
35865
|
-
},
|
|
35866
|
-
input.session_id
|
|
35867
|
-
);
|
|
35866
|
+
exportError = `Fan-out captured successfully, but local export failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
35868
35867
|
}
|
|
35869
35868
|
}
|
|
35870
|
-
return emit(hosted, exports2);
|
|
35869
|
+
return emit(hosted, exports2, exportError);
|
|
35871
35870
|
}
|
|
35872
35871
|
);
|
|
35873
35872
|
}
|
|
@@ -41571,6 +41570,19 @@ var init_browser_agent_console = __esm({
|
|
|
41571
41570
|
});
|
|
41572
41571
|
|
|
41573
41572
|
// src/api/stripe-routes.ts
|
|
41573
|
+
async function cancelStandaloneMemorySub(memSubId, baseSubId) {
|
|
41574
|
+
if (!memSubId || memSubId === baseSubId) return;
|
|
41575
|
+
try {
|
|
41576
|
+
const memSub = await stripe.subscriptions.retrieve(memSubId);
|
|
41577
|
+
const priceId = memSub.items.data[0]?.price?.id;
|
|
41578
|
+
const stillOpen = memSub.status !== "canceled" && !memSub.cancel_at_period_end;
|
|
41579
|
+
if (priceId && priceId in MEMORY_PLANS && stillOpen) {
|
|
41580
|
+
await stripe.subscriptions.update(memSubId, { cancel_at_period_end: true });
|
|
41581
|
+
}
|
|
41582
|
+
} catch (err) {
|
|
41583
|
+
console.error("[stripe] could not wind down standalone memory subscription", memSubId, err instanceof Error ? err.message : err);
|
|
41584
|
+
}
|
|
41585
|
+
}
|
|
41574
41586
|
function linePriceId(line) {
|
|
41575
41587
|
const l = line;
|
|
41576
41588
|
return l?.price?.id ?? l?.pricing?.price_details?.price ?? l?.plan?.id;
|
|
@@ -41625,14 +41637,7 @@ var init_stripe_routes = __esm({
|
|
|
41625
41637
|
await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id, { neverExpires: tier.credits_never_expire });
|
|
41626
41638
|
await setSubscriptionTier(user.id, tier.tier, tier.concurrency, subId ?? user.subscription_id);
|
|
41627
41639
|
if (tier.includes_memory) {
|
|
41628
|
-
|
|
41629
|
-
if (standaloneMemorySub) {
|
|
41630
|
-
try {
|
|
41631
|
-
await stripe.subscriptions.update(standaloneMemorySub, { cancel_at_period_end: true });
|
|
41632
|
-
} catch (err) {
|
|
41633
|
-
console.error("[stripe] could not wind down standalone memory subscription", standaloneMemorySub, err instanceof Error ? err.message : err);
|
|
41634
|
-
}
|
|
41635
|
-
}
|
|
41640
|
+
await cancelStandaloneMemorySub(user.memory_subscription_id, subId ?? user.subscription_id);
|
|
41636
41641
|
await setMemoryPlan(user.id, "pro", null);
|
|
41637
41642
|
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "pro", subscription_tier: tier.tier }));
|
|
41638
41643
|
}
|
|
@@ -41684,6 +41689,7 @@ var init_stripe_routes = __esm({
|
|
|
41684
41689
|
await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
|
|
41685
41690
|
if (tier.includes_memory) {
|
|
41686
41691
|
const newMemoryPlan = live ? "pro" : "free";
|
|
41692
|
+
if (live) await cancelStandaloneMemorySub(user.memory_subscription_id, sub.id);
|
|
41687
41693
|
await setMemoryPlan(user.id, newMemoryPlan, live ? sub.id : null);
|
|
41688
41694
|
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: newMemoryPlan }));
|
|
41689
41695
|
} else if (user.memory_subscription_id === sub.id) {
|
|
@@ -50129,15 +50135,25 @@ var init_server = __esm({
|
|
|
50129
50135
|
customerId = customer.id;
|
|
50130
50136
|
await setStripeCustomerId(user.id, customerId);
|
|
50131
50137
|
}
|
|
50132
|
-
|
|
50133
|
-
|
|
50138
|
+
let existingSubId = user.subscription_id;
|
|
50139
|
+
if (!existingSubId) {
|
|
50140
|
+
try {
|
|
50141
|
+
const active = await stripeClient.subscriptions.list({ customer: customerId, status: "active", limit: 20 });
|
|
50142
|
+
const baseSub = active.data.find((s) => findBasePlanItem(s));
|
|
50143
|
+
if (baseSub) existingSubId = baseSub.id;
|
|
50144
|
+
} catch (err) {
|
|
50145
|
+
console.warn("[billing/subscribe] could not check for existing subscription", err instanceof Error ? err.message : err);
|
|
50146
|
+
}
|
|
50147
|
+
}
|
|
50148
|
+
if (existingSubId) {
|
|
50149
|
+
const sub = await stripeClient.subscriptions.retrieve(existingSubId);
|
|
50134
50150
|
const itemId = findBasePlanItem(sub)?.id;
|
|
50135
50151
|
if (itemId) {
|
|
50136
|
-
await stripeClient.subscriptions.update(
|
|
50152
|
+
await stripeClient.subscriptions.update(existingSubId, {
|
|
50137
50153
|
items: [{ id: itemId, price: tier.price_id }],
|
|
50138
50154
|
proration_behavior: "create_prorations"
|
|
50139
50155
|
});
|
|
50140
|
-
const billed = await chargeTierChangeNow(stripeClient,
|
|
50156
|
+
const billed = await chargeTierChangeNow(stripeClient, existingSubId, customerId);
|
|
50141
50157
|
if (!billed.ok) {
|
|
50142
50158
|
return c.json({
|
|
50143
50159
|
updated: true,
|
|
@@ -50255,15 +50271,25 @@ var init_server = __esm({
|
|
|
50255
50271
|
customerId = customer.id;
|
|
50256
50272
|
await setStripeCustomerId(user.id, customerId);
|
|
50257
50273
|
}
|
|
50258
|
-
|
|
50259
|
-
|
|
50274
|
+
let existingSubId = user.subscription_id;
|
|
50275
|
+
if (!existingSubId) {
|
|
50276
|
+
try {
|
|
50277
|
+
const active = await stripeClient.subscriptions.list({ customer: customerId, status: "active", limit: 20 });
|
|
50278
|
+
const baseSub = active.data.find((s) => findBasePlanItem(s));
|
|
50279
|
+
if (baseSub) existingSubId = baseSub.id;
|
|
50280
|
+
} catch (err) {
|
|
50281
|
+
console.warn("[billing/subscribe] could not check for existing subscription", err instanceof Error ? err.message : err);
|
|
50282
|
+
}
|
|
50283
|
+
}
|
|
50284
|
+
if (existingSubId) {
|
|
50285
|
+
const sub = await stripeClient.subscriptions.retrieve(existingSubId);
|
|
50260
50286
|
const itemId = findBasePlanItem(sub)?.id;
|
|
50261
50287
|
if (itemId) {
|
|
50262
|
-
await stripeClient.subscriptions.update(
|
|
50288
|
+
await stripeClient.subscriptions.update(existingSubId, {
|
|
50263
50289
|
items: [{ id: itemId, price: tier.price_id }],
|
|
50264
50290
|
proration_behavior: "create_prorations"
|
|
50265
50291
|
});
|
|
50266
|
-
const billed = await chargeTierChangeNow(stripeClient,
|
|
50292
|
+
const billed = await chargeTierChangeNow(stripeClient, existingSubId, customerId);
|
|
50267
50293
|
if (!billed.ok) {
|
|
50268
50294
|
return c.json({
|
|
50269
50295
|
updated: true,
|