mcp-scraper 0.3.45 → 0.3.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +423 -170
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +444 -184
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-BRVX6RDN.js → chunk-7TFJJAPE.js} +4 -3
- package/dist/chunk-7TFJJAPE.js.map +1 -0
- package/dist/{chunk-E7BZFMWB.js → chunk-A6BAV444.js} +481 -216
- package/dist/chunk-A6BAV444.js.map +1 -0
- package/dist/chunk-NTCAVBHU.js +7 -0
- package/dist/chunk-NTCAVBHU.js.map +1 -0
- package/dist/{server-MKUN6NQZ.js → server-H6GAAEQ6.js} +61 -85
- package/dist/server-H6GAAEQ6.js.map +1 -0
- package/dist/{worker-4CVMSUZR.js → worker-WY7C45MP.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-BRVX6RDN.js.map +0 -1
- package/dist/chunk-E7BZFMWB.js.map +0 -1
- package/dist/chunk-NVTFLLEG.js +0 -7
- package/dist/chunk-NVTFLLEG.js.map +0 -1
- package/dist/server-MKUN6NQZ.js.map +0 -1
- /package/dist/{worker-4CVMSUZR.js.map → worker-WY7C45MP.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -10015,6 +10015,15 @@ var init_blob_store = __esm({
|
|
|
10015
10015
|
(0, import_node_fs2.writeFileSync)(path6, data);
|
|
10016
10016
|
return { key, url: `file://${path6}`, bytes: byteLength(data), contentType };
|
|
10017
10017
|
}
|
|
10018
|
+
async get(key) {
|
|
10019
|
+
const path6 = (0, import_node_path2.join)(this.baseDir, "blobs", key);
|
|
10020
|
+
try {
|
|
10021
|
+
const { readFileSync: readFileSync4 } = await import("fs");
|
|
10022
|
+
return readFileSync4(path6);
|
|
10023
|
+
} catch {
|
|
10024
|
+
return null;
|
|
10025
|
+
}
|
|
10026
|
+
}
|
|
10018
10027
|
};
|
|
10019
10028
|
cached = null;
|
|
10020
10029
|
VercelBlobStore = class {
|
|
@@ -10034,6 +10043,19 @@ var init_blob_store = __esm({
|
|
|
10034
10043
|
});
|
|
10035
10044
|
return { key, url: result.url, bytes: byteLength(data), contentType };
|
|
10036
10045
|
}
|
|
10046
|
+
async get(key) {
|
|
10047
|
+
try {
|
|
10048
|
+
const { list } = await import("@vercel/blob");
|
|
10049
|
+
const res = await list({ prefix: key, token: this.token, limit: 1 });
|
|
10050
|
+
const match = res.blobs.find((b) => b.pathname === key || b.pathname.startsWith(key));
|
|
10051
|
+
if (!match) return null;
|
|
10052
|
+
const resp = await fetch(match.url);
|
|
10053
|
+
if (!resp.ok) return null;
|
|
10054
|
+
return Buffer.from(await resp.arrayBuffer());
|
|
10055
|
+
} catch {
|
|
10056
|
+
return null;
|
|
10057
|
+
}
|
|
10058
|
+
}
|
|
10037
10059
|
};
|
|
10038
10060
|
}
|
|
10039
10061
|
});
|
|
@@ -10266,8 +10288,8 @@ var init_rates = __esm({
|
|
|
10266
10288
|
label: "Video breakdown (frame-by-frame + transcript)",
|
|
10267
10289
|
aliases: ["video_frame_analysis", "video breakdown", "video analysis", "analyze video"],
|
|
10268
10290
|
credits: mcToCredits(MC_COSTS.video_analysis),
|
|
10269
|
-
unit: "per
|
|
10270
|
-
notes: "Full multi-lens video breakdown: samples
|
|
10291
|
+
unit: "per 120 frames (max 480)",
|
|
10292
|
+
notes: "Full multi-lens video breakdown: samples frames across a video (up to 30 minutes), analyzes each with vision AI, transcribes the audio, then produces summary, pacing/energy, words-per-minute, topic outline, key points, hook analysis, visual style, and a how-to-replicate recipe with a quality-control pass. $1 per 120 frames requested (max 480 = $4); billed down automatically if the video cannot use the requested frames, and refunded fully if the run fails."
|
|
10271
10293
|
}
|
|
10272
10294
|
];
|
|
10273
10295
|
CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
|
|
@@ -10343,6 +10365,7 @@ var init_rates = __esm({
|
|
|
10343
10365
|
REDDIT_THREAD_REFUND: "reddit_thread_refund",
|
|
10344
10366
|
VIDEO_ANALYSIS: "video_analysis",
|
|
10345
10367
|
VIDEO_ANALYSIS_REFUND: "video_analysis_refund",
|
|
10368
|
+
VIDEO_ANALYSIS_ADJUST: "video_analysis_adjust",
|
|
10346
10369
|
MEMORY_AI: "memory_ai",
|
|
10347
10370
|
MEMORY_AI_REFUND: "memory_ai_refund"
|
|
10348
10371
|
};
|
|
@@ -16635,6 +16658,9 @@ var init_memory = __esm({
|
|
|
16635
16658
|
function invalidRequest5(message) {
|
|
16636
16659
|
return { error_code: "invalid_request", message };
|
|
16637
16660
|
}
|
|
16661
|
+
function tiersFor(frames) {
|
|
16662
|
+
return Math.ceil(Math.min(Math.max(frames, 1), 480) / 120);
|
|
16663
|
+
}
|
|
16638
16664
|
var import_hono9, import_zod20, videoApp, AnalyzeBodySchema;
|
|
16639
16665
|
var init_video_routes = __esm({
|
|
16640
16666
|
"src/api/video-routes.ts"() {
|
|
@@ -16649,7 +16675,7 @@ var init_video_routes = __esm({
|
|
|
16649
16675
|
AnalyzeBodySchema = import_zod20.z.object({
|
|
16650
16676
|
sourceUrl: import_zod20.z.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
16651
16677
|
intervalS: import_zod20.z.number().min(1).max(30).optional(),
|
|
16652
|
-
maxFrames: import_zod20.z.number().int().min(1).max(
|
|
16678
|
+
maxFrames: import_zod20.z.number().int().min(1).max(480).optional(),
|
|
16653
16679
|
detail: import_zod20.z.enum(["fast", "standard", "deep"]).optional(),
|
|
16654
16680
|
vault: import_zod20.z.string().trim().min(1).optional()
|
|
16655
16681
|
});
|
|
@@ -16660,21 +16686,24 @@ var init_video_routes = __esm({
|
|
|
16660
16686
|
const user = c.get("user");
|
|
16661
16687
|
const { key: memKey, error: memErr } = await getOrCreateUserMemoryKey(user);
|
|
16662
16688
|
if (!memKey) return c.json({ error_code: "memory_unavailable", message: memErr ?? "memory unavailable" }, 502);
|
|
16663
|
-
const
|
|
16664
|
-
|
|
16689
|
+
const requestedFrames = Math.min(Math.max(body.maxFrames ?? 120, 1), 480);
|
|
16690
|
+
const tiers = tiersFor(requestedFrames);
|
|
16691
|
+
const holdMc = tiers * MC_COSTS.video_analysis;
|
|
16692
|
+
const { ok, balance_mc } = await debitMc(user.id, holdMc, LedgerOperation.VIDEO_ANALYSIS, body.sourceUrl);
|
|
16693
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
|
|
16665
16694
|
const started = await memoryCall(
|
|
16666
16695
|
"videoAnalyzeStartTool",
|
|
16667
16696
|
{
|
|
16668
16697
|
sourceUrl: body.sourceUrl,
|
|
16669
16698
|
intervalS: body.intervalS ?? 2,
|
|
16670
|
-
maxFrames:
|
|
16699
|
+
maxFrames: requestedFrames,
|
|
16671
16700
|
detail: body.detail ?? "standard",
|
|
16672
16701
|
vault: body.vault ?? "Library"
|
|
16673
16702
|
},
|
|
16674
16703
|
memKey
|
|
16675
16704
|
);
|
|
16676
16705
|
if (!started.ok || !started.runId) {
|
|
16677
|
-
await creditMc(user.id,
|
|
16706
|
+
await creditMc(user.id, holdMc, LedgerOperation.VIDEO_ANALYSIS_REFUND, `start-failed:${body.sourceUrl}`);
|
|
16678
16707
|
return c.json({ error_code: "video_start_failed", message: started.error ?? "could not start analysis" }, 502);
|
|
16679
16708
|
}
|
|
16680
16709
|
return c.json({
|
|
@@ -16693,11 +16722,30 @@ var init_video_routes = __esm({
|
|
|
16693
16722
|
if (!memKey) return c.json({ error_code: "memory_unavailable", message: memErr ?? "memory unavailable" }, 502);
|
|
16694
16723
|
const st = await memoryCall("videoAnalyzeStatusTool", { runId }, memKey);
|
|
16695
16724
|
if (!st.ok) return c.json({ error_code: "not_found", message: st.error ?? "run not found" }, 404);
|
|
16725
|
+
const requestedTiers = tiersFor(st.maxFrames ?? 120);
|
|
16726
|
+
let reconciliation = null;
|
|
16696
16727
|
if (st.status === "failed") {
|
|
16728
|
+
const refundMc = requestedTiers * MC_COSTS.video_analysis;
|
|
16697
16729
|
const refundOp = `${LedgerOperation.VIDEO_ANALYSIS_REFUND}:${runId}`;
|
|
16698
16730
|
if (!await ledgerExistsForOperation(user.id, refundOp)) {
|
|
16699
|
-
await creditMc(user.id,
|
|
16731
|
+
await creditMc(user.id, refundMc, refundOp, `run-failed:${runId}`);
|
|
16700
16732
|
}
|
|
16733
|
+
reconciliation = { billedMc: requestedTiers * MC_COSTS.video_analysis, refundedMc: refundMc, effectiveFrames: null };
|
|
16734
|
+
}
|
|
16735
|
+
if (st.status === "done") {
|
|
16736
|
+
const effectiveTiers = tiersFor(st.frameCount ?? 120);
|
|
16737
|
+
const diff = requestedTiers - effectiveTiers;
|
|
16738
|
+
if (diff > 0) {
|
|
16739
|
+
const adjustOp = `${LedgerOperation.VIDEO_ANALYSIS_ADJUST}:${runId}`;
|
|
16740
|
+
if (!await ledgerExistsForOperation(user.id, adjustOp)) {
|
|
16741
|
+
await creditMc(user.id, diff * MC_COSTS.video_analysis, adjustOp, `tier-reconcile:${runId}`);
|
|
16742
|
+
}
|
|
16743
|
+
}
|
|
16744
|
+
reconciliation = {
|
|
16745
|
+
billedMc: requestedTiers * MC_COSTS.video_analysis,
|
|
16746
|
+
refundedMc: Math.max(diff, 0) * MC_COSTS.video_analysis,
|
|
16747
|
+
effectiveFrames: st.frameCount ?? null
|
|
16748
|
+
};
|
|
16701
16749
|
}
|
|
16702
16750
|
return c.json({
|
|
16703
16751
|
ok: true,
|
|
@@ -16706,7 +16754,8 @@ var init_video_routes = __esm({
|
|
|
16706
16754
|
progress: st.progress ?? null,
|
|
16707
16755
|
frameCount: st.frameCount ?? null,
|
|
16708
16756
|
...st.status === "done" ? { artifactPath: st.artifactPath ?? null, report: st.report ?? null } : {},
|
|
16709
|
-
...st.error ? { error: st.error } : {}
|
|
16757
|
+
...st.error ? { error: st.error } : {},
|
|
16758
|
+
...reconciliation ? { reconciliation } : {}
|
|
16710
16759
|
});
|
|
16711
16760
|
});
|
|
16712
16761
|
}
|
|
@@ -17839,6 +17888,57 @@ var init_workflow_catalog = __esm({
|
|
|
17839
17888
|
}
|
|
17840
17889
|
});
|
|
17841
17890
|
|
|
17891
|
+
// src/mcp/report-artifact-offload.ts
|
|
17892
|
+
async function offloadReport(toolName, ownerId, report) {
|
|
17893
|
+
const timestamp2 = Date.now();
|
|
17894
|
+
const random = (0, import_node_crypto7.randomBytes)(6).toString("hex");
|
|
17895
|
+
const key = `${REPORT_BLOB_PREFIX}${ownerId}/${toolName}/${timestamp2}-${random}.md`;
|
|
17896
|
+
const stored = await getBlobStore().put(key, report, "text/markdown");
|
|
17897
|
+
return {
|
|
17898
|
+
artifactId: stored.key,
|
|
17899
|
+
bytes: stored.bytes,
|
|
17900
|
+
expiresAt: new Date(timestamp2 + REPORT_BLOB_TTL_MS).toISOString(),
|
|
17901
|
+
preview: report.slice(0, PREVIEW_CHARS)
|
|
17902
|
+
};
|
|
17903
|
+
}
|
|
17904
|
+
function artifactOwnerId(artifactId) {
|
|
17905
|
+
if (!artifactId.startsWith(REPORT_BLOB_PREFIX)) return null;
|
|
17906
|
+
const rest = artifactId.slice(REPORT_BLOB_PREFIX.length);
|
|
17907
|
+
const segment = rest.split("/")[0];
|
|
17908
|
+
return segment || null;
|
|
17909
|
+
}
|
|
17910
|
+
async function readArtifactWindow(artifactId, offset, maxBytes) {
|
|
17911
|
+
const buf = await getBlobStore().get(artifactId);
|
|
17912
|
+
if (!buf) return null;
|
|
17913
|
+
const totalBytes = buf.length;
|
|
17914
|
+
const start = Math.max(0, offset);
|
|
17915
|
+
const end = Math.min(totalBytes, start + maxBytes);
|
|
17916
|
+
const slice = buf.subarray(start, end);
|
|
17917
|
+
const nextOffset = end < totalBytes ? end : null;
|
|
17918
|
+
return { text: slice.toString("utf8"), totalBytes, nextOffset };
|
|
17919
|
+
}
|
|
17920
|
+
function summaryEnvelope(executiveSummary, offloaded) {
|
|
17921
|
+
return [
|
|
17922
|
+
executiveSummary.trim(),
|
|
17923
|
+
"",
|
|
17924
|
+
"--- Full report stored as artifact ---",
|
|
17925
|
+
`artifactId: ${offloaded.artifactId} \xB7 ${offloaded.bytes} bytes \xB7 expires ${offloaded.expiresAt}`,
|
|
17926
|
+
"Read it with report_artifact_read (supports offset/maxBytes windowing)."
|
|
17927
|
+
].join("\n");
|
|
17928
|
+
}
|
|
17929
|
+
var import_node_crypto7, REPORT_BLOB_TTL_MS, REPORT_BLOB_PREFIX, PREVIEW_CHARS, ARTIFACT_OFFLOAD_ENABLED;
|
|
17930
|
+
var init_report_artifact_offload = __esm({
|
|
17931
|
+
"src/mcp/report-artifact-offload.ts"() {
|
|
17932
|
+
"use strict";
|
|
17933
|
+
import_node_crypto7 = require("crypto");
|
|
17934
|
+
init_blob_store();
|
|
17935
|
+
REPORT_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
17936
|
+
REPORT_BLOB_PREFIX = "mcp-reports/";
|
|
17937
|
+
PREVIEW_CHARS = 2e3;
|
|
17938
|
+
ARTIFACT_OFFLOAD_ENABLED = process.env.MCP_SCRAPER_ARTIFACT_OFFLOAD !== "false";
|
|
17939
|
+
}
|
|
17940
|
+
});
|
|
17941
|
+
|
|
17842
17942
|
// src/mcp/mcp-response-formatter.ts
|
|
17843
17943
|
function configureReportSaving(enabled) {
|
|
17844
17944
|
reportSavingEnabled = enabled;
|
|
@@ -18004,6 +18104,20 @@ function oneBlock(content, diskContent) {
|
|
|
18004
18104
|
\u{1F4C4} Saved: \`${filePath}\`` : content;
|
|
18005
18105
|
return { content: [{ type: "text", text }] };
|
|
18006
18106
|
}
|
|
18107
|
+
async function maybeOffload(toolName, ctx, fullText, summaryText, structuredContent) {
|
|
18108
|
+
if (!ctx?.hosted || !ARTIFACT_OFFLOAD_ENABLED) return null;
|
|
18109
|
+
const bytes = Buffer.byteLength(fullText);
|
|
18110
|
+
if (bytes <= INLINE_BUDGET_BYTES) return null;
|
|
18111
|
+
const offloaded = await offloadReport(toolName, ctx.ownerId, fullText);
|
|
18112
|
+
return {
|
|
18113
|
+
content: [{ type: "text", text: summaryEnvelope(summaryText, offloaded) }],
|
|
18114
|
+
structuredContent: { ...structuredContent, artifact: offloaded }
|
|
18115
|
+
};
|
|
18116
|
+
}
|
|
18117
|
+
function capArray(items, cap) {
|
|
18118
|
+
if (items.length <= cap) return { items };
|
|
18119
|
+
return { items: items.slice(0, cap), truncatedCount: items.length - cap };
|
|
18120
|
+
}
|
|
18007
18121
|
function workflowRecipeTable(recipes) {
|
|
18008
18122
|
if (!recipes.length) return "";
|
|
18009
18123
|
return [
|
|
@@ -18355,7 +18469,7 @@ ${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${memSection}$
|
|
|
18355
18469
|
}
|
|
18356
18470
|
return { ...textResult, structuredContent };
|
|
18357
18471
|
}
|
|
18358
|
-
function formatMapSiteUrls(raw, input) {
|
|
18472
|
+
async function formatMapSiteUrls(raw, input, ctx) {
|
|
18359
18473
|
const parsed = parseData(raw);
|
|
18360
18474
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
18361
18475
|
const d = parsed.data;
|
|
@@ -18396,20 +18510,28 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
|
|
|
18396
18510
|
- Extract content from all pages: use \`extract_site\`
|
|
18397
18511
|
- Scrape a single page: use \`extract_url\``
|
|
18398
18512
|
].filter(Boolean).join("\n");
|
|
18399
|
-
|
|
18400
|
-
|
|
18401
|
-
|
|
18402
|
-
|
|
18403
|
-
|
|
18404
|
-
|
|
18405
|
-
|
|
18406
|
-
|
|
18407
|
-
|
|
18408
|
-
|
|
18409
|
-
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
18410
|
-
durationMs: d.durationMs ?? 0
|
|
18411
|
-
}
|
|
18513
|
+
const structuredContent = {
|
|
18514
|
+
startUrl: d.startUrl ?? input.url,
|
|
18515
|
+
totalFound: d.totalFound ?? urls.length,
|
|
18516
|
+
truncated: d.truncated === true,
|
|
18517
|
+
okCount: ok.length,
|
|
18518
|
+
redirectCount: redirects.length,
|
|
18519
|
+
brokenCount: broken.length,
|
|
18520
|
+
inventoryFile: inventoryFile ?? null,
|
|
18521
|
+
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
18522
|
+
durationMs: d.durationMs ?? 0
|
|
18412
18523
|
};
|
|
18524
|
+
const summary = `# URL Map: ${input.url}
|
|
18525
|
+
**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s
|
|
18526
|
+
|
|
18527
|
+
## Summary
|
|
18528
|
+
- 2xx: ${ok.length}
|
|
18529
|
+
- 3xx: ${redirects.length}
|
|
18530
|
+
- 4xx+: ${broken.length}`;
|
|
18531
|
+
const capped = capArray(structuredContent.urls, STRUCTURED_ARRAY_CAP);
|
|
18532
|
+
const offloaded = await maybeOffload("map_site_urls", ctx, full, summary, { ...structuredContent, urls: capped.items, truncatedCount: capped.truncatedCount });
|
|
18533
|
+
if (offloaded) return offloaded;
|
|
18534
|
+
return { ...oneBlock(full), structuredContent };
|
|
18413
18535
|
}
|
|
18414
18536
|
function buildSeoExport(siteUrl, pages, branding) {
|
|
18415
18537
|
const { edges, metrics } = buildLinkGraph(pages, siteUrl);
|
|
@@ -18429,7 +18551,7 @@ function buildSeoExport(siteUrl, pages, branding) {
|
|
|
18429
18551
|
branding: branding ?? void 0
|
|
18430
18552
|
};
|
|
18431
18553
|
}
|
|
18432
|
-
function formatExtractSite(raw, input) {
|
|
18554
|
+
async function formatExtractSite(raw, input, ctx) {
|
|
18433
18555
|
const parsed = parseData(raw);
|
|
18434
18556
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
18435
18557
|
const d = parsed.data;
|
|
@@ -18464,6 +18586,36 @@ function formatExtractSite(raw, input) {
|
|
|
18464
18586
|
schemaTypes: schemaTypesOf(p)
|
|
18465
18587
|
})));
|
|
18466
18588
|
const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
|
|
18589
|
+
const summaryHeader = [
|
|
18590
|
+
`# Site Extract: ${input.url}`,
|
|
18591
|
+
durationLine,
|
|
18592
|
+
`
|
|
18593
|
+
## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
|
|
18594
|
+
| # | Title | URL | Schema |
|
|
18595
|
+
|---|-------|-----|--------|
|
|
18596
|
+
${preview}`
|
|
18597
|
+
].join("\n");
|
|
18598
|
+
if (!bulk && ctx?.hosted) {
|
|
18599
|
+
const pageDetails2 = pages.map((p, i) => {
|
|
18600
|
+
const body = (p.bodyMarkdown ?? "").trim();
|
|
18601
|
+
return [
|
|
18602
|
+
`
|
|
18603
|
+
## ${i + 1}. ${p.title ?? "Untitled"}`,
|
|
18604
|
+
`- **URL:** ${p.url}`,
|
|
18605
|
+
p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
|
|
18606
|
+
body ? `
|
|
18607
|
+
${body}` : "_(no content extracted)_"
|
|
18608
|
+
].filter(Boolean).join("\n");
|
|
18609
|
+
}).join("\n");
|
|
18610
|
+
const fullForOffload = `${summaryHeader}
|
|
18611
|
+
|
|
18612
|
+
---
|
|
18613
|
+
# Full Page Content
|
|
18614
|
+
${pageDetails2}${tips}`;
|
|
18615
|
+
const capped = capArray(structuredContent.pages, STRUCTURED_ARRAY_CAP);
|
|
18616
|
+
const offloaded = await maybeOffload("extract_site", ctx, fullForOffload, `${summaryHeader}${tips}`, { ...structuredContent, pages: capped.items, truncatedCount: capped.truncatedCount });
|
|
18617
|
+
if (offloaded) return offloaded;
|
|
18618
|
+
}
|
|
18467
18619
|
const location2 = bulk ? `
|
|
18468
18620
|
## \u{1F4C1} Bulk scrape saved
|
|
18469
18621
|
- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
|
|
@@ -18516,7 +18668,7 @@ ${body}` : "_(no content extracted)_"
|
|
|
18516
18668
|
${pageDetails}${tips}`;
|
|
18517
18669
|
return { ...oneBlock(full, diskReport), structuredContent };
|
|
18518
18670
|
}
|
|
18519
|
-
async function formatAuditSite(raw, input) {
|
|
18671
|
+
async function formatAuditSite(raw, input, ctx) {
|
|
18520
18672
|
const parsed = parseData(raw);
|
|
18521
18673
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
18522
18674
|
const d = parsed.data;
|
|
@@ -18547,6 +18699,24 @@ async function formatAuditSite(raw, input) {
|
|
|
18547
18699
|
images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
|
|
18548
18700
|
links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
|
|
18549
18701
|
};
|
|
18702
|
+
const summary = [
|
|
18703
|
+
`# Technical SEO Audit: ${input.url}`,
|
|
18704
|
+
durationLine,
|
|
18705
|
+
`
|
|
18706
|
+
## Top issues
|
|
18707
|
+
${topIssues || "_none found_"}`,
|
|
18708
|
+
`
|
|
18709
|
+
## Links
|
|
18710
|
+
${linkLine}`,
|
|
18711
|
+
`
|
|
18712
|
+
## Images
|
|
18713
|
+
${imgLine}`
|
|
18714
|
+
].join("\n");
|
|
18715
|
+
if (!bulk && ctx?.hosted) {
|
|
18716
|
+
const fullForOffload = [summary, "\n---\n# Full Audit Report", seo.reportMd, renderImageSection(imageAudit), renderLinkReport(seo.linkReport)].join("\n");
|
|
18717
|
+
const offloaded = await maybeOffload("audit_site", ctx, fullForOffload, summary, structuredContent);
|
|
18718
|
+
if (offloaded) return offloaded;
|
|
18719
|
+
}
|
|
18550
18720
|
const location2 = bulk ? `
|
|
18551
18721
|
## \u{1F4C1} Technical audit saved
|
|
18552
18722
|
- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
|
|
@@ -19317,7 +19487,7 @@ ${rows}`,
|
|
|
19317
19487
|
}
|
|
19318
19488
|
};
|
|
19319
19489
|
}
|
|
19320
|
-
function formatDirectoryWorkflow(raw, input) {
|
|
19490
|
+
async function formatDirectoryWorkflow(raw, input, ctx) {
|
|
19321
19491
|
const parsed = parseData(raw);
|
|
19322
19492
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
19323
19493
|
const d = parsed.data;
|
|
@@ -19369,26 +19539,29 @@ ${businessRows}` : null,
|
|
|
19369
19539
|
durationMs != null ? `
|
|
19370
19540
|
*Completed in ${(durationMs / 1e3).toFixed(1)}s*` : null
|
|
19371
19541
|
].filter(Boolean).join("\n");
|
|
19372
|
-
|
|
19373
|
-
|
|
19374
|
-
|
|
19375
|
-
|
|
19376
|
-
|
|
19377
|
-
|
|
19378
|
-
|
|
19379
|
-
|
|
19380
|
-
|
|
19381
|
-
|
|
19382
|
-
|
|
19383
|
-
|
|
19384
|
-
|
|
19385
|
-
|
|
19386
|
-
|
|
19387
|
-
|
|
19388
|
-
cities,
|
|
19389
|
-
durationMs: durationMs ?? 0
|
|
19390
|
-
}
|
|
19542
|
+
const structuredContent = {
|
|
19543
|
+
query: d.query,
|
|
19544
|
+
state: d.state,
|
|
19545
|
+
minPopulation: d.minPopulation,
|
|
19546
|
+
populationYear: d.populationYear,
|
|
19547
|
+
maxResultsPerCity: d.maxResultsPerCity,
|
|
19548
|
+
concurrency: d.concurrency,
|
|
19549
|
+
censusSourceUrl: d.censusSourceUrl,
|
|
19550
|
+
usZipsSourcePath: d.usZipsSourcePath ?? null,
|
|
19551
|
+
warnings,
|
|
19552
|
+
extractedAt: d.extractedAt,
|
|
19553
|
+
selectedCityCount: d.selectedCityCount,
|
|
19554
|
+
totalResultCount,
|
|
19555
|
+
csvPath,
|
|
19556
|
+
cities,
|
|
19557
|
+
durationMs: durationMs ?? 0
|
|
19391
19558
|
};
|
|
19559
|
+
const summary = `# Directory Workflow: ${input.query}
|
|
19560
|
+
**Markets:** ${cities.length} \xB7 **Maps results:** ${totalResultCount} \xB7 **State:** ${d.state ?? input.state ?? "US"}`;
|
|
19561
|
+
const capped = capArray(cities, STRUCTURED_ARRAY_CAP);
|
|
19562
|
+
const offloaded = await maybeOffload("directory_workflow", ctx, full, summary, { ...structuredContent, cities: capped.items, truncatedCount: capped.truncatedCount });
|
|
19563
|
+
if (offloaded) return offloaded;
|
|
19564
|
+
return { ...oneBlock(full), structuredContent };
|
|
19392
19565
|
}
|
|
19393
19566
|
function formatMapsPlaceIntel(raw, input) {
|
|
19394
19567
|
const parsed = parseData(raw);
|
|
@@ -19908,7 +20081,7 @@ ${rows}` : ""
|
|
|
19908
20081
|
}
|
|
19909
20082
|
};
|
|
19910
20083
|
}
|
|
19911
|
-
var import_node_fs5, import_node_os5, import_node_path7, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD;
|
|
20084
|
+
var import_node_fs5, import_node_os5, import_node_path7, INLINE_BUDGET_BYTES, STRUCTURED_ARRAY_CAP, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD;
|
|
19912
20085
|
var init_mcp_response_formatter = __esm({
|
|
19913
20086
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
19914
20087
|
"use strict";
|
|
@@ -19921,6 +20094,9 @@ var init_mcp_response_formatter = __esm({
|
|
|
19921
20094
|
init_seo_link_report();
|
|
19922
20095
|
init_seo_issues();
|
|
19923
20096
|
init_image_audit();
|
|
20097
|
+
init_report_artifact_offload();
|
|
20098
|
+
INLINE_BUDGET_BYTES = Number(process.env.MCP_SCRAPER_INLINE_BUDGET ?? 5e4);
|
|
20099
|
+
STRUCTURED_ARRAY_CAP = 25;
|
|
19924
20100
|
reportSavingEnabled = true;
|
|
19925
20101
|
BULK_PAGE_THRESHOLD = 25;
|
|
19926
20102
|
BULK_URL_THRESHOLD = 500;
|
|
@@ -22482,7 +22658,7 @@ async function readManifestFromSummary(summary) {
|
|
|
22482
22658
|
function webhookSignature(body, timestamp2) {
|
|
22483
22659
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
22484
22660
|
if (!secret2) return null;
|
|
22485
|
-
return (0,
|
|
22661
|
+
return (0, import_node_crypto8.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
22486
22662
|
}
|
|
22487
22663
|
async function deliverWorkflowWebhook(input) {
|
|
22488
22664
|
if (!input.webhookUrl) return;
|
|
@@ -22689,11 +22865,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
22689
22865
|
}
|
|
22690
22866
|
return { dispatched: results.length, results };
|
|
22691
22867
|
}
|
|
22692
|
-
var
|
|
22868
|
+
var import_node_crypto8, import_promises9, import_hono12, import_zod29, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
22693
22869
|
var init_workflow_routes = __esm({
|
|
22694
22870
|
"src/api/workflow-routes.ts"() {
|
|
22695
22871
|
"use strict";
|
|
22696
|
-
|
|
22872
|
+
import_node_crypto8 = require("crypto");
|
|
22697
22873
|
import_promises9 = require("fs/promises");
|
|
22698
22874
|
import_hono12 = require("hono");
|
|
22699
22875
|
import_zod29 = require("zod");
|
|
@@ -22974,7 +23150,7 @@ var init_workflow_routes = __esm({
|
|
|
22974
23150
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
22975
23151
|
function sha256(value) {
|
|
22976
23152
|
if (!value) return null;
|
|
22977
|
-
return (0,
|
|
23153
|
+
return (0, import_node_crypto9.createHash)("sha256").update(value).digest("hex");
|
|
22978
23154
|
}
|
|
22979
23155
|
function countWords(markdown) {
|
|
22980
23156
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -23274,11 +23450,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
23274
23450
|
}
|
|
23275
23451
|
};
|
|
23276
23452
|
}
|
|
23277
|
-
var
|
|
23453
|
+
var import_node_crypto9, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
23278
23454
|
var init_page_snapshot_extractor = __esm({
|
|
23279
23455
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
23280
23456
|
"use strict";
|
|
23281
|
-
|
|
23457
|
+
import_node_crypto9 = require("crypto");
|
|
23282
23458
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
23283
23459
|
init_kpo_extractor();
|
|
23284
23460
|
init_url_utils();
|
|
@@ -25854,18 +26030,18 @@ var PACKAGE_VERSION;
|
|
|
25854
26030
|
var init_version = __esm({
|
|
25855
26031
|
"src/version.ts"() {
|
|
25856
26032
|
"use strict";
|
|
25857
|
-
PACKAGE_VERSION = "0.3.
|
|
26033
|
+
PACKAGE_VERSION = "0.3.47";
|
|
25858
26034
|
}
|
|
25859
26035
|
});
|
|
25860
26036
|
|
|
25861
26037
|
// src/mcp/server-instructions.ts
|
|
25862
|
-
|
|
25863
|
-
|
|
25864
|
-
|
|
25865
|
-
"use strict";
|
|
25866
|
-
SERVER_INSTRUCTIONS = `
|
|
26038
|
+
function serverInstructions(savesReportsLocally) {
|
|
26039
|
+
const reportLine = savesReportsLocally ? "All report-producing tools also save a full Markdown report to disk." : "On this hosted endpoint, small reports return inline; large reports are stored as artifacts \u2014 read them back with report_artifact_read.";
|
|
26040
|
+
return `
|
|
25867
26041
|
# MCP Scraper
|
|
25868
26042
|
|
|
26043
|
+
${reportLine}
|
|
26044
|
+
|
|
25869
26045
|
Scrapes and analyzes web, search, maps, social, and site data. **Pick a tool by NAME from the map
|
|
25870
26046
|
below, then load its schema before calling.** Where a tool needs a value another tool produces, the
|
|
25871
26047
|
seam is noted so you can chain them.
|
|
@@ -25949,14 +26125,34 @@ Multi-step orchestrations \u2014 prefer these over hand-chaining primitives when
|
|
|
25949
26125
|
sites. It returns a saved folder/artifact plus a summary, not the full content inline.
|
|
25950
26126
|
- Browser sessions and media transcription cost credits and take longer \u2014 prefer the cheapest tool that
|
|
25951
26127
|
answers the question (plain search/extract before browser agents).
|
|
25952
|
-
- Large results are saved to disk or an artifact and returned as a summary plus a path
|
|
25953
|
-
full detail rather than expecting the whole payload inline.
|
|
26128
|
+
- Large results are saved to disk or an artifact and returned as a summary plus a path or artifactId;
|
|
26129
|
+
read it back for full detail rather than expecting the whole payload inline.
|
|
25954
26130
|
`.trim();
|
|
26131
|
+
}
|
|
26132
|
+
var SERVER_INSTRUCTIONS;
|
|
26133
|
+
var init_server_instructions = __esm({
|
|
26134
|
+
"src/mcp/server-instructions.ts"() {
|
|
26135
|
+
"use strict";
|
|
26136
|
+
SERVER_INSTRUCTIONS = serverInstructions(true);
|
|
26137
|
+
}
|
|
26138
|
+
});
|
|
26139
|
+
|
|
26140
|
+
// src/mcp/output-schema-registry.ts
|
|
26141
|
+
function recordOutputSchema(name, schema) {
|
|
26142
|
+
OUTPUT_SCHEMAS[name] = schema;
|
|
26143
|
+
return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
|
|
26144
|
+
}
|
|
26145
|
+
var ADVERTISE_OUTPUT_SCHEMAS, OUTPUT_SCHEMAS;
|
|
26146
|
+
var init_output_schema_registry = __esm({
|
|
26147
|
+
"src/mcp/output-schema-registry.ts"() {
|
|
26148
|
+
"use strict";
|
|
26149
|
+
ADVERTISE_OUTPUT_SCHEMAS = process.env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true";
|
|
26150
|
+
OUTPUT_SCHEMAS = {};
|
|
25955
26151
|
}
|
|
25956
26152
|
});
|
|
25957
26153
|
|
|
25958
26154
|
// src/mcp/mcp-tool-schemas.ts
|
|
25959
|
-
var import_zod31, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
26155
|
+
var import_zod31, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema;
|
|
25960
26156
|
var init_mcp_tool_schemas = __esm({
|
|
25961
26157
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
25962
26158
|
"use strict";
|
|
@@ -26030,7 +26226,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26030
26226
|
VideoFrameAnalysisInputSchema = {
|
|
26031
26227
|
sourceUrl: import_zod31.z.string().min(1).describe("A DIRECT video file URL (.mp4/.webm/.mov). Not a YouTube/Facebook/Instagram page URL \u2014 resolve those to a direct media URL first. Videos up to 30 minutes are supported."),
|
|
26032
26228
|
intervalS: import_zod31.z.number().min(1).max(30).optional().describe("Preferred seconds between sampled frames (1-30, default 2). Automatically widened for long videos so the whole duration is covered within the frame budget."),
|
|
26033
|
-
maxFrames: import_zod31.z.number().int().min(1).max(
|
|
26229
|
+
maxFrames: import_zod31.z.number().int().min(1).max(480).optional().describe("Max frames analyzed (<=480, default 120). $1 per 120 frames requested \u2014 120=$1 \u2026 480=$4 \u2014 automatically refunded down if the video cannot use them (minimum 1s between frames). Frames are spread evenly across the whole video."),
|
|
26034
26230
|
detail: import_zod31.z.enum(["fast", "standard", "deep"]).optional().describe("Analysis depth. Default standard."),
|
|
26035
26231
|
vault: import_zod31.z.string().min(1).optional().describe('Memory vault to save the finished breakdown into. Default "Library".')
|
|
26036
26232
|
};
|
|
@@ -26105,12 +26301,18 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26105
26301
|
maxResultsPerCity: import_zod31.z.number().int().min(1).max(50).default(50).describe("Google Maps candidates to collect per city."),
|
|
26106
26302
|
concurrency: import_zod31.z.number().int().min(1).max(5).default(5).describe("City Maps searches to run in parallel."),
|
|
26107
26303
|
includeZipGroups: import_zod31.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available (MCP_SCRAPER_USZIPS_CSV_PATH or usZipsCsvPath)."),
|
|
26108
|
-
usZipsCsvPath: import_zod31.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
26304
|
+
usZipsCsvPath: import_zod31.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead. For ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the server, or pass this in local/test mode."),
|
|
26109
26305
|
saveCsv: import_zod31.z.boolean().default(true).describe("Save a directory-ready CSV of results to the MCP Scraper output directory and return its path."),
|
|
26110
26306
|
proxyMode: import_zod31.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting per city search. Defaults to location (city/ZIP-group residential targeting); configured forces the service proxy; none is local debugging only."),
|
|
26111
26307
|
proxyZip: import_zod31.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting; normally omitted."),
|
|
26112
26308
|
debug: import_zod31.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
26113
26309
|
};
|
|
26310
|
+
ArtifactPointerOutputSchema = import_zod31.z.object({
|
|
26311
|
+
artifactId: import_zod31.z.string(),
|
|
26312
|
+
bytes: import_zod31.z.number().int().min(0),
|
|
26313
|
+
expiresAt: import_zod31.z.string(),
|
|
26314
|
+
preview: import_zod31.z.string()
|
|
26315
|
+
});
|
|
26114
26316
|
RankTrackerModeSchema = import_zod31.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
26115
26317
|
RankTrackerBlueprintInputSchema = {
|
|
26116
26318
|
projectName: import_zod31.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
@@ -26224,7 +26426,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26224
26426
|
attempts: import_zod31.z.array(MapsSearchAttemptOutput),
|
|
26225
26427
|
results: import_zod31.z.array(DirectoryMapsBusinessOutput)
|
|
26226
26428
|
})),
|
|
26227
|
-
durationMs: import_zod31.z.number().int().min(0)
|
|
26429
|
+
durationMs: import_zod31.z.number().int().min(0),
|
|
26430
|
+
truncatedCount: import_zod31.z.number().int().min(0).optional(),
|
|
26431
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26228
26432
|
};
|
|
26229
26433
|
RankTrackerToolPlanOutput = import_zod31.z.object({
|
|
26230
26434
|
tool: import_zod31.z.string(),
|
|
@@ -26344,7 +26548,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26344
26548
|
title: NullableString,
|
|
26345
26549
|
schemaTypes: import_zod31.z.array(import_zod31.z.string())
|
|
26346
26550
|
})),
|
|
26347
|
-
durationMs: import_zod31.z.number().min(0)
|
|
26551
|
+
durationMs: import_zod31.z.number().min(0),
|
|
26552
|
+
truncatedCount: import_zod31.z.number().int().min(0).optional(),
|
|
26553
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26348
26554
|
};
|
|
26349
26555
|
AuditSiteOutputSchema = {
|
|
26350
26556
|
url: import_zod31.z.string(),
|
|
@@ -26364,7 +26570,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26364
26570
|
orphans: import_zod31.z.number().int().min(0),
|
|
26365
26571
|
brokenInternal: import_zod31.z.number().int().min(0),
|
|
26366
26572
|
externalDomains: import_zod31.z.number().int().min(0)
|
|
26367
|
-
})
|
|
26573
|
+
}),
|
|
26574
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26368
26575
|
};
|
|
26369
26576
|
MapsPlaceIntelOutputSchema = {
|
|
26370
26577
|
name: import_zod31.z.string(),
|
|
@@ -26436,7 +26643,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26436
26643
|
url: import_zod31.z.string(),
|
|
26437
26644
|
status: import_zod31.z.number().int().nullable()
|
|
26438
26645
|
})),
|
|
26439
|
-
durationMs: import_zod31.z.number().min(0)
|
|
26646
|
+
durationMs: import_zod31.z.number().min(0),
|
|
26647
|
+
truncatedCount: import_zod31.z.number().int().min(0).optional(),
|
|
26648
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26440
26649
|
};
|
|
26441
26650
|
YoutubeHarvestOutputSchema = {
|
|
26442
26651
|
mode: import_zod31.z.string(),
|
|
@@ -26480,7 +26689,12 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26480
26689
|
frameCount: import_zod31.z.number().int().nullable().optional(),
|
|
26481
26690
|
artifactPath: NullableString,
|
|
26482
26691
|
report: NullableString,
|
|
26483
|
-
error: NullableString
|
|
26692
|
+
error: NullableString,
|
|
26693
|
+
reconciliation: import_zod31.z.object({
|
|
26694
|
+
billedMc: import_zod31.z.number().int(),
|
|
26695
|
+
refundedMc: import_zod31.z.number().int(),
|
|
26696
|
+
effectiveFrames: import_zod31.z.number().int().nullable()
|
|
26697
|
+
}).nullable().optional()
|
|
26484
26698
|
};
|
|
26485
26699
|
RedditThreadOutputSchema = {
|
|
26486
26700
|
sourceUrl: NullableString,
|
|
@@ -26868,6 +27082,17 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26868
27082
|
timeoutMs: import_zod31.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
26869
27083
|
debug: import_zod31.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
26870
27084
|
};
|
|
27085
|
+
ReportArtifactReadInputSchema = {
|
|
27086
|
+
artifactId: import_zod31.z.string().min(1).describe("Artifact id returned inline by a tool whose result was too large to inline. Use only a returned artifactId; do not construct one yourself."),
|
|
27087
|
+
offset: import_zod31.z.number().int().min(0).default(0).describe("Byte offset to start reading from. Pass the previous call's nextOffset to continue."),
|
|
27088
|
+
maxBytes: import_zod31.z.number().int().min(1e3).max(1e5).default(2e4).describe("Maximum bytes of artifact text to return in this window.")
|
|
27089
|
+
};
|
|
27090
|
+
ReportArtifactReadOutputSchema = {
|
|
27091
|
+
artifactId: import_zod31.z.string(),
|
|
27092
|
+
text: import_zod31.z.string(),
|
|
27093
|
+
totalBytes: import_zod31.z.number().int().min(0),
|
|
27094
|
+
nextOffset: import_zod31.z.number().int().min(0).nullable()
|
|
27095
|
+
};
|
|
26871
27096
|
}
|
|
26872
27097
|
});
|
|
26873
27098
|
|
|
@@ -27201,6 +27426,9 @@ var init_rank_tracker_blueprint = __esm({
|
|
|
27201
27426
|
});
|
|
27202
27427
|
|
|
27203
27428
|
// src/mcp/paa-mcp-server.ts
|
|
27429
|
+
function hashOwnerId(callerKey) {
|
|
27430
|
+
return (0, import_node_crypto10.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
27431
|
+
}
|
|
27204
27432
|
function liveWebToolAnnotations(title) {
|
|
27205
27433
|
return {
|
|
27206
27434
|
title,
|
|
@@ -27213,16 +27441,16 @@ function liveWebToolAnnotations(title) {
|
|
|
27213
27441
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
27214
27442
|
server.registerTool("capture_serp_snapshot", {
|
|
27215
27443
|
title: "SERP Intelligence Snapshot",
|
|
27216
|
-
description: "Capture a structured SERP Intelligence Google
|
|
27444
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
27217
27445
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
27218
|
-
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
27446
|
+
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
27219
27447
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
27220
27448
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
27221
27449
|
server.registerTool("capture_serp_page_snapshots", {
|
|
27222
27450
|
title: "SERP Intelligence Page Snapshots",
|
|
27223
|
-
description: "Capture public ranking
|
|
27451
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
|
|
27224
27452
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
27225
|
-
outputSchema: CaptureSerpPageSnapshotsOutputSchema,
|
|
27453
|
+
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
27226
27454
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
27227
27455
|
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
27228
27456
|
}
|
|
@@ -27270,230 +27498,251 @@ function registerSavedReportResources(server) {
|
|
|
27270
27498
|
);
|
|
27271
27499
|
}
|
|
27272
27500
|
function buildPaaExtractorMcpServer(executor, options = {}) {
|
|
27273
|
-
const server = new import_mcp.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions:
|
|
27501
|
+
const server = new import_mcp.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: serverInstructions(options.savesReportsLocally !== false) });
|
|
27274
27502
|
registerPaaExtractorMcpTools(server, executor, options);
|
|
27275
27503
|
return server;
|
|
27276
27504
|
}
|
|
27277
27505
|
function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
27278
27506
|
const savesReports = options.savesReportsLocally !== false;
|
|
27279
|
-
const
|
|
27280
|
-
const
|
|
27507
|
+
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
27508
|
+
const ownerId = options.ownerId ?? "local";
|
|
27509
|
+
const ctx = { hosted: !savesReports, ownerId };
|
|
27281
27510
|
if (savesReports) registerSavedReportResources(server);
|
|
27282
27511
|
server.registerTool("harvest_paa", {
|
|
27283
27512
|
title: "Google PAA + SERP Harvest",
|
|
27284
|
-
description:
|
|
27513
|
+
description: "Best default tool for Google search research: People Also Ask questions with answers/sources, organic SERP, local pack, entity IDs, and AI Overview. Split topic from location; leave proxyMode unset. Warn the user before maxQuestions above 100 \u2014 deep harvests can run several minutes with no interim progress, billed per extracted question.",
|
|
27285
27514
|
inputSchema: HarvestPaaInputSchema,
|
|
27286
|
-
outputSchema: HarvestPaaOutputSchema,
|
|
27515
|
+
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
27287
27516
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
27288
27517
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
27289
27518
|
server.registerTool("search_serp", {
|
|
27290
27519
|
title: "Google SERP Lookup",
|
|
27291
|
-
description:
|
|
27520
|
+
description: "Fast Google SERP lookup without PAA expansion \u2014 rankings, organic results, local pack, positions. Split topic from location; leave proxyMode unset.",
|
|
27292
27521
|
inputSchema: SearchSerpInputSchema,
|
|
27293
|
-
outputSchema: SearchSerpOutputSchema,
|
|
27522
|
+
outputSchema: recordOutputSchema("search_serp", SearchSerpOutputSchema),
|
|
27294
27523
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
27295
27524
|
}, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
|
|
27296
27525
|
server.registerTool("extract_url", {
|
|
27297
27526
|
title: "Single URL Extract",
|
|
27298
|
-
description:
|
|
27527
|
+
description: "Extract structured data from one public URL: content, schema, headings, metadata, screenshots, branding, or media assets. Set depositToVault:true to save the full page into the user's MCP Memory vault server-side (not returned to chat).",
|
|
27299
27528
|
inputSchema: ExtractUrlInputSchema,
|
|
27300
|
-
outputSchema: ExtractUrlOutputSchema,
|
|
27529
|
+
outputSchema: recordOutputSchema("extract_url", ExtractUrlOutputSchema),
|
|
27301
27530
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
27302
27531
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
27303
27532
|
server.registerTool("map_site_urls", {
|
|
27304
27533
|
title: "Site URL Map",
|
|
27305
|
-
description:
|
|
27534
|
+
description: `Map/crawl a public website for a sitemap, URL inventory, or broken-link scan. Returns internal URLs with HTTP status; ${fileBehavior("maps over 500 URLs are written to a local CSV file instead of inlined.", "large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
27306
27535
|
inputSchema: MapSiteUrlsInputSchema,
|
|
27307
|
-
outputSchema: MapSiteUrlsOutputSchema,
|
|
27536
|
+
outputSchema: recordOutputSchema("map_site_urls", MapSiteUrlsOutputSchema),
|
|
27308
27537
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
27309
|
-
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
27538
|
+
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input, ctx));
|
|
27310
27539
|
server.registerTool("extract_site", {
|
|
27311
27540
|
title: "Multi-Page Site Content Crawl",
|
|
27312
|
-
description:
|
|
27541
|
+
description: `Crawl a public website and return page CONTENT (Markdown) across multiple pages. ${fileBehavior("Bulk crawls over 25 pages are saved as per-page Markdown files in a local folder instead of inlined.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Content only \u2014 for a technical SEO audit use audit_site instead.`,
|
|
27313
27542
|
inputSchema: ExtractSiteInputSchema,
|
|
27314
|
-
outputSchema: ExtractSiteOutputSchema,
|
|
27543
|
+
outputSchema: recordOutputSchema("extract_site", ExtractSiteOutputSchema),
|
|
27315
27544
|
annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
|
|
27316
|
-
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
27545
|
+
}, async (input) => formatExtractSite(await executor.extractSite(input), input, ctx));
|
|
27317
27546
|
server.registerTool("audit_site", {
|
|
27318
27547
|
title: "Technical SEO Audit",
|
|
27319
|
-
description:
|
|
27548
|
+
description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. ${fileBehavior("Writes a folder of analysis files plus per-page content, and returns a summary plus the folder path.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Use extract_site instead for plain page content.`,
|
|
27320
27549
|
inputSchema: AuditSiteInputSchema,
|
|
27321
|
-
outputSchema: AuditSiteOutputSchema,
|
|
27550
|
+
outputSchema: recordOutputSchema("audit_site", AuditSiteOutputSchema),
|
|
27322
27551
|
annotations: liveWebToolAnnotations("Technical SEO Audit")
|
|
27323
|
-
}, async (input) => formatAuditSite(await executor.auditSite(input), input));
|
|
27552
|
+
}, async (input) => formatAuditSite(await executor.auditSite(input), input, ctx));
|
|
27324
27553
|
server.registerTool("youtube_harvest", {
|
|
27325
27554
|
title: "YouTube Video Harvest",
|
|
27326
|
-
description:
|
|
27555
|
+
description: 'Harvest YouTube video metadata by topic search or channel library. Use mode "search" for keyword/topic requests, mode "channel" for @handles/channel IDs/URLs. Returns titles, views, durations, and videoIds.',
|
|
27327
27556
|
inputSchema: YoutubeHarvestInputSchema,
|
|
27328
|
-
outputSchema: YoutubeHarvestOutputSchema,
|
|
27557
|
+
outputSchema: recordOutputSchema("youtube_harvest", YoutubeHarvestOutputSchema),
|
|
27329
27558
|
annotations: liveWebToolAnnotations("YouTube Video Harvest")
|
|
27330
27559
|
}, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
|
|
27331
27560
|
server.registerTool("youtube_transcribe", {
|
|
27332
27561
|
title: "YouTube Transcription",
|
|
27333
|
-
description:
|
|
27562
|
+
description: "Fetch and transcribe captions from a YouTube video. Pass videoId from youtube_harvest, or a url the user pasted. Returns full transcript, timestamped chunks, and word count.",
|
|
27334
27563
|
inputSchema: YoutubeTranscribeInputSchema,
|
|
27335
|
-
outputSchema: YoutubeTranscribeOutputSchema,
|
|
27564
|
+
outputSchema: recordOutputSchema("youtube_transcribe", YoutubeTranscribeOutputSchema),
|
|
27336
27565
|
annotations: liveWebToolAnnotations("YouTube Transcription")
|
|
27337
27566
|
}, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
|
|
27338
27567
|
server.registerTool("facebook_page_intel", {
|
|
27339
27568
|
title: "Facebook Advertiser Ad Intel",
|
|
27340
|
-
description:
|
|
27569
|
+
description: "Harvest ads from a Facebook advertiser: copy, creative angles, CTAs, and direct video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name.",
|
|
27341
27570
|
inputSchema: FacebookPageIntelInputSchema,
|
|
27342
|
-
outputSchema: FacebookPageIntelOutputSchema,
|
|
27571
|
+
outputSchema: recordOutputSchema("facebook_page_intel", FacebookPageIntelOutputSchema),
|
|
27343
27572
|
annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
|
|
27344
27573
|
}, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
|
|
27345
27574
|
server.registerTool("facebook_ad_search", {
|
|
27346
27575
|
title: "Facebook Ad Library Search",
|
|
27347
|
-
description:
|
|
27576
|
+
description: "Search Facebook Ad Library to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs.",
|
|
27348
27577
|
inputSchema: FacebookAdSearchInputSchema,
|
|
27349
|
-
outputSchema: FacebookAdSearchOutputSchema,
|
|
27578
|
+
outputSchema: recordOutputSchema("facebook_ad_search", FacebookAdSearchOutputSchema),
|
|
27350
27579
|
annotations: liveWebToolAnnotations("Facebook Ad Library Search")
|
|
27351
27580
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
27352
27581
|
server.registerTool("reddit_thread", {
|
|
27353
27582
|
title: "Reddit Thread + Comments",
|
|
27354
|
-
description:
|
|
27583
|
+
description: "Capture a Reddit post and its comment tree from a reddit.com thread URL \u2014 comments, opinions, audience voice. Handles Reddit's bot protection automatically; pass maxComments to cap the list.",
|
|
27355
27584
|
inputSchema: RedditThreadInputSchema,
|
|
27356
|
-
outputSchema: RedditThreadOutputSchema,
|
|
27585
|
+
outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
|
|
27357
27586
|
annotations: liveWebToolAnnotations("Reddit Thread + Comments")
|
|
27358
27587
|
}, async (input) => formatRedditThread(await executor.redditThread(input), input));
|
|
27359
27588
|
server.registerTool("video_frame_analysis", {
|
|
27360
27589
|
title: "Video Breakdown (frame-by-frame + transcript)",
|
|
27361
|
-
description: "Produce a deep frame-by-frame + transcript breakdown of a video \u2014 pacing, hook, visual style, and how to replicate it. Pass a DIRECT video file URL (.mp4/.webm/.mov), not a page URL.
|
|
27590
|
+
description: "Produce a deep frame-by-frame + transcript breakdown of a video \u2014 pacing, hook, visual style, and how to replicate it. Pass a DIRECT video file URL (.mp4/.webm/.mov), not a page URL. Costs $1 per 120 frames requested (max 480 = $4; refunded down if the video can't use them; refunded fully on failure): returns a runId immediately; poll video_frame_analysis_status until done. Videos up to 30 minutes.",
|
|
27362
27591
|
inputSchema: VideoFrameAnalysisInputSchema,
|
|
27363
|
-
outputSchema: VideoFrameAnalysisOutputSchema,
|
|
27592
|
+
outputSchema: recordOutputSchema("video_frame_analysis", VideoFrameAnalysisOutputSchema),
|
|
27364
27593
|
annotations: { title: "Video Breakdown", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
|
|
27365
27594
|
}, async (input) => executor.videoFrameAnalysis(input));
|
|
27366
27595
|
server.registerTool("video_frame_analysis_status", {
|
|
27367
27596
|
title: "Video Breakdown Status",
|
|
27368
|
-
description: 'Check progress of a video breakdown started with video_frame_analysis, using its runId. Free to call. When status is "done" it returns the full report and vault path; stop polling on "done" or "failed".',
|
|
27597
|
+
description: 'Check progress of a video breakdown started with video_frame_analysis, using its runId. Free to call. When status is "done" it returns the full report and vault path; stop polling on "done" or "failed". Reports the billed tier reconciliation when done.',
|
|
27369
27598
|
inputSchema: VideoFrameAnalysisStatusInputSchema,
|
|
27370
|
-
outputSchema: VideoFrameAnalysisStatusOutputSchema,
|
|
27599
|
+
outputSchema: recordOutputSchema("video_frame_analysis_status", VideoFrameAnalysisStatusOutputSchema),
|
|
27371
27600
|
annotations: { title: "Video Breakdown Status", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
27372
27601
|
}, async (input) => executor.videoFrameAnalysisStatus(input));
|
|
27373
27602
|
server.registerTool("facebook_ad_transcribe", {
|
|
27374
27603
|
title: "Facebook Ad Transcription",
|
|
27375
27604
|
description: "Transcribe audio from a Facebook ad video CDN URL returned by facebook_page_intel. Use only that direct videoUrl value \u2014 do not pass public Facebook post/reel/share URLs (use facebook_video_transcribe for those).",
|
|
27376
27605
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
27377
|
-
outputSchema: FacebookAdTranscribeOutputSchema,
|
|
27606
|
+
outputSchema: recordOutputSchema("facebook_ad_transcribe", FacebookAdTranscribeOutputSchema),
|
|
27378
27607
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
27379
27608
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
27380
27609
|
server.registerTool("google_ads_search", {
|
|
27381
27610
|
title: "Google Ads Transparency Search",
|
|
27382
|
-
description:
|
|
27611
|
+
description: "Search the Google Ads Transparency Center to find advertisers by domain or brand name. Returns advertisers with advertiser ID and approximate ad count, to pass to google_ads_page_intel.",
|
|
27383
27612
|
inputSchema: GoogleAdsSearchInputSchema,
|
|
27384
|
-
outputSchema: GoogleAdsSearchOutputSchema,
|
|
27613
|
+
outputSchema: recordOutputSchema("google_ads_search", GoogleAdsSearchOutputSchema),
|
|
27385
27614
|
annotations: liveWebToolAnnotations("Google Ads Transparency Search")
|
|
27386
27615
|
}, async (input) => formatGoogleAdsSearch(await executor.googleAdsSearch(input), input));
|
|
27387
27616
|
server.registerTool("google_ads_page_intel", {
|
|
27388
27617
|
title: "Google Ads Advertiser Intel",
|
|
27389
|
-
description:
|
|
27618
|
+
description: "Harvest an advertiser's ad creatives from the Google Ads Transparency Center: format, image URLs, and \u2014 for video ads \u2014 a YouTube video ID or direct video URL. Accepts advertiserId (from google_ads_search) or a domain.",
|
|
27390
27619
|
inputSchema: GoogleAdsPageIntelInputSchema,
|
|
27391
|
-
outputSchema: GoogleAdsPageIntelOutputSchema,
|
|
27620
|
+
outputSchema: recordOutputSchema("google_ads_page_intel", GoogleAdsPageIntelOutputSchema),
|
|
27392
27621
|
annotations: liveWebToolAnnotations("Google Ads Advertiser Intel")
|
|
27393
27622
|
}, async (input) => formatGoogleAdsPageIntel(await executor.googleAdsPageIntel(input), input));
|
|
27394
27623
|
server.registerTool("google_ads_transcribe", {
|
|
27395
27624
|
title: "Google Ad Video Transcription",
|
|
27396
27625
|
description: "Transcribe audio from a Google video ad's direct videoUrl (a googlevideo.com playback URL) returned by google_ads_page_intel. For YouTube-hosted ads, use youtube_transcribe with the returned youtubeVideoId instead.",
|
|
27397
27626
|
inputSchema: GoogleAdsTranscribeInputSchema,
|
|
27398
|
-
outputSchema: GoogleAdsTranscribeOutputSchema,
|
|
27627
|
+
outputSchema: recordOutputSchema("google_ads_transcribe", GoogleAdsTranscribeOutputSchema),
|
|
27399
27628
|
annotations: liveWebToolAnnotations("Google Ad Video Transcription")
|
|
27400
27629
|
}, async (input) => formatGoogleAdsTranscribe(await executor.googleAdsTranscribe(input), input));
|
|
27401
27630
|
server.registerTool("facebook_video_transcribe", {
|
|
27402
27631
|
title: "Facebook Organic Video Transcription",
|
|
27403
|
-
description:
|
|
27632
|
+
description: "Transcribe audio from an organic Facebook reel/video/post/share URL (including fb.watch). Renders the page, selects the best public CDN MP4, and returns the transcript plus resolved video metadata.",
|
|
27404
27633
|
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
27405
|
-
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
27634
|
+
outputSchema: recordOutputSchema("facebook_video_transcribe", FacebookVideoTranscribeOutputSchema),
|
|
27406
27635
|
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
27407
27636
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
27408
27637
|
server.registerTool("instagram_profile_content", {
|
|
27409
27638
|
title: "Instagram Profile Content Discovery",
|
|
27410
|
-
description:
|
|
27639
|
+
description: "Discover Instagram profile grid content links (posts/reels/tv) for a handle or profile URL, for later selection with instagram_media_download. Returns profile stats and collected URLs.",
|
|
27411
27640
|
inputSchema: InstagramProfileContentInputSchema,
|
|
27412
|
-
outputSchema: InstagramProfileContentOutputSchema,
|
|
27641
|
+
outputSchema: recordOutputSchema("instagram_profile_content", InstagramProfileContentOutputSchema),
|
|
27413
27642
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
27414
27643
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
27415
27644
|
server.registerTool("instagram_media_download", {
|
|
27416
27645
|
title: "Instagram Post/Reel Media Download",
|
|
27417
|
-
description:
|
|
27646
|
+
description: "Extract and download media from one Instagram post, reel, or tv URL \u2014 image, caption, video/audio tracks, optional muxed MP4, or transcript. Selects the best video/audio track pair and muxes when ffmpeg is available.",
|
|
27418
27647
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
27419
|
-
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
27648
|
+
outputSchema: recordOutputSchema("instagram_media_download", InstagramMediaDownloadOutputSchema),
|
|
27420
27649
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
27421
27650
|
}, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
|
|
27422
27651
|
server.registerTool("maps_place_intel", {
|
|
27423
27652
|
title: "Google Maps Business Profile Details",
|
|
27424
|
-
description:
|
|
27653
|
+
description: "Extract Google Maps business intelligence for one known/named business: rating, reviews, category, address, phone, hours, entity IDs. Not for category searches or multi-business prospect lists \u2014 use maps_search for those. Split business name from location.",
|
|
27425
27654
|
inputSchema: MapsPlaceIntelInputSchema,
|
|
27426
|
-
outputSchema: MapsPlaceIntelOutputSchema,
|
|
27655
|
+
outputSchema: recordOutputSchema("maps_place_intel", MapsPlaceIntelOutputSchema),
|
|
27427
27656
|
annotations: liveWebToolAnnotations("Google Maps Business Profile Details")
|
|
27428
27657
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
27429
27658
|
server.registerTool("maps_search", {
|
|
27430
27659
|
title: "Google Maps Business Search",
|
|
27431
|
-
description:
|
|
27660
|
+
description: "Search Google Maps for multiple businesses by category, niche, or local market \u2014 leads, prospects, competitors, or beyond the 3-pack. Leave proxyMode unset. Returns up to 50 candidates (default 10) with names, place URLs, CIDs, ratings.",
|
|
27432
27661
|
inputSchema: MapsSearchInputSchema,
|
|
27433
|
-
outputSchema: MapsSearchOutputSchema,
|
|
27662
|
+
outputSchema: recordOutputSchema("maps_search", MapsSearchOutputSchema),
|
|
27434
27663
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
27435
27664
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
27436
27665
|
server.registerTool("directory_workflow", {
|
|
27437
27666
|
title: "Directory Workflow: Markets + Maps",
|
|
27438
|
-
description:
|
|
27667
|
+
description: `Build directory/prospecting datasets: selects US city markets from Census population data, optionally joins configured ZIP groups, then runs Google Maps business searches per city in parallel. Use for "all cities over 100k population in a state" or market+Maps workflows. ${fileBehavior("Saves a CSV of results per city.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
27439
27668
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
27440
|
-
outputSchema: DirectoryWorkflowOutputSchema,
|
|
27669
|
+
outputSchema: recordOutputSchema("directory_workflow", DirectoryWorkflowOutputSchema),
|
|
27441
27670
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
27442
|
-
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input));
|
|
27671
|
+
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input, ctx));
|
|
27443
27672
|
server.registerTool("workflow_list", {
|
|
27444
27673
|
title: "Workflow Catalog",
|
|
27445
27674
|
description: "List MCP Scraper higher-level workflows and recipes \u2014 market analysis, ICP research, CRO audits, competitive positioning, content gap briefs, AI search visibility, and more. Returns runnable workflow ids plus tool-chain guidance.",
|
|
27446
27675
|
inputSchema: WorkflowListInputSchema,
|
|
27447
|
-
outputSchema: WorkflowListOutputSchema,
|
|
27676
|
+
outputSchema: recordOutputSchema("workflow_list", WorkflowListOutputSchema),
|
|
27448
27677
|
annotations: localPlanningToolAnnotations("Workflow Catalog")
|
|
27449
27678
|
}, async (input) => formatWorkflowList(await executor.workflowList(input), input));
|
|
27450
27679
|
server.registerTool("workflow_suggest", {
|
|
27451
27680
|
title: "Workflow Intent Router",
|
|
27452
27681
|
description: "Route a high-level business/research goal (market analysis, ICP research, CRO audit, competitor comparison, content gap brief, AI search visibility, etc) to the right MCP Scraper workflow/tool chain. Free; tells you what to run next.",
|
|
27453
27682
|
inputSchema: WorkflowSuggestInputSchema,
|
|
27454
|
-
outputSchema: WorkflowSuggestOutputSchema,
|
|
27683
|
+
outputSchema: recordOutputSchema("workflow_suggest", WorkflowSuggestOutputSchema),
|
|
27455
27684
|
annotations: localPlanningToolAnnotations("Workflow Intent Router")
|
|
27456
27685
|
}, async (input) => formatWorkflowSuggest(input));
|
|
27457
27686
|
server.registerTool("workflow_run", {
|
|
27458
27687
|
title: "Run Workflow",
|
|
27459
|
-
description:
|
|
27688
|
+
description: "Start a higher-level MCP Scraper workflow (directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language). Use after workflow_suggest or workflow_list. Stepwise workflows return runId + nextStep \u2014 call workflow_step with the runId until done is true.",
|
|
27460
27689
|
inputSchema: WorkflowRunInputSchema,
|
|
27461
|
-
outputSchema: WorkflowRunOutputSchema,
|
|
27690
|
+
outputSchema: recordOutputSchema("workflow_run", WorkflowRunOutputSchema),
|
|
27462
27691
|
annotations: liveWebToolAnnotations("Run Workflow")
|
|
27463
27692
|
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
27464
27693
|
server.registerTool("workflow_step", {
|
|
27465
27694
|
title: "Advance Workflow Step",
|
|
27466
|
-
description:
|
|
27695
|
+
description: "Run the next leg of a stepwise workflow started with workflow_run. Pass the runId; each call executes one step and returns its output plus nextStep. Keep calling with the same runId until done is true.",
|
|
27467
27696
|
inputSchema: WorkflowStepInputSchema,
|
|
27468
|
-
outputSchema: WorkflowStepOutputSchema,
|
|
27697
|
+
outputSchema: recordOutputSchema("workflow_step", WorkflowStepOutputSchema),
|
|
27469
27698
|
annotations: liveWebToolAnnotations("Advance Workflow Step")
|
|
27470
27699
|
}, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
|
|
27471
27700
|
server.registerTool("workflow_status", {
|
|
27472
27701
|
title: "Workflow Status",
|
|
27473
27702
|
description: "Fetch a hosted workflow run by id and list its current status and artifacts, to re-open a run or recover artifact ids. Use only a runId returned by workflow_run/workflow_step/workflow_status.",
|
|
27474
27703
|
inputSchema: WorkflowStatusInputSchema,
|
|
27475
|
-
outputSchema: WorkflowStatusOutputSchema,
|
|
27704
|
+
outputSchema: recordOutputSchema("workflow_status", WorkflowStatusOutputSchema),
|
|
27476
27705
|
annotations: liveWebToolAnnotations("Workflow Status")
|
|
27477
27706
|
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
27478
27707
|
server.registerTool("workflow_artifact_read", {
|
|
27479
27708
|
title: "Read Workflow Artifact",
|
|
27480
27709
|
description: "Read a workflow artifact back into context by run id and artifact id, so final deliverables are grounded in generated evidence rather than memory. Use workflow_status first when artifact ids are unknown. Use maxBytes to limit large artifacts.",
|
|
27481
27710
|
inputSchema: WorkflowArtifactReadInputSchema,
|
|
27482
|
-
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
27711
|
+
outputSchema: recordOutputSchema("workflow_artifact_read", WorkflowArtifactReadOutputSchema),
|
|
27483
27712
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
27484
27713
|
}, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
|
|
27714
|
+
server.registerTool("report_artifact_read", {
|
|
27715
|
+
title: "Read Report Artifact",
|
|
27716
|
+
description: "Read back a stored report artifact by artifactId (returned by any tool whose result was too large to inline). Windowed: pass offset/maxBytes and keep reading until nextOffset is null.",
|
|
27717
|
+
inputSchema: ReportArtifactReadInputSchema,
|
|
27718
|
+
outputSchema: recordOutputSchema("report_artifact_read", ReportArtifactReadOutputSchema),
|
|
27719
|
+
annotations: liveWebToolAnnotations("Read Report Artifact")
|
|
27720
|
+
}, async (input) => {
|
|
27721
|
+
const owner = artifactOwnerId(input.artifactId);
|
|
27722
|
+
if (!owner || owner !== ownerId) {
|
|
27723
|
+
return { content: [{ type: "text", text: "Artifact not found or not owned by this caller." }], isError: true };
|
|
27724
|
+
}
|
|
27725
|
+
const window2 = await readArtifactWindow(input.artifactId, input.offset ?? 0, input.maxBytes ?? 2e4);
|
|
27726
|
+
if (!window2) {
|
|
27727
|
+
return { content: [{ type: "text", text: "Artifact not found or expired." }], isError: true };
|
|
27728
|
+
}
|
|
27729
|
+
return {
|
|
27730
|
+
content: [{ type: "text", text: window2.text }],
|
|
27731
|
+
structuredContent: { artifactId: input.artifactId, text: window2.text, totalBytes: window2.totalBytes, nextOffset: window2.nextOffset }
|
|
27732
|
+
};
|
|
27733
|
+
});
|
|
27485
27734
|
server.registerTool("rank_tracker_workflow", {
|
|
27486
27735
|
title: "Rank Tracker Blueprint Builder",
|
|
27487
27736
|
description: "Generate a build-ready database schema, cron plan, and implementation prompt for a rank tracker powered by MCP Scraper (Maps, organic, AI Overview, or PAA tracking). Local planning only \u2014 does not call the web or spend credits.",
|
|
27488
27737
|
inputSchema: RankTrackerBlueprintInputSchema,
|
|
27489
|
-
outputSchema: RankTrackerBlueprintOutputSchema,
|
|
27738
|
+
outputSchema: recordOutputSchema("rank_tracker_workflow", RankTrackerBlueprintOutputSchema),
|
|
27490
27739
|
annotations: localPlanningToolAnnotations("Rank Tracker Blueprint Builder")
|
|
27491
27740
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
27492
27741
|
server.registerTool("credits_info", {
|
|
27493
27742
|
title: "MCP Scraper Credits & Costs",
|
|
27494
27743
|
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades \u2014 balance, tool costs, concurrency limits, billing URL. Does not expose payment methods or card information.",
|
|
27495
27744
|
inputSchema: CreditsInfoInputSchema,
|
|
27496
|
-
outputSchema: CreditsInfoOutputSchema,
|
|
27745
|
+
outputSchema: recordOutputSchema("credits_info", CreditsInfoOutputSchema),
|
|
27497
27746
|
annotations: {
|
|
27498
27747
|
title: "MCP Scraper Credits & Costs",
|
|
27499
27748
|
readOnlyHint: true,
|
|
@@ -27503,16 +27752,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
27503
27752
|
}
|
|
27504
27753
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
27505
27754
|
}
|
|
27506
|
-
var import_mcp, import_node_fs8, import_node_path11;
|
|
27755
|
+
var import_mcp, import_node_fs8, import_node_path11, import_node_crypto10;
|
|
27507
27756
|
var init_paa_mcp_server = __esm({
|
|
27508
27757
|
"src/mcp/paa-mcp-server.ts"() {
|
|
27509
27758
|
"use strict";
|
|
27510
27759
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
27511
27760
|
import_node_fs8 = require("fs");
|
|
27512
27761
|
import_node_path11 = require("path");
|
|
27762
|
+
import_node_crypto10 = require("crypto");
|
|
27513
27763
|
init_version();
|
|
27514
27764
|
init_mcp_response_formatter();
|
|
27515
27765
|
init_server_instructions();
|
|
27766
|
+
init_output_schema_registry();
|
|
27767
|
+
init_report_artifact_offload();
|
|
27516
27768
|
init_mcp_tool_schemas();
|
|
27517
27769
|
init_rank_tracker_blueprint();
|
|
27518
27770
|
init_mcp_response_formatter();
|
|
@@ -28724,7 +28976,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28724
28976
|
title: "Save a Site Login to a Profile",
|
|
28725
28977
|
description: "Save a logged-in browser session so this server can reuse a site the user is signed into (ChatGPT, Claude, Reddit, any account-gated site). Returns a watch_url; the user signs in fresh (existing browser cookies are NOT imported) and cookies save to a named profile. ONE profile holds MANY logins \u2014 call again with the same profile and a different domain to stack another account. NOT for one-off scraping (use extract_url) or driving the browser (use browser_open). After sign-in, poll browser_profile_list until AUTHENTICATED, then browser_open with the profile.",
|
|
28726
28978
|
inputSchema: BrowserProfileConnectInputSchema,
|
|
28727
|
-
outputSchema: BrowserProfileConnectOutputSchema,
|
|
28979
|
+
outputSchema: recordOutputSchema("browser_profile_connect", BrowserProfileConnectOutputSchema),
|
|
28728
28980
|
annotations: annotations("Save a Site Login to a Profile")
|
|
28729
28981
|
},
|
|
28730
28982
|
async (input) => {
|
|
@@ -28782,7 +29034,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28782
29034
|
title: "List Saved Logins in a Profile",
|
|
28783
29035
|
description: "List every site login saved in a profile with its auth status (NEEDS_AUTH/AUTHENTICATED), email, and note. Use to check what's connected, or to poll a just-saved login until AUTHENTICATED. Read-only, no cost. Pass profile (or email to derive it); narrow with domain or connection_id.",
|
|
28784
29036
|
inputSchema: BrowserProfileListInputSchema,
|
|
28785
|
-
outputSchema: BrowserProfileListOutputSchema,
|
|
29037
|
+
outputSchema: recordOutputSchema("browser_profile_list", BrowserProfileListOutputSchema),
|
|
28786
29038
|
annotations: annotations("List Saved Logins in a Profile", true)
|
|
28787
29039
|
},
|
|
28788
29040
|
async (input) => {
|
|
@@ -28816,7 +29068,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28816
29068
|
title: "Open Browser Session",
|
|
28817
29069
|
description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load a session already logged into that profile's sites (set one up first with browser_profile_connect). Returns a session_id used by all other browser_* tools.",
|
|
28818
29070
|
inputSchema: BrowserOpenInputSchema,
|
|
28819
|
-
outputSchema: BrowserOpenOutputSchema,
|
|
29071
|
+
outputSchema: recordOutputSchema("browser_open", BrowserOpenOutputSchema),
|
|
28820
29072
|
annotations: annotations("Open Browser Session")
|
|
28821
29073
|
},
|
|
28822
29074
|
async (input) => {
|
|
@@ -28850,7 +29102,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28850
29102
|
title: "See Page (Screenshot + Elements)",
|
|
28851
29103
|
description: "Capture what the browser currently shows: a screenshot plus a text snapshot of interactive elements with x,y coordinates, page url/title, and visible text. Primary way to perceive the page; click elements by their listed x,y. If a Cloudflare/CAPTCHA challenge is visible, wait and screenshot again rather than clicking it.",
|
|
28852
29104
|
inputSchema: BrowserSessionInputSchema,
|
|
28853
|
-
outputSchema: BrowserScreenshotOutputSchema,
|
|
29105
|
+
outputSchema: recordOutputSchema("browser_screenshot", BrowserScreenshotOutputSchema),
|
|
28854
29106
|
annotations: annotations("See Page", true)
|
|
28855
29107
|
},
|
|
28856
29108
|
async (input) => {
|
|
@@ -28882,7 +29134,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28882
29134
|
title: "Read Page Text + Elements",
|
|
28883
29135
|
description: "Return the page url, title, visible text, and interactive elements (with x,y) without an image. Cheaper than browser_screenshot when you only need to read content or find a click target.",
|
|
28884
29136
|
inputSchema: BrowserSessionInputSchema,
|
|
28885
|
-
outputSchema: BrowserReadOutputSchema,
|
|
29137
|
+
outputSchema: recordOutputSchema("browser_read", BrowserReadOutputSchema),
|
|
28886
29138
|
annotations: annotations("Read Page", true)
|
|
28887
29139
|
},
|
|
28888
29140
|
async (input) => {
|
|
@@ -28906,7 +29158,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28906
29158
|
title: "Locate DOM Targets",
|
|
28907
29159
|
description: "Locate exact visible DOM elements or text ranges and return left/top/width/height bounds in screenshot pixels. Use before drawing annotations that must circle, box, underline, or point to a real element. Prefer CSS selectors; use text when selector is unknown.",
|
|
28908
29160
|
inputSchema: BrowserLocateInputSchema,
|
|
28909
|
-
outputSchema: BrowserLocateOutputSchema,
|
|
29161
|
+
outputSchema: recordOutputSchema("browser_locate", BrowserLocateOutputSchema),
|
|
28910
29162
|
annotations: annotations("Locate DOM Targets", true)
|
|
28911
29163
|
},
|
|
28912
29164
|
async (input) => {
|
|
@@ -28931,7 +29183,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28931
29183
|
title: "Navigate To URL",
|
|
28932
29184
|
description: "Navigate an existing browser session to a URL. Use browser_open first if no session exists; follow with browser_screenshot to see the loaded page.",
|
|
28933
29185
|
inputSchema: BrowserGotoInputSchema,
|
|
28934
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29186
|
+
outputSchema: recordOutputSchema("browser_goto", BrowserActionOutputSchema),
|
|
28935
29187
|
annotations: annotations("Navigate To URL")
|
|
28936
29188
|
},
|
|
28937
29189
|
async (input) => {
|
|
@@ -28945,7 +29197,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28945
29197
|
title: "Click",
|
|
28946
29198
|
description: "Click a visible page target using screenshot pixel coordinates. Use x/y only from the latest browser_screenshot, browser_read, or browser_locate result; do not guess coordinates.",
|
|
28947
29199
|
inputSchema: BrowserClickInputSchema,
|
|
28948
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29200
|
+
outputSchema: recordOutputSchema("browser_click", BrowserActionOutputSchema),
|
|
28949
29201
|
annotations: annotations("Click")
|
|
28950
29202
|
},
|
|
28951
29203
|
async (input) => {
|
|
@@ -28964,7 +29216,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28964
29216
|
title: "Type Text",
|
|
28965
29217
|
description: 'Type text into the currently focused browser field. Click or Tab to the field first if focus is uncertain. Use browser_press with ["Return"] to submit.',
|
|
28966
29218
|
inputSchema: BrowserTypeInputSchema,
|
|
28967
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29219
|
+
outputSchema: recordOutputSchema("browser_type", BrowserActionOutputSchema),
|
|
28968
29220
|
annotations: annotations("Type Text")
|
|
28969
29221
|
},
|
|
28970
29222
|
async (input) => {
|
|
@@ -28978,7 +29230,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28978
29230
|
title: "Scroll",
|
|
28979
29231
|
description: "Scroll the page to reveal more content. Positive delta_y scrolls down; negative scrolls up.",
|
|
28980
29232
|
inputSchema: BrowserScrollInputSchema,
|
|
28981
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29233
|
+
outputSchema: recordOutputSchema("browser_scroll", BrowserActionOutputSchema),
|
|
28982
29234
|
annotations: annotations("Scroll")
|
|
28983
29235
|
},
|
|
28984
29236
|
async (input) => {
|
|
@@ -28997,7 +29249,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28997
29249
|
title: "Press Keys",
|
|
28998
29250
|
description: "Press keyboard keys or combinations in the active browser session \u2014 submit, Escape, Tab navigation, select-all, or shortcuts. Use browser_type for text entry.",
|
|
28999
29251
|
inputSchema: BrowserPressInputSchema,
|
|
29000
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29252
|
+
outputSchema: recordOutputSchema("browser_press", BrowserActionOutputSchema),
|
|
29001
29253
|
annotations: annotations("Press Keys")
|
|
29002
29254
|
},
|
|
29003
29255
|
async (input) => {
|
|
@@ -29011,7 +29263,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29011
29263
|
title: "Start Recording",
|
|
29012
29264
|
description: "Start recording an MP4 replay of the session. Returns replay_id and a download_url. Stop with browser_replay_stop.",
|
|
29013
29265
|
inputSchema: BrowserSessionInputSchema,
|
|
29014
|
-
outputSchema: BrowserReplayStartOutputSchema,
|
|
29266
|
+
outputSchema: recordOutputSchema("browser_replay_start", BrowserReplayStartOutputSchema),
|
|
29015
29267
|
annotations: annotations("Start Recording")
|
|
29016
29268
|
},
|
|
29017
29269
|
async (input) => {
|
|
@@ -29034,7 +29286,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29034
29286
|
title: "Stop Recording",
|
|
29035
29287
|
description: "Stop a replay recording and expose its final view_url/download_url. Use browser_replay_download to save the MP4.",
|
|
29036
29288
|
inputSchema: BrowserReplayStopInputSchema,
|
|
29037
|
-
outputSchema: BrowserReplayStopOutputSchema,
|
|
29289
|
+
outputSchema: recordOutputSchema("browser_replay_stop", BrowserReplayStopOutputSchema),
|
|
29038
29290
|
annotations: annotations("Stop Recording")
|
|
29039
29291
|
},
|
|
29040
29292
|
async (input) => {
|
|
@@ -29057,7 +29309,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29057
29309
|
title: "List Replay Videos",
|
|
29058
29310
|
description: "List replay recordings for a browser session, including view_url and download_url when available.",
|
|
29059
29311
|
inputSchema: BrowserSessionInputSchema,
|
|
29060
|
-
outputSchema: BrowserListReplaysOutputSchema,
|
|
29312
|
+
outputSchema: recordOutputSchema("browser_list_replays", BrowserListReplaysOutputSchema),
|
|
29061
29313
|
annotations: annotations("List Replay Videos", true)
|
|
29062
29314
|
},
|
|
29063
29315
|
async (input) => {
|
|
@@ -29077,9 +29329,9 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29077
29329
|
"browser_replay_download",
|
|
29078
29330
|
{
|
|
29079
29331
|
title: "Download Replay MP4",
|
|
29080
|
-
description: "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
29332
|
+
description: opts.savesReportsLocally === false ? "Download a replay recording. Returns the download_url; fetch it directly (nothing is saved on this hosted endpoint). Use after browser_replay_stop or browser_list_replays." : "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
29081
29333
|
inputSchema: BrowserReplayDownloadInputSchema,
|
|
29082
|
-
outputSchema: BrowserReplayDownloadOutputSchema,
|
|
29334
|
+
outputSchema: recordOutputSchema("browser_replay_download", BrowserReplayDownloadOutputSchema),
|
|
29083
29335
|
annotations: annotations("Download Replay MP4")
|
|
29084
29336
|
},
|
|
29085
29337
|
async (input) => {
|
|
@@ -29103,7 +29355,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29103
29355
|
title: "Mark Replay Annotation",
|
|
29104
29356
|
description: "While a replay is actively recording, locate one exact DOM target and return a ready-to-use annotation with DOM bounds and replay-relative timing, instead of guessing start_seconds or rectangles. Pass the returned annotations to browser_replay_annotate after stopping the replay.",
|
|
29105
29357
|
inputSchema: BrowserReplayMarkInputSchema,
|
|
29106
|
-
outputSchema: BrowserReplayMarkOutputSchema,
|
|
29358
|
+
outputSchema: recordOutputSchema("browser_replay_mark", BrowserReplayMarkOutputSchema),
|
|
29107
29359
|
annotations: annotations("Mark Replay Annotation", true)
|
|
29108
29360
|
},
|
|
29109
29361
|
async (input) => {
|
|
@@ -29153,7 +29405,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29153
29405
|
title: "Annotate Replay MP4",
|
|
29154
29406
|
description: "Download a browser replay MP4, render visual annotations (circles/boxes/arrows/labels) over it, and save a new annotated MP4. Prefer annotations from browser_replay_mark for accurate timing; otherwise use exact bounds from browser_locate. Pass source_width/source_height if the replay video size differs from the screenshot coordinate space.",
|
|
29155
29407
|
inputSchema: BrowserReplayAnnotateInputSchema,
|
|
29156
|
-
outputSchema: BrowserReplayAnnotateOutputSchema,
|
|
29408
|
+
outputSchema: recordOutputSchema("browser_replay_annotate", BrowserReplayAnnotateOutputSchema),
|
|
29157
29409
|
annotations: annotations("Annotate Replay MP4")
|
|
29158
29410
|
},
|
|
29159
29411
|
async (input) => {
|
|
@@ -29195,7 +29447,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29195
29447
|
title: "Close Browser Session",
|
|
29196
29448
|
description: "Close and release a browser session when the task is done, to end active browser billing. Use browser_list_sessions first to recover a session_id.",
|
|
29197
29449
|
inputSchema: BrowserSessionInputSchema,
|
|
29198
|
-
outputSchema: BrowserCloseOutputSchema,
|
|
29450
|
+
outputSchema: recordOutputSchema("browser_close", BrowserCloseOutputSchema),
|
|
29199
29451
|
annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
29200
29452
|
},
|
|
29201
29453
|
async (input) => {
|
|
@@ -29216,7 +29468,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29216
29468
|
title: "List Browser Sessions",
|
|
29217
29469
|
description: "List browser sessions and their status, with a watch_url for each. Use to recover a session_id or decide which session to close.",
|
|
29218
29470
|
inputSchema: BrowserListInputSchema,
|
|
29219
|
-
outputSchema: BrowserListSessionsOutputSchema,
|
|
29471
|
+
outputSchema: recordOutputSchema("browser_list_sessions", BrowserListSessionsOutputSchema),
|
|
29220
29472
|
annotations: annotations("List Browser Sessions", true)
|
|
29221
29473
|
},
|
|
29222
29474
|
async (input) => {
|
|
@@ -29238,7 +29490,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29238
29490
|
title: "Capture AI Search Fan-Out",
|
|
29239
29491
|
description: "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account \u2014 only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview \u2014 use harvest_paa for that.",
|
|
29240
29492
|
inputSchema: BrowserCaptureFanoutInputSchema,
|
|
29241
|
-
outputSchema: BrowserCaptureFanoutOutputSchema,
|
|
29493
|
+
outputSchema: recordOutputSchema("query_fanout_workflow", BrowserCaptureFanoutOutputSchema),
|
|
29242
29494
|
annotations: annotations("Capture AI Search Fan-Out")
|
|
29243
29495
|
},
|
|
29244
29496
|
async (input) => {
|
|
@@ -29303,6 +29555,7 @@ var init_browser_agent_mcp_server = __esm({
|
|
|
29303
29555
|
init_browser_agent_tool_schemas();
|
|
29304
29556
|
init_replay_annotator();
|
|
29305
29557
|
init_outbound_sanitize();
|
|
29558
|
+
init_output_schema_registry();
|
|
29306
29559
|
}
|
|
29307
29560
|
});
|
|
29308
29561
|
|
|
@@ -29418,9 +29671,9 @@ var init_mcp_routes = __esm({
|
|
|
29418
29671
|
enableJsonResponse: true
|
|
29419
29672
|
});
|
|
29420
29673
|
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
29421
|
-
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false });
|
|
29674
|
+
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
|
|
29422
29675
|
registerSerpIntelligenceCaptureTools(server, executor);
|
|
29423
|
-
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl });
|
|
29676
|
+
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
29424
29677
|
await server.connect(transport);
|
|
29425
29678
|
return transport.handleRequest(c.req.raw);
|
|
29426
29679
|
} catch {
|
|
@@ -29541,7 +29794,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
29541
29794
|
}
|
|
29542
29795
|
async function createSessionRow(input) {
|
|
29543
29796
|
const db = getDb();
|
|
29544
|
-
const id = `bas_${(0,
|
|
29797
|
+
const id = `bas_${(0, import_node_crypto11.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
29545
29798
|
await db.execute({
|
|
29546
29799
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
29547
29800
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -29610,7 +29863,7 @@ async function recordAction(input) {
|
|
|
29610
29863
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
29611
29864
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
29612
29865
|
args: [
|
|
29613
|
-
`baa_${(0,
|
|
29866
|
+
`baa_${(0, import_node_crypto11.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
29614
29867
|
input.sessionId,
|
|
29615
29868
|
input.type,
|
|
29616
29869
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -29650,11 +29903,11 @@ async function listReplayRows(sessionId) {
|
|
|
29650
29903
|
});
|
|
29651
29904
|
return res.rows;
|
|
29652
29905
|
}
|
|
29653
|
-
var
|
|
29906
|
+
var import_node_crypto11, _ready2;
|
|
29654
29907
|
var init_browser_agent_db = __esm({
|
|
29655
29908
|
"src/api/browser-agent-db.ts"() {
|
|
29656
29909
|
"use strict";
|
|
29657
|
-
|
|
29910
|
+
import_node_crypto11 = require("crypto");
|
|
29658
29911
|
init_db();
|
|
29659
29912
|
_ready2 = false;
|
|
29660
29913
|
}
|
|
@@ -31780,7 +32033,7 @@ async function getKeys() {
|
|
|
31780
32033
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
31781
32034
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
31782
32035
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
31783
|
-
const kid = (0,
|
|
32036
|
+
const kid = (0, import_node_crypto12.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
31784
32037
|
publicJwk.kid = kid;
|
|
31785
32038
|
publicJwk.alg = "RS256";
|
|
31786
32039
|
publicJwk.use = "sig";
|
|
@@ -31959,23 +32212,23 @@ async function validateAuthRequest(p) {
|
|
|
31959
32212
|
}
|
|
31960
32213
|
function pkceMatches(verifier, challenge) {
|
|
31961
32214
|
if (!verifier) return false;
|
|
31962
|
-
const computed = (0,
|
|
32215
|
+
const computed = (0, import_node_crypto12.createHash)("sha256").update(verifier).digest("base64url");
|
|
31963
32216
|
return computed === challenge;
|
|
31964
32217
|
}
|
|
31965
32218
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
31966
32219
|
const { privateKey, kid } = await getKeys();
|
|
31967
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
32220
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto12.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
31968
32221
|
}
|
|
31969
32222
|
function tokenErrorResponse(c, error, description, status) {
|
|
31970
32223
|
return c.json({ error, error_description: description }, status);
|
|
31971
32224
|
}
|
|
31972
|
-
var import_hono17, import_cookie,
|
|
32225
|
+
var import_hono17, import_cookie, import_node_crypto12, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
31973
32226
|
var init_oauth_routes = __esm({
|
|
31974
32227
|
"src/api/oauth-routes.ts"() {
|
|
31975
32228
|
"use strict";
|
|
31976
32229
|
import_hono17 = require("hono");
|
|
31977
32230
|
import_cookie = require("hono/cookie");
|
|
31978
|
-
|
|
32231
|
+
import_node_crypto12 = require("crypto");
|
|
31979
32232
|
import_jose2 = require("jose");
|
|
31980
32233
|
init_session();
|
|
31981
32234
|
init_db();
|
|
@@ -32061,7 +32314,7 @@ var init_oauth_routes = __esm({
|
|
|
32061
32314
|
}
|
|
32062
32315
|
}
|
|
32063
32316
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
32064
|
-
const clientId = `client_${(0,
|
|
32317
|
+
const clientId = `client_${(0, import_node_crypto12.randomBytes)(16).toString("hex")}`;
|
|
32065
32318
|
await registerClient(clientId, redirectUris, clientName);
|
|
32066
32319
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
32067
32320
|
return c.json({
|
|
@@ -32114,7 +32367,7 @@ var init_oauth_routes = __esm({
|
|
|
32114
32367
|
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
32115
32368
|
if (action !== "approve") return c.text("unsupported action", 400);
|
|
32116
32369
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
32117
|
-
const code = `code_${(0,
|
|
32370
|
+
const code = `code_${(0, import_node_crypto12.randomBytes)(32).toString("base64url")}`;
|
|
32118
32371
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
32119
32372
|
await putCode({
|
|
32120
32373
|
code,
|
|
@@ -32153,7 +32406,7 @@ var init_oauth_routes = __esm({
|
|
|
32153
32406
|
const plan = user ? resolvePlan(user) : "free";
|
|
32154
32407
|
const audience = record.resource ?? RESOURCE();
|
|
32155
32408
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
32156
|
-
const refreshToken = `rt_${(0,
|
|
32409
|
+
const refreshToken = `rt_${(0, import_node_crypto12.randomBytes)(40).toString("base64url")}`;
|
|
32157
32410
|
await putRefresh({
|
|
32158
32411
|
refresh_token: refreshToken,
|
|
32159
32412
|
client_id: clientId,
|
|
@@ -32183,7 +32436,7 @@ var init_oauth_routes = __esm({
|
|
|
32183
32436
|
const plan = user ? resolvePlan(user) : "free";
|
|
32184
32437
|
const audience = record.resource ?? RESOURCE();
|
|
32185
32438
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
32186
|
-
const nextRefresh = `rt_${(0,
|
|
32439
|
+
const nextRefresh = `rt_${(0, import_node_crypto12.randomBytes)(40).toString("base64url")}`;
|
|
32187
32440
|
await rotateRefresh(refreshToken, {
|
|
32188
32441
|
refresh_token: nextRefresh,
|
|
32189
32442
|
client_id: record.client_id,
|