mcp-scraper 0.3.45 → 0.3.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +378 -157
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +435 -180
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-E7BZFMWB.js → chunk-SS5YBZVI.js} +472 -212
- package/dist/chunk-SS5YBZVI.js.map +1 -0
- package/dist/chunk-WT6OT2T3.js +7 -0
- package/dist/chunk-WT6OT2T3.js.map +1 -0
- package/dist/{server-MKUN6NQZ.js → server-MGE3GYJW.js} +26 -76
- package/dist/server-MGE3GYJW.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-E7BZFMWB.js.map +0 -1
- package/dist/chunk-NVTFLLEG.js +0 -7
- package/dist/chunk-NVTFLLEG.js.map +0 -1
- package/dist/server-MKUN6NQZ.js.map +0 -1
package/dist/bin/api-server.cjs
CHANGED
|
@@ -10015,6 +10015,15 @@ var init_blob_store = __esm({
|
|
|
10015
10015
|
(0, import_node_fs2.writeFileSync)(path6, data);
|
|
10016
10016
|
return { key, url: `file://${path6}`, bytes: byteLength(data), contentType };
|
|
10017
10017
|
}
|
|
10018
|
+
async get(key) {
|
|
10019
|
+
const path6 = (0, import_node_path2.join)(this.baseDir, "blobs", key);
|
|
10020
|
+
try {
|
|
10021
|
+
const { readFileSync: readFileSync4 } = await import("fs");
|
|
10022
|
+
return readFileSync4(path6);
|
|
10023
|
+
} catch {
|
|
10024
|
+
return null;
|
|
10025
|
+
}
|
|
10026
|
+
}
|
|
10018
10027
|
};
|
|
10019
10028
|
cached = null;
|
|
10020
10029
|
VercelBlobStore = class {
|
|
@@ -10034,6 +10043,19 @@ var init_blob_store = __esm({
|
|
|
10034
10043
|
});
|
|
10035
10044
|
return { key, url: result.url, bytes: byteLength(data), contentType };
|
|
10036
10045
|
}
|
|
10046
|
+
async get(key) {
|
|
10047
|
+
try {
|
|
10048
|
+
const { list } = await import("@vercel/blob");
|
|
10049
|
+
const res = await list({ prefix: key, token: this.token, limit: 1 });
|
|
10050
|
+
const match = res.blobs.find((b) => b.pathname === key || b.pathname.startsWith(key));
|
|
10051
|
+
if (!match) return null;
|
|
10052
|
+
const resp = await fetch(match.url);
|
|
10053
|
+
if (!resp.ok) return null;
|
|
10054
|
+
return Buffer.from(await resp.arrayBuffer());
|
|
10055
|
+
} catch {
|
|
10056
|
+
return null;
|
|
10057
|
+
}
|
|
10058
|
+
}
|
|
10037
10059
|
};
|
|
10038
10060
|
}
|
|
10039
10061
|
});
|
|
@@ -17839,6 +17861,57 @@ var init_workflow_catalog = __esm({
|
|
|
17839
17861
|
}
|
|
17840
17862
|
});
|
|
17841
17863
|
|
|
17864
|
+
// src/mcp/report-artifact-offload.ts
|
|
17865
|
+
async function offloadReport(toolName, ownerId, report) {
|
|
17866
|
+
const timestamp2 = Date.now();
|
|
17867
|
+
const random = (0, import_node_crypto7.randomBytes)(6).toString("hex");
|
|
17868
|
+
const key = `${REPORT_BLOB_PREFIX}${ownerId}/${toolName}/${timestamp2}-${random}.md`;
|
|
17869
|
+
const stored = await getBlobStore().put(key, report, "text/markdown");
|
|
17870
|
+
return {
|
|
17871
|
+
artifactId: stored.key,
|
|
17872
|
+
bytes: stored.bytes,
|
|
17873
|
+
expiresAt: new Date(timestamp2 + REPORT_BLOB_TTL_MS).toISOString(),
|
|
17874
|
+
preview: report.slice(0, PREVIEW_CHARS)
|
|
17875
|
+
};
|
|
17876
|
+
}
|
|
17877
|
+
function artifactOwnerId(artifactId) {
|
|
17878
|
+
if (!artifactId.startsWith(REPORT_BLOB_PREFIX)) return null;
|
|
17879
|
+
const rest = artifactId.slice(REPORT_BLOB_PREFIX.length);
|
|
17880
|
+
const segment = rest.split("/")[0];
|
|
17881
|
+
return segment || null;
|
|
17882
|
+
}
|
|
17883
|
+
async function readArtifactWindow(artifactId, offset, maxBytes) {
|
|
17884
|
+
const buf = await getBlobStore().get(artifactId);
|
|
17885
|
+
if (!buf) return null;
|
|
17886
|
+
const totalBytes = buf.length;
|
|
17887
|
+
const start = Math.max(0, offset);
|
|
17888
|
+
const end = Math.min(totalBytes, start + maxBytes);
|
|
17889
|
+
const slice = buf.subarray(start, end);
|
|
17890
|
+
const nextOffset = end < totalBytes ? end : null;
|
|
17891
|
+
return { text: slice.toString("utf8"), totalBytes, nextOffset };
|
|
17892
|
+
}
|
|
17893
|
+
function summaryEnvelope(executiveSummary, offloaded) {
|
|
17894
|
+
return [
|
|
17895
|
+
executiveSummary.trim(),
|
|
17896
|
+
"",
|
|
17897
|
+
"--- Full report stored as artifact ---",
|
|
17898
|
+
`artifactId: ${offloaded.artifactId} \xB7 ${offloaded.bytes} bytes \xB7 expires ${offloaded.expiresAt}`,
|
|
17899
|
+
"Read it with report_artifact_read (supports offset/maxBytes windowing)."
|
|
17900
|
+
].join("\n");
|
|
17901
|
+
}
|
|
17902
|
+
var import_node_crypto7, REPORT_BLOB_TTL_MS, REPORT_BLOB_PREFIX, PREVIEW_CHARS, ARTIFACT_OFFLOAD_ENABLED;
|
|
17903
|
+
var init_report_artifact_offload = __esm({
|
|
17904
|
+
"src/mcp/report-artifact-offload.ts"() {
|
|
17905
|
+
"use strict";
|
|
17906
|
+
import_node_crypto7 = require("crypto");
|
|
17907
|
+
init_blob_store();
|
|
17908
|
+
REPORT_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
17909
|
+
REPORT_BLOB_PREFIX = "mcp-reports/";
|
|
17910
|
+
PREVIEW_CHARS = 2e3;
|
|
17911
|
+
ARTIFACT_OFFLOAD_ENABLED = process.env.MCP_SCRAPER_ARTIFACT_OFFLOAD !== "false";
|
|
17912
|
+
}
|
|
17913
|
+
});
|
|
17914
|
+
|
|
17842
17915
|
// src/mcp/mcp-response-formatter.ts
|
|
17843
17916
|
function configureReportSaving(enabled) {
|
|
17844
17917
|
reportSavingEnabled = enabled;
|
|
@@ -18004,6 +18077,20 @@ function oneBlock(content, diskContent) {
|
|
|
18004
18077
|
\u{1F4C4} Saved: \`${filePath}\`` : content;
|
|
18005
18078
|
return { content: [{ type: "text", text }] };
|
|
18006
18079
|
}
|
|
18080
|
+
async function maybeOffload(toolName, ctx, fullText, summaryText, structuredContent) {
|
|
18081
|
+
if (!ctx?.hosted || !ARTIFACT_OFFLOAD_ENABLED) return null;
|
|
18082
|
+
const bytes = Buffer.byteLength(fullText);
|
|
18083
|
+
if (bytes <= INLINE_BUDGET_BYTES) return null;
|
|
18084
|
+
const offloaded = await offloadReport(toolName, ctx.ownerId, fullText);
|
|
18085
|
+
return {
|
|
18086
|
+
content: [{ type: "text", text: summaryEnvelope(summaryText, offloaded) }],
|
|
18087
|
+
structuredContent: { ...structuredContent, artifact: offloaded }
|
|
18088
|
+
};
|
|
18089
|
+
}
|
|
18090
|
+
function capArray(items, cap) {
|
|
18091
|
+
if (items.length <= cap) return { items };
|
|
18092
|
+
return { items: items.slice(0, cap), truncatedCount: items.length - cap };
|
|
18093
|
+
}
|
|
18007
18094
|
function workflowRecipeTable(recipes) {
|
|
18008
18095
|
if (!recipes.length) return "";
|
|
18009
18096
|
return [
|
|
@@ -18355,7 +18442,7 @@ ${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${memSection}$
|
|
|
18355
18442
|
}
|
|
18356
18443
|
return { ...textResult, structuredContent };
|
|
18357
18444
|
}
|
|
18358
|
-
function formatMapSiteUrls(raw, input) {
|
|
18445
|
+
async function formatMapSiteUrls(raw, input, ctx) {
|
|
18359
18446
|
const parsed = parseData(raw);
|
|
18360
18447
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
18361
18448
|
const d = parsed.data;
|
|
@@ -18396,20 +18483,28 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
|
|
|
18396
18483
|
- Extract content from all pages: use \`extract_site\`
|
|
18397
18484
|
- Scrape a single page: use \`extract_url\``
|
|
18398
18485
|
].filter(Boolean).join("\n");
|
|
18399
|
-
|
|
18400
|
-
|
|
18401
|
-
|
|
18402
|
-
|
|
18403
|
-
|
|
18404
|
-
|
|
18405
|
-
|
|
18406
|
-
|
|
18407
|
-
|
|
18408
|
-
|
|
18409
|
-
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
18410
|
-
durationMs: d.durationMs ?? 0
|
|
18411
|
-
}
|
|
18486
|
+
const structuredContent = {
|
|
18487
|
+
startUrl: d.startUrl ?? input.url,
|
|
18488
|
+
totalFound: d.totalFound ?? urls.length,
|
|
18489
|
+
truncated: d.truncated === true,
|
|
18490
|
+
okCount: ok.length,
|
|
18491
|
+
redirectCount: redirects.length,
|
|
18492
|
+
brokenCount: broken.length,
|
|
18493
|
+
inventoryFile: inventoryFile ?? null,
|
|
18494
|
+
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
18495
|
+
durationMs: d.durationMs ?? 0
|
|
18412
18496
|
};
|
|
18497
|
+
const summary = `# URL Map: ${input.url}
|
|
18498
|
+
**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s
|
|
18499
|
+
|
|
18500
|
+
## Summary
|
|
18501
|
+
- 2xx: ${ok.length}
|
|
18502
|
+
- 3xx: ${redirects.length}
|
|
18503
|
+
- 4xx+: ${broken.length}`;
|
|
18504
|
+
const capped = capArray(structuredContent.urls, STRUCTURED_ARRAY_CAP);
|
|
18505
|
+
const offloaded = await maybeOffload("map_site_urls", ctx, full, summary, { ...structuredContent, urls: capped.items, truncatedCount: capped.truncatedCount });
|
|
18506
|
+
if (offloaded) return offloaded;
|
|
18507
|
+
return { ...oneBlock(full), structuredContent };
|
|
18413
18508
|
}
|
|
18414
18509
|
function buildSeoExport(siteUrl, pages, branding) {
|
|
18415
18510
|
const { edges, metrics } = buildLinkGraph(pages, siteUrl);
|
|
@@ -18429,7 +18524,7 @@ function buildSeoExport(siteUrl, pages, branding) {
|
|
|
18429
18524
|
branding: branding ?? void 0
|
|
18430
18525
|
};
|
|
18431
18526
|
}
|
|
18432
|
-
function formatExtractSite(raw, input) {
|
|
18527
|
+
async function formatExtractSite(raw, input, ctx) {
|
|
18433
18528
|
const parsed = parseData(raw);
|
|
18434
18529
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
18435
18530
|
const d = parsed.data;
|
|
@@ -18464,6 +18559,36 @@ function formatExtractSite(raw, input) {
|
|
|
18464
18559
|
schemaTypes: schemaTypesOf(p)
|
|
18465
18560
|
})));
|
|
18466
18561
|
const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
|
|
18562
|
+
const summaryHeader = [
|
|
18563
|
+
`# Site Extract: ${input.url}`,
|
|
18564
|
+
durationLine,
|
|
18565
|
+
`
|
|
18566
|
+
## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
|
|
18567
|
+
| # | Title | URL | Schema |
|
|
18568
|
+
|---|-------|-----|--------|
|
|
18569
|
+
${preview}`
|
|
18570
|
+
].join("\n");
|
|
18571
|
+
if (!bulk && ctx?.hosted) {
|
|
18572
|
+
const pageDetails2 = pages.map((p, i) => {
|
|
18573
|
+
const body = (p.bodyMarkdown ?? "").trim();
|
|
18574
|
+
return [
|
|
18575
|
+
`
|
|
18576
|
+
## ${i + 1}. ${p.title ?? "Untitled"}`,
|
|
18577
|
+
`- **URL:** ${p.url}`,
|
|
18578
|
+
p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
|
|
18579
|
+
body ? `
|
|
18580
|
+
${body}` : "_(no content extracted)_"
|
|
18581
|
+
].filter(Boolean).join("\n");
|
|
18582
|
+
}).join("\n");
|
|
18583
|
+
const fullForOffload = `${summaryHeader}
|
|
18584
|
+
|
|
18585
|
+
---
|
|
18586
|
+
# Full Page Content
|
|
18587
|
+
${pageDetails2}${tips}`;
|
|
18588
|
+
const capped = capArray(structuredContent.pages, STRUCTURED_ARRAY_CAP);
|
|
18589
|
+
const offloaded = await maybeOffload("extract_site", ctx, fullForOffload, `${summaryHeader}${tips}`, { ...structuredContent, pages: capped.items, truncatedCount: capped.truncatedCount });
|
|
18590
|
+
if (offloaded) return offloaded;
|
|
18591
|
+
}
|
|
18467
18592
|
const location2 = bulk ? `
|
|
18468
18593
|
## \u{1F4C1} Bulk scrape saved
|
|
18469
18594
|
- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
|
|
@@ -18516,7 +18641,7 @@ ${body}` : "_(no content extracted)_"
|
|
|
18516
18641
|
${pageDetails}${tips}`;
|
|
18517
18642
|
return { ...oneBlock(full, diskReport), structuredContent };
|
|
18518
18643
|
}
|
|
18519
|
-
async function formatAuditSite(raw, input) {
|
|
18644
|
+
async function formatAuditSite(raw, input, ctx) {
|
|
18520
18645
|
const parsed = parseData(raw);
|
|
18521
18646
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
18522
18647
|
const d = parsed.data;
|
|
@@ -18547,6 +18672,24 @@ async function formatAuditSite(raw, input) {
|
|
|
18547
18672
|
images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
|
|
18548
18673
|
links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
|
|
18549
18674
|
};
|
|
18675
|
+
const summary = [
|
|
18676
|
+
`# Technical SEO Audit: ${input.url}`,
|
|
18677
|
+
durationLine,
|
|
18678
|
+
`
|
|
18679
|
+
## Top issues
|
|
18680
|
+
${topIssues || "_none found_"}`,
|
|
18681
|
+
`
|
|
18682
|
+
## Links
|
|
18683
|
+
${linkLine}`,
|
|
18684
|
+
`
|
|
18685
|
+
## Images
|
|
18686
|
+
${imgLine}`
|
|
18687
|
+
].join("\n");
|
|
18688
|
+
if (!bulk && ctx?.hosted) {
|
|
18689
|
+
const fullForOffload = [summary, "\n---\n# Full Audit Report", seo.reportMd, renderImageSection(imageAudit), renderLinkReport(seo.linkReport)].join("\n");
|
|
18690
|
+
const offloaded = await maybeOffload("audit_site", ctx, fullForOffload, summary, structuredContent);
|
|
18691
|
+
if (offloaded) return offloaded;
|
|
18692
|
+
}
|
|
18550
18693
|
const location2 = bulk ? `
|
|
18551
18694
|
## \u{1F4C1} Technical audit saved
|
|
18552
18695
|
- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
|
|
@@ -19317,7 +19460,7 @@ ${rows}`,
|
|
|
19317
19460
|
}
|
|
19318
19461
|
};
|
|
19319
19462
|
}
|
|
19320
|
-
function formatDirectoryWorkflow(raw, input) {
|
|
19463
|
+
async function formatDirectoryWorkflow(raw, input, ctx) {
|
|
19321
19464
|
const parsed = parseData(raw);
|
|
19322
19465
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
19323
19466
|
const d = parsed.data;
|
|
@@ -19369,26 +19512,29 @@ ${businessRows}` : null,
|
|
|
19369
19512
|
durationMs != null ? `
|
|
19370
19513
|
*Completed in ${(durationMs / 1e3).toFixed(1)}s*` : null
|
|
19371
19514
|
].filter(Boolean).join("\n");
|
|
19372
|
-
|
|
19373
|
-
|
|
19374
|
-
|
|
19375
|
-
|
|
19376
|
-
|
|
19377
|
-
|
|
19378
|
-
|
|
19379
|
-
|
|
19380
|
-
|
|
19381
|
-
|
|
19382
|
-
|
|
19383
|
-
|
|
19384
|
-
|
|
19385
|
-
|
|
19386
|
-
|
|
19387
|
-
|
|
19388
|
-
cities,
|
|
19389
|
-
durationMs: durationMs ?? 0
|
|
19390
|
-
}
|
|
19515
|
+
const structuredContent = {
|
|
19516
|
+
query: d.query,
|
|
19517
|
+
state: d.state,
|
|
19518
|
+
minPopulation: d.minPopulation,
|
|
19519
|
+
populationYear: d.populationYear,
|
|
19520
|
+
maxResultsPerCity: d.maxResultsPerCity,
|
|
19521
|
+
concurrency: d.concurrency,
|
|
19522
|
+
censusSourceUrl: d.censusSourceUrl,
|
|
19523
|
+
usZipsSourcePath: d.usZipsSourcePath ?? null,
|
|
19524
|
+
warnings,
|
|
19525
|
+
extractedAt: d.extractedAt,
|
|
19526
|
+
selectedCityCount: d.selectedCityCount,
|
|
19527
|
+
totalResultCount,
|
|
19528
|
+
csvPath,
|
|
19529
|
+
cities,
|
|
19530
|
+
durationMs: durationMs ?? 0
|
|
19391
19531
|
};
|
|
19532
|
+
const summary = `# Directory Workflow: ${input.query}
|
|
19533
|
+
**Markets:** ${cities.length} \xB7 **Maps results:** ${totalResultCount} \xB7 **State:** ${d.state ?? input.state ?? "US"}`;
|
|
19534
|
+
const capped = capArray(cities, STRUCTURED_ARRAY_CAP);
|
|
19535
|
+
const offloaded = await maybeOffload("directory_workflow", ctx, full, summary, { ...structuredContent, cities: capped.items, truncatedCount: capped.truncatedCount });
|
|
19536
|
+
if (offloaded) return offloaded;
|
|
19537
|
+
return { ...oneBlock(full), structuredContent };
|
|
19392
19538
|
}
|
|
19393
19539
|
function formatMapsPlaceIntel(raw, input) {
|
|
19394
19540
|
const parsed = parseData(raw);
|
|
@@ -19908,7 +20054,7 @@ ${rows}` : ""
|
|
|
19908
20054
|
}
|
|
19909
20055
|
};
|
|
19910
20056
|
}
|
|
19911
|
-
var import_node_fs5, import_node_os5, import_node_path7, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD;
|
|
20057
|
+
var import_node_fs5, import_node_os5, import_node_path7, INLINE_BUDGET_BYTES, STRUCTURED_ARRAY_CAP, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD;
|
|
19912
20058
|
var init_mcp_response_formatter = __esm({
|
|
19913
20059
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
19914
20060
|
"use strict";
|
|
@@ -19921,6 +20067,9 @@ var init_mcp_response_formatter = __esm({
|
|
|
19921
20067
|
init_seo_link_report();
|
|
19922
20068
|
init_seo_issues();
|
|
19923
20069
|
init_image_audit();
|
|
20070
|
+
init_report_artifact_offload();
|
|
20071
|
+
INLINE_BUDGET_BYTES = Number(process.env.MCP_SCRAPER_INLINE_BUDGET ?? 5e4);
|
|
20072
|
+
STRUCTURED_ARRAY_CAP = 25;
|
|
19924
20073
|
reportSavingEnabled = true;
|
|
19925
20074
|
BULK_PAGE_THRESHOLD = 25;
|
|
19926
20075
|
BULK_URL_THRESHOLD = 500;
|
|
@@ -22482,7 +22631,7 @@ async function readManifestFromSummary(summary) {
|
|
|
22482
22631
|
function webhookSignature(body, timestamp2) {
|
|
22483
22632
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
22484
22633
|
if (!secret2) return null;
|
|
22485
|
-
return (0,
|
|
22634
|
+
return (0, import_node_crypto8.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
22486
22635
|
}
|
|
22487
22636
|
async function deliverWorkflowWebhook(input) {
|
|
22488
22637
|
if (!input.webhookUrl) return;
|
|
@@ -22689,11 +22838,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
22689
22838
|
}
|
|
22690
22839
|
return { dispatched: results.length, results };
|
|
22691
22840
|
}
|
|
22692
|
-
var
|
|
22841
|
+
var import_node_crypto8, import_promises9, import_hono12, import_zod29, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
22693
22842
|
var init_workflow_routes = __esm({
|
|
22694
22843
|
"src/api/workflow-routes.ts"() {
|
|
22695
22844
|
"use strict";
|
|
22696
|
-
|
|
22845
|
+
import_node_crypto8 = require("crypto");
|
|
22697
22846
|
import_promises9 = require("fs/promises");
|
|
22698
22847
|
import_hono12 = require("hono");
|
|
22699
22848
|
import_zod29 = require("zod");
|
|
@@ -22974,7 +23123,7 @@ var init_workflow_routes = __esm({
|
|
|
22974
23123
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
22975
23124
|
function sha256(value) {
|
|
22976
23125
|
if (!value) return null;
|
|
22977
|
-
return (0,
|
|
23126
|
+
return (0, import_node_crypto9.createHash)("sha256").update(value).digest("hex");
|
|
22978
23127
|
}
|
|
22979
23128
|
function countWords(markdown) {
|
|
22980
23129
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -23274,11 +23423,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
23274
23423
|
}
|
|
23275
23424
|
};
|
|
23276
23425
|
}
|
|
23277
|
-
var
|
|
23426
|
+
var import_node_crypto9, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
23278
23427
|
var init_page_snapshot_extractor = __esm({
|
|
23279
23428
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
23280
23429
|
"use strict";
|
|
23281
|
-
|
|
23430
|
+
import_node_crypto9 = require("crypto");
|
|
23282
23431
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
23283
23432
|
init_kpo_extractor();
|
|
23284
23433
|
init_url_utils();
|
|
@@ -25854,18 +26003,18 @@ var PACKAGE_VERSION;
|
|
|
25854
26003
|
var init_version = __esm({
|
|
25855
26004
|
"src/version.ts"() {
|
|
25856
26005
|
"use strict";
|
|
25857
|
-
PACKAGE_VERSION = "0.3.
|
|
26006
|
+
PACKAGE_VERSION = "0.3.46";
|
|
25858
26007
|
}
|
|
25859
26008
|
});
|
|
25860
26009
|
|
|
25861
26010
|
// src/mcp/server-instructions.ts
|
|
25862
|
-
|
|
25863
|
-
|
|
25864
|
-
|
|
25865
|
-
"use strict";
|
|
25866
|
-
SERVER_INSTRUCTIONS = `
|
|
26011
|
+
function serverInstructions(savesReportsLocally) {
|
|
26012
|
+
const reportLine = savesReportsLocally ? "All report-producing tools also save a full Markdown report to disk." : "On this hosted endpoint, small reports return inline; large reports are stored as artifacts \u2014 read them back with report_artifact_read.";
|
|
26013
|
+
return `
|
|
25867
26014
|
# MCP Scraper
|
|
25868
26015
|
|
|
26016
|
+
${reportLine}
|
|
26017
|
+
|
|
25869
26018
|
Scrapes and analyzes web, search, maps, social, and site data. **Pick a tool by NAME from the map
|
|
25870
26019
|
below, then load its schema before calling.** Where a tool needs a value another tool produces, the
|
|
25871
26020
|
seam is noted so you can chain them.
|
|
@@ -25949,14 +26098,34 @@ Multi-step orchestrations \u2014 prefer these over hand-chaining primitives when
|
|
|
25949
26098
|
sites. It returns a saved folder/artifact plus a summary, not the full content inline.
|
|
25950
26099
|
- Browser sessions and media transcription cost credits and take longer \u2014 prefer the cheapest tool that
|
|
25951
26100
|
answers the question (plain search/extract before browser agents).
|
|
25952
|
-
- Large results are saved to disk or an artifact and returned as a summary plus a path
|
|
25953
|
-
full detail rather than expecting the whole payload inline.
|
|
26101
|
+
- Large results are saved to disk or an artifact and returned as a summary plus a path or artifactId;
|
|
26102
|
+
read it back for full detail rather than expecting the whole payload inline.
|
|
25954
26103
|
`.trim();
|
|
26104
|
+
}
|
|
26105
|
+
var SERVER_INSTRUCTIONS;
|
|
26106
|
+
var init_server_instructions = __esm({
|
|
26107
|
+
"src/mcp/server-instructions.ts"() {
|
|
26108
|
+
"use strict";
|
|
26109
|
+
SERVER_INSTRUCTIONS = serverInstructions(true);
|
|
26110
|
+
}
|
|
26111
|
+
});
|
|
26112
|
+
|
|
26113
|
+
// src/mcp/output-schema-registry.ts
|
|
26114
|
+
function recordOutputSchema(name, schema) {
|
|
26115
|
+
OUTPUT_SCHEMAS[name] = schema;
|
|
26116
|
+
return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
|
|
26117
|
+
}
|
|
26118
|
+
var ADVERTISE_OUTPUT_SCHEMAS, OUTPUT_SCHEMAS;
|
|
26119
|
+
var init_output_schema_registry = __esm({
|
|
26120
|
+
"src/mcp/output-schema-registry.ts"() {
|
|
26121
|
+
"use strict";
|
|
26122
|
+
ADVERTISE_OUTPUT_SCHEMAS = process.env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true";
|
|
26123
|
+
OUTPUT_SCHEMAS = {};
|
|
25955
26124
|
}
|
|
25956
26125
|
});
|
|
25957
26126
|
|
|
25958
26127
|
// src/mcp/mcp-tool-schemas.ts
|
|
25959
|
-
var import_zod31, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
26128
|
+
var import_zod31, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema;
|
|
25960
26129
|
var init_mcp_tool_schemas = __esm({
|
|
25961
26130
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
25962
26131
|
"use strict";
|
|
@@ -26105,12 +26274,18 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26105
26274
|
maxResultsPerCity: import_zod31.z.number().int().min(1).max(50).default(50).describe("Google Maps candidates to collect per city."),
|
|
26106
26275
|
concurrency: import_zod31.z.number().int().min(1).max(5).default(5).describe("City Maps searches to run in parallel."),
|
|
26107
26276
|
includeZipGroups: import_zod31.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available (MCP_SCRAPER_USZIPS_CSV_PATH or usZipsCsvPath)."),
|
|
26108
|
-
usZipsCsvPath: import_zod31.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
26277
|
+
usZipsCsvPath: import_zod31.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead. For ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the server, or pass this in local/test mode."),
|
|
26109
26278
|
saveCsv: import_zod31.z.boolean().default(true).describe("Save a directory-ready CSV of results to the MCP Scraper output directory and return its path."),
|
|
26110
26279
|
proxyMode: import_zod31.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting per city search. Defaults to location (city/ZIP-group residential targeting); configured forces the service proxy; none is local debugging only."),
|
|
26111
26280
|
proxyZip: import_zod31.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting; normally omitted."),
|
|
26112
26281
|
debug: import_zod31.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
26113
26282
|
};
|
|
26283
|
+
ArtifactPointerOutputSchema = import_zod31.z.object({
|
|
26284
|
+
artifactId: import_zod31.z.string(),
|
|
26285
|
+
bytes: import_zod31.z.number().int().min(0),
|
|
26286
|
+
expiresAt: import_zod31.z.string(),
|
|
26287
|
+
preview: import_zod31.z.string()
|
|
26288
|
+
});
|
|
26114
26289
|
RankTrackerModeSchema = import_zod31.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
26115
26290
|
RankTrackerBlueprintInputSchema = {
|
|
26116
26291
|
projectName: import_zod31.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
@@ -26224,7 +26399,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26224
26399
|
attempts: import_zod31.z.array(MapsSearchAttemptOutput),
|
|
26225
26400
|
results: import_zod31.z.array(DirectoryMapsBusinessOutput)
|
|
26226
26401
|
})),
|
|
26227
|
-
durationMs: import_zod31.z.number().int().min(0)
|
|
26402
|
+
durationMs: import_zod31.z.number().int().min(0),
|
|
26403
|
+
truncatedCount: import_zod31.z.number().int().min(0).optional(),
|
|
26404
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26228
26405
|
};
|
|
26229
26406
|
RankTrackerToolPlanOutput = import_zod31.z.object({
|
|
26230
26407
|
tool: import_zod31.z.string(),
|
|
@@ -26344,7 +26521,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26344
26521
|
title: NullableString,
|
|
26345
26522
|
schemaTypes: import_zod31.z.array(import_zod31.z.string())
|
|
26346
26523
|
})),
|
|
26347
|
-
durationMs: import_zod31.z.number().min(0)
|
|
26524
|
+
durationMs: import_zod31.z.number().min(0),
|
|
26525
|
+
truncatedCount: import_zod31.z.number().int().min(0).optional(),
|
|
26526
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26348
26527
|
};
|
|
26349
26528
|
AuditSiteOutputSchema = {
|
|
26350
26529
|
url: import_zod31.z.string(),
|
|
@@ -26364,7 +26543,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26364
26543
|
orphans: import_zod31.z.number().int().min(0),
|
|
26365
26544
|
brokenInternal: import_zod31.z.number().int().min(0),
|
|
26366
26545
|
externalDomains: import_zod31.z.number().int().min(0)
|
|
26367
|
-
})
|
|
26546
|
+
}),
|
|
26547
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26368
26548
|
};
|
|
26369
26549
|
MapsPlaceIntelOutputSchema = {
|
|
26370
26550
|
name: import_zod31.z.string(),
|
|
@@ -26436,7 +26616,9 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26436
26616
|
url: import_zod31.z.string(),
|
|
26437
26617
|
status: import_zod31.z.number().int().nullable()
|
|
26438
26618
|
})),
|
|
26439
|
-
durationMs: import_zod31.z.number().min(0)
|
|
26619
|
+
durationMs: import_zod31.z.number().min(0),
|
|
26620
|
+
truncatedCount: import_zod31.z.number().int().min(0).optional(),
|
|
26621
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
26440
26622
|
};
|
|
26441
26623
|
YoutubeHarvestOutputSchema = {
|
|
26442
26624
|
mode: import_zod31.z.string(),
|
|
@@ -26868,6 +27050,17 @@ var init_mcp_tool_schemas = __esm({
|
|
|
26868
27050
|
timeoutMs: import_zod31.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
26869
27051
|
debug: import_zod31.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
26870
27052
|
};
|
|
27053
|
+
ReportArtifactReadInputSchema = {
|
|
27054
|
+
artifactId: import_zod31.z.string().min(1).describe("Artifact id returned inline by a tool whose result was too large to inline. Use only a returned artifactId; do not construct one yourself."),
|
|
27055
|
+
offset: import_zod31.z.number().int().min(0).default(0).describe("Byte offset to start reading from. Pass the previous call's nextOffset to continue."),
|
|
27056
|
+
maxBytes: import_zod31.z.number().int().min(1e3).max(1e5).default(2e4).describe("Maximum bytes of artifact text to return in this window.")
|
|
27057
|
+
};
|
|
27058
|
+
ReportArtifactReadOutputSchema = {
|
|
27059
|
+
artifactId: import_zod31.z.string(),
|
|
27060
|
+
text: import_zod31.z.string(),
|
|
27061
|
+
totalBytes: import_zod31.z.number().int().min(0),
|
|
27062
|
+
nextOffset: import_zod31.z.number().int().min(0).nullable()
|
|
27063
|
+
};
|
|
26871
27064
|
}
|
|
26872
27065
|
});
|
|
26873
27066
|
|
|
@@ -27201,6 +27394,9 @@ var init_rank_tracker_blueprint = __esm({
|
|
|
27201
27394
|
});
|
|
27202
27395
|
|
|
27203
27396
|
// src/mcp/paa-mcp-server.ts
|
|
27397
|
+
function hashOwnerId(callerKey) {
|
|
27398
|
+
return (0, import_node_crypto10.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
27399
|
+
}
|
|
27204
27400
|
function liveWebToolAnnotations(title) {
|
|
27205
27401
|
return {
|
|
27206
27402
|
title,
|
|
@@ -27213,16 +27409,16 @@ function liveWebToolAnnotations(title) {
|
|
|
27213
27409
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
27214
27410
|
server.registerTool("capture_serp_snapshot", {
|
|
27215
27411
|
title: "SERP Intelligence Snapshot",
|
|
27216
|
-
description: "Capture a structured SERP Intelligence Google
|
|
27412
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
27217
27413
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
27218
|
-
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
27414
|
+
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
27219
27415
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
27220
27416
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
27221
27417
|
server.registerTool("capture_serp_page_snapshots", {
|
|
27222
27418
|
title: "SERP Intelligence Page Snapshots",
|
|
27223
|
-
description: "Capture public ranking
|
|
27419
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
|
|
27224
27420
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
27225
|
-
outputSchema: CaptureSerpPageSnapshotsOutputSchema,
|
|
27421
|
+
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
27226
27422
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
27227
27423
|
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
27228
27424
|
}
|
|
@@ -27270,230 +27466,251 @@ function registerSavedReportResources(server) {
|
|
|
27270
27466
|
);
|
|
27271
27467
|
}
|
|
27272
27468
|
function buildPaaExtractorMcpServer(executor, options = {}) {
|
|
27273
|
-
const server = new import_mcp.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions:
|
|
27469
|
+
const server = new import_mcp.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: serverInstructions(options.savesReportsLocally !== false) });
|
|
27274
27470
|
registerPaaExtractorMcpTools(server, executor, options);
|
|
27275
27471
|
return server;
|
|
27276
27472
|
}
|
|
27277
27473
|
function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
27278
27474
|
const savesReports = options.savesReportsLocally !== false;
|
|
27279
|
-
const
|
|
27280
|
-
const
|
|
27475
|
+
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
27476
|
+
const ownerId = options.ownerId ?? "local";
|
|
27477
|
+
const ctx = { hosted: !savesReports, ownerId };
|
|
27281
27478
|
if (savesReports) registerSavedReportResources(server);
|
|
27282
27479
|
server.registerTool("harvest_paa", {
|
|
27283
27480
|
title: "Google PAA + SERP Harvest",
|
|
27284
|
-
description:
|
|
27481
|
+
description: "Best default tool for Google search research: People Also Ask questions with answers/sources, organic SERP, local pack, entity IDs, and AI Overview. Split topic from location; leave proxyMode unset. Warn the user before maxQuestions above 100 \u2014 deep harvests can run several minutes with no interim progress, billed per extracted question.",
|
|
27285
27482
|
inputSchema: HarvestPaaInputSchema,
|
|
27286
|
-
outputSchema: HarvestPaaOutputSchema,
|
|
27483
|
+
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
27287
27484
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
27288
27485
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
27289
27486
|
server.registerTool("search_serp", {
|
|
27290
27487
|
title: "Google SERP Lookup",
|
|
27291
|
-
description:
|
|
27488
|
+
description: "Fast Google SERP lookup without PAA expansion \u2014 rankings, organic results, local pack, positions. Split topic from location; leave proxyMode unset.",
|
|
27292
27489
|
inputSchema: SearchSerpInputSchema,
|
|
27293
|
-
outputSchema: SearchSerpOutputSchema,
|
|
27490
|
+
outputSchema: recordOutputSchema("search_serp", SearchSerpOutputSchema),
|
|
27294
27491
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
27295
27492
|
}, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
|
|
27296
27493
|
server.registerTool("extract_url", {
|
|
27297
27494
|
title: "Single URL Extract",
|
|
27298
|
-
description:
|
|
27495
|
+
description: "Extract structured data from one public URL: content, schema, headings, metadata, screenshots, branding, or media assets. Set depositToVault:true to save the full page into the user's MCP Memory vault server-side (not returned to chat).",
|
|
27299
27496
|
inputSchema: ExtractUrlInputSchema,
|
|
27300
|
-
outputSchema: ExtractUrlOutputSchema,
|
|
27497
|
+
outputSchema: recordOutputSchema("extract_url", ExtractUrlOutputSchema),
|
|
27301
27498
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
27302
27499
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
27303
27500
|
server.registerTool("map_site_urls", {
|
|
27304
27501
|
title: "Site URL Map",
|
|
27305
|
-
description:
|
|
27502
|
+
description: `Map/crawl a public website for a sitemap, URL inventory, or broken-link scan. Returns internal URLs with HTTP status; ${fileBehavior("maps over 500 URLs are written to a local CSV file instead of inlined.", "large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
27306
27503
|
inputSchema: MapSiteUrlsInputSchema,
|
|
27307
|
-
outputSchema: MapSiteUrlsOutputSchema,
|
|
27504
|
+
outputSchema: recordOutputSchema("map_site_urls", MapSiteUrlsOutputSchema),
|
|
27308
27505
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
27309
|
-
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
27506
|
+
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input, ctx));
|
|
27310
27507
|
server.registerTool("extract_site", {
|
|
27311
27508
|
title: "Multi-Page Site Content Crawl",
|
|
27312
|
-
description:
|
|
27509
|
+
description: `Crawl a public website and return page CONTENT (Markdown) across multiple pages. ${fileBehavior("Bulk crawls over 25 pages are saved as per-page Markdown files in a local folder instead of inlined.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Content only \u2014 for a technical SEO audit use audit_site instead.`,
|
|
27313
27510
|
inputSchema: ExtractSiteInputSchema,
|
|
27314
|
-
outputSchema: ExtractSiteOutputSchema,
|
|
27511
|
+
outputSchema: recordOutputSchema("extract_site", ExtractSiteOutputSchema),
|
|
27315
27512
|
annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
|
|
27316
|
-
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
27513
|
+
}, async (input) => formatExtractSite(await executor.extractSite(input), input, ctx));
|
|
27317
27514
|
server.registerTool("audit_site", {
|
|
27318
27515
|
title: "Technical SEO Audit",
|
|
27319
|
-
description:
|
|
27516
|
+
description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. ${fileBehavior("Writes a folder of analysis files plus per-page content, and returns a summary plus the folder path.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Use extract_site instead for plain page content.`,
|
|
27320
27517
|
inputSchema: AuditSiteInputSchema,
|
|
27321
|
-
outputSchema: AuditSiteOutputSchema,
|
|
27518
|
+
outputSchema: recordOutputSchema("audit_site", AuditSiteOutputSchema),
|
|
27322
27519
|
annotations: liveWebToolAnnotations("Technical SEO Audit")
|
|
27323
|
-
}, async (input) => formatAuditSite(await executor.auditSite(input), input));
|
|
27520
|
+
}, async (input) => formatAuditSite(await executor.auditSite(input), input, ctx));
|
|
27324
27521
|
server.registerTool("youtube_harvest", {
|
|
27325
27522
|
title: "YouTube Video Harvest",
|
|
27326
|
-
description:
|
|
27523
|
+
description: 'Harvest YouTube video metadata by topic search or channel library. Use mode "search" for keyword/topic requests, mode "channel" for @handles/channel IDs/URLs. Returns titles, views, durations, and videoIds.',
|
|
27327
27524
|
inputSchema: YoutubeHarvestInputSchema,
|
|
27328
|
-
outputSchema: YoutubeHarvestOutputSchema,
|
|
27525
|
+
outputSchema: recordOutputSchema("youtube_harvest", YoutubeHarvestOutputSchema),
|
|
27329
27526
|
annotations: liveWebToolAnnotations("YouTube Video Harvest")
|
|
27330
27527
|
}, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
|
|
27331
27528
|
server.registerTool("youtube_transcribe", {
|
|
27332
27529
|
title: "YouTube Transcription",
|
|
27333
|
-
description:
|
|
27530
|
+
description: "Fetch and transcribe captions from a YouTube video. Pass videoId from youtube_harvest, or a url the user pasted. Returns full transcript, timestamped chunks, and word count.",
|
|
27334
27531
|
inputSchema: YoutubeTranscribeInputSchema,
|
|
27335
|
-
outputSchema: YoutubeTranscribeOutputSchema,
|
|
27532
|
+
outputSchema: recordOutputSchema("youtube_transcribe", YoutubeTranscribeOutputSchema),
|
|
27336
27533
|
annotations: liveWebToolAnnotations("YouTube Transcription")
|
|
27337
27534
|
}, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
|
|
27338
27535
|
server.registerTool("facebook_page_intel", {
|
|
27339
27536
|
title: "Facebook Advertiser Ad Intel",
|
|
27340
|
-
description:
|
|
27537
|
+
description: "Harvest ads from a Facebook advertiser: copy, creative angles, CTAs, and direct video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name.",
|
|
27341
27538
|
inputSchema: FacebookPageIntelInputSchema,
|
|
27342
|
-
outputSchema: FacebookPageIntelOutputSchema,
|
|
27539
|
+
outputSchema: recordOutputSchema("facebook_page_intel", FacebookPageIntelOutputSchema),
|
|
27343
27540
|
annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
|
|
27344
27541
|
}, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
|
|
27345
27542
|
server.registerTool("facebook_ad_search", {
|
|
27346
27543
|
title: "Facebook Ad Library Search",
|
|
27347
|
-
description:
|
|
27544
|
+
description: "Search Facebook Ad Library to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs.",
|
|
27348
27545
|
inputSchema: FacebookAdSearchInputSchema,
|
|
27349
|
-
outputSchema: FacebookAdSearchOutputSchema,
|
|
27546
|
+
outputSchema: recordOutputSchema("facebook_ad_search", FacebookAdSearchOutputSchema),
|
|
27350
27547
|
annotations: liveWebToolAnnotations("Facebook Ad Library Search")
|
|
27351
27548
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
27352
27549
|
server.registerTool("reddit_thread", {
|
|
27353
27550
|
title: "Reddit Thread + Comments",
|
|
27354
|
-
description:
|
|
27551
|
+
description: "Capture a Reddit post and its comment tree from a reddit.com thread URL \u2014 comments, opinions, audience voice. Handles Reddit's bot protection automatically; pass maxComments to cap the list.",
|
|
27355
27552
|
inputSchema: RedditThreadInputSchema,
|
|
27356
|
-
outputSchema: RedditThreadOutputSchema,
|
|
27553
|
+
outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
|
|
27357
27554
|
annotations: liveWebToolAnnotations("Reddit Thread + Comments")
|
|
27358
27555
|
}, async (input) => formatRedditThread(await executor.redditThread(input), input));
|
|
27359
27556
|
server.registerTool("video_frame_analysis", {
|
|
27360
27557
|
title: "Video Breakdown (frame-by-frame + transcript)",
|
|
27361
27558
|
description: "Produce a deep frame-by-frame + transcript breakdown of a video \u2014 pacing, hook, visual style, and how to replicate it. Pass a DIRECT video file URL (.mp4/.webm/.mov), not a page URL. Runs asynchronously and costs a flat $1 (refunded on failure): returns a runId immediately; poll video_frame_analysis_status until done.",
|
|
27362
27559
|
inputSchema: VideoFrameAnalysisInputSchema,
|
|
27363
|
-
outputSchema: VideoFrameAnalysisOutputSchema,
|
|
27560
|
+
outputSchema: recordOutputSchema("video_frame_analysis", VideoFrameAnalysisOutputSchema),
|
|
27364
27561
|
annotations: { title: "Video Breakdown", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
|
|
27365
27562
|
}, async (input) => executor.videoFrameAnalysis(input));
|
|
27366
27563
|
server.registerTool("video_frame_analysis_status", {
|
|
27367
27564
|
title: "Video Breakdown Status",
|
|
27368
27565
|
description: 'Check progress of a video breakdown started with video_frame_analysis, using its runId. Free to call. When status is "done" it returns the full report and vault path; stop polling on "done" or "failed".',
|
|
27369
27566
|
inputSchema: VideoFrameAnalysisStatusInputSchema,
|
|
27370
|
-
outputSchema: VideoFrameAnalysisStatusOutputSchema,
|
|
27567
|
+
outputSchema: recordOutputSchema("video_frame_analysis_status", VideoFrameAnalysisStatusOutputSchema),
|
|
27371
27568
|
annotations: { title: "Video Breakdown Status", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
27372
27569
|
}, async (input) => executor.videoFrameAnalysisStatus(input));
|
|
27373
27570
|
server.registerTool("facebook_ad_transcribe", {
|
|
27374
27571
|
title: "Facebook Ad Transcription",
|
|
27375
27572
|
description: "Transcribe audio from a Facebook ad video CDN URL returned by facebook_page_intel. Use only that direct videoUrl value \u2014 do not pass public Facebook post/reel/share URLs (use facebook_video_transcribe for those).",
|
|
27376
27573
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
27377
|
-
outputSchema: FacebookAdTranscribeOutputSchema,
|
|
27574
|
+
outputSchema: recordOutputSchema("facebook_ad_transcribe", FacebookAdTranscribeOutputSchema),
|
|
27378
27575
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
27379
27576
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
27380
27577
|
server.registerTool("google_ads_search", {
|
|
27381
27578
|
title: "Google Ads Transparency Search",
|
|
27382
|
-
description:
|
|
27579
|
+
description: "Search the Google Ads Transparency Center to find advertisers by domain or brand name. Returns advertisers with advertiser ID and approximate ad count, to pass to google_ads_page_intel.",
|
|
27383
27580
|
inputSchema: GoogleAdsSearchInputSchema,
|
|
27384
|
-
outputSchema: GoogleAdsSearchOutputSchema,
|
|
27581
|
+
outputSchema: recordOutputSchema("google_ads_search", GoogleAdsSearchOutputSchema),
|
|
27385
27582
|
annotations: liveWebToolAnnotations("Google Ads Transparency Search")
|
|
27386
27583
|
}, async (input) => formatGoogleAdsSearch(await executor.googleAdsSearch(input), input));
|
|
27387
27584
|
server.registerTool("google_ads_page_intel", {
|
|
27388
27585
|
title: "Google Ads Advertiser Intel",
|
|
27389
|
-
description:
|
|
27586
|
+
description: "Harvest an advertiser's ad creatives from the Google Ads Transparency Center: format, image URLs, and \u2014 for video ads \u2014 a YouTube video ID or direct video URL. Accepts advertiserId (from google_ads_search) or a domain.",
|
|
27390
27587
|
inputSchema: GoogleAdsPageIntelInputSchema,
|
|
27391
|
-
outputSchema: GoogleAdsPageIntelOutputSchema,
|
|
27588
|
+
outputSchema: recordOutputSchema("google_ads_page_intel", GoogleAdsPageIntelOutputSchema),
|
|
27392
27589
|
annotations: liveWebToolAnnotations("Google Ads Advertiser Intel")
|
|
27393
27590
|
}, async (input) => formatGoogleAdsPageIntel(await executor.googleAdsPageIntel(input), input));
|
|
27394
27591
|
server.registerTool("google_ads_transcribe", {
|
|
27395
27592
|
title: "Google Ad Video Transcription",
|
|
27396
27593
|
description: "Transcribe audio from a Google video ad's direct videoUrl (a googlevideo.com playback URL) returned by google_ads_page_intel. For YouTube-hosted ads, use youtube_transcribe with the returned youtubeVideoId instead.",
|
|
27397
27594
|
inputSchema: GoogleAdsTranscribeInputSchema,
|
|
27398
|
-
outputSchema: GoogleAdsTranscribeOutputSchema,
|
|
27595
|
+
outputSchema: recordOutputSchema("google_ads_transcribe", GoogleAdsTranscribeOutputSchema),
|
|
27399
27596
|
annotations: liveWebToolAnnotations("Google Ad Video Transcription")
|
|
27400
27597
|
}, async (input) => formatGoogleAdsTranscribe(await executor.googleAdsTranscribe(input), input));
|
|
27401
27598
|
server.registerTool("facebook_video_transcribe", {
|
|
27402
27599
|
title: "Facebook Organic Video Transcription",
|
|
27403
|
-
description:
|
|
27600
|
+
description: "Transcribe audio from an organic Facebook reel/video/post/share URL (including fb.watch). Renders the page, selects the best public CDN MP4, and returns the transcript plus resolved video metadata.",
|
|
27404
27601
|
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
27405
|
-
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
27602
|
+
outputSchema: recordOutputSchema("facebook_video_transcribe", FacebookVideoTranscribeOutputSchema),
|
|
27406
27603
|
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
27407
27604
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
27408
27605
|
server.registerTool("instagram_profile_content", {
|
|
27409
27606
|
title: "Instagram Profile Content Discovery",
|
|
27410
|
-
description:
|
|
27607
|
+
description: "Discover Instagram profile grid content links (posts/reels/tv) for a handle or profile URL, for later selection with instagram_media_download. Returns profile stats and collected URLs.",
|
|
27411
27608
|
inputSchema: InstagramProfileContentInputSchema,
|
|
27412
|
-
outputSchema: InstagramProfileContentOutputSchema,
|
|
27609
|
+
outputSchema: recordOutputSchema("instagram_profile_content", InstagramProfileContentOutputSchema),
|
|
27413
27610
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
27414
27611
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
27415
27612
|
server.registerTool("instagram_media_download", {
|
|
27416
27613
|
title: "Instagram Post/Reel Media Download",
|
|
27417
|
-
description:
|
|
27614
|
+
description: "Extract and download media from one Instagram post, reel, or tv URL \u2014 image, caption, video/audio tracks, optional muxed MP4, or transcript. Selects the best video/audio track pair and muxes when ffmpeg is available.",
|
|
27418
27615
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
27419
|
-
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
27616
|
+
outputSchema: recordOutputSchema("instagram_media_download", InstagramMediaDownloadOutputSchema),
|
|
27420
27617
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
27421
27618
|
}, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
|
|
27422
27619
|
server.registerTool("maps_place_intel", {
|
|
27423
27620
|
title: "Google Maps Business Profile Details",
|
|
27424
|
-
description:
|
|
27621
|
+
description: "Extract Google Maps business intelligence for one known/named business: rating, reviews, category, address, phone, hours, entity IDs. Not for category searches or multi-business prospect lists \u2014 use maps_search for those. Split business name from location.",
|
|
27425
27622
|
inputSchema: MapsPlaceIntelInputSchema,
|
|
27426
|
-
outputSchema: MapsPlaceIntelOutputSchema,
|
|
27623
|
+
outputSchema: recordOutputSchema("maps_place_intel", MapsPlaceIntelOutputSchema),
|
|
27427
27624
|
annotations: liveWebToolAnnotations("Google Maps Business Profile Details")
|
|
27428
27625
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
27429
27626
|
server.registerTool("maps_search", {
|
|
27430
27627
|
title: "Google Maps Business Search",
|
|
27431
|
-
description:
|
|
27628
|
+
description: "Search Google Maps for multiple businesses by category, niche, or local market \u2014 leads, prospects, competitors, or beyond the 3-pack. Leave proxyMode unset. Returns up to 50 candidates (default 10) with names, place URLs, CIDs, ratings.",
|
|
27432
27629
|
inputSchema: MapsSearchInputSchema,
|
|
27433
|
-
outputSchema: MapsSearchOutputSchema,
|
|
27630
|
+
outputSchema: recordOutputSchema("maps_search", MapsSearchOutputSchema),
|
|
27434
27631
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
27435
27632
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
27436
27633
|
server.registerTool("directory_workflow", {
|
|
27437
27634
|
title: "Directory Workflow: Markets + Maps",
|
|
27438
|
-
description:
|
|
27635
|
+
description: `Build directory/prospecting datasets: selects US city markets from Census population data, optionally joins configured ZIP groups, then runs Google Maps business searches per city in parallel. Use for "all cities over 100k population in a state" or market+Maps workflows. ${fileBehavior("Saves a CSV of results per city.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
27439
27636
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
27440
|
-
outputSchema: DirectoryWorkflowOutputSchema,
|
|
27637
|
+
outputSchema: recordOutputSchema("directory_workflow", DirectoryWorkflowOutputSchema),
|
|
27441
27638
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
27442
|
-
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input));
|
|
27639
|
+
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input, ctx));
|
|
27443
27640
|
server.registerTool("workflow_list", {
|
|
27444
27641
|
title: "Workflow Catalog",
|
|
27445
27642
|
description: "List MCP Scraper higher-level workflows and recipes \u2014 market analysis, ICP research, CRO audits, competitive positioning, content gap briefs, AI search visibility, and more. Returns runnable workflow ids plus tool-chain guidance.",
|
|
27446
27643
|
inputSchema: WorkflowListInputSchema,
|
|
27447
|
-
outputSchema: WorkflowListOutputSchema,
|
|
27644
|
+
outputSchema: recordOutputSchema("workflow_list", WorkflowListOutputSchema),
|
|
27448
27645
|
annotations: localPlanningToolAnnotations("Workflow Catalog")
|
|
27449
27646
|
}, async (input) => formatWorkflowList(await executor.workflowList(input), input));
|
|
27450
27647
|
server.registerTool("workflow_suggest", {
|
|
27451
27648
|
title: "Workflow Intent Router",
|
|
27452
27649
|
description: "Route a high-level business/research goal (market analysis, ICP research, CRO audit, competitor comparison, content gap brief, AI search visibility, etc) to the right MCP Scraper workflow/tool chain. Free; tells you what to run next.",
|
|
27453
27650
|
inputSchema: WorkflowSuggestInputSchema,
|
|
27454
|
-
outputSchema: WorkflowSuggestOutputSchema,
|
|
27651
|
+
outputSchema: recordOutputSchema("workflow_suggest", WorkflowSuggestOutputSchema),
|
|
27455
27652
|
annotations: localPlanningToolAnnotations("Workflow Intent Router")
|
|
27456
27653
|
}, async (input) => formatWorkflowSuggest(input));
|
|
27457
27654
|
server.registerTool("workflow_run", {
|
|
27458
27655
|
title: "Run Workflow",
|
|
27459
|
-
description:
|
|
27656
|
+
description: "Start a higher-level MCP Scraper workflow (directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language). Use after workflow_suggest or workflow_list. Stepwise workflows return runId + nextStep \u2014 call workflow_step with the runId until done is true.",
|
|
27460
27657
|
inputSchema: WorkflowRunInputSchema,
|
|
27461
|
-
outputSchema: WorkflowRunOutputSchema,
|
|
27658
|
+
outputSchema: recordOutputSchema("workflow_run", WorkflowRunOutputSchema),
|
|
27462
27659
|
annotations: liveWebToolAnnotations("Run Workflow")
|
|
27463
27660
|
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
27464
27661
|
server.registerTool("workflow_step", {
|
|
27465
27662
|
title: "Advance Workflow Step",
|
|
27466
|
-
description:
|
|
27663
|
+
description: "Run the next leg of a stepwise workflow started with workflow_run. Pass the runId; each call executes one step and returns its output plus nextStep. Keep calling with the same runId until done is true.",
|
|
27467
27664
|
inputSchema: WorkflowStepInputSchema,
|
|
27468
|
-
outputSchema: WorkflowStepOutputSchema,
|
|
27665
|
+
outputSchema: recordOutputSchema("workflow_step", WorkflowStepOutputSchema),
|
|
27469
27666
|
annotations: liveWebToolAnnotations("Advance Workflow Step")
|
|
27470
27667
|
}, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
|
|
27471
27668
|
server.registerTool("workflow_status", {
|
|
27472
27669
|
title: "Workflow Status",
|
|
27473
27670
|
description: "Fetch a hosted workflow run by id and list its current status and artifacts, to re-open a run or recover artifact ids. Use only a runId returned by workflow_run/workflow_step/workflow_status.",
|
|
27474
27671
|
inputSchema: WorkflowStatusInputSchema,
|
|
27475
|
-
outputSchema: WorkflowStatusOutputSchema,
|
|
27672
|
+
outputSchema: recordOutputSchema("workflow_status", WorkflowStatusOutputSchema),
|
|
27476
27673
|
annotations: liveWebToolAnnotations("Workflow Status")
|
|
27477
27674
|
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
27478
27675
|
server.registerTool("workflow_artifact_read", {
|
|
27479
27676
|
title: "Read Workflow Artifact",
|
|
27480
27677
|
description: "Read a workflow artifact back into context by run id and artifact id, so final deliverables are grounded in generated evidence rather than memory. Use workflow_status first when artifact ids are unknown. Use maxBytes to limit large artifacts.",
|
|
27481
27678
|
inputSchema: WorkflowArtifactReadInputSchema,
|
|
27482
|
-
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
27679
|
+
outputSchema: recordOutputSchema("workflow_artifact_read", WorkflowArtifactReadOutputSchema),
|
|
27483
27680
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
27484
27681
|
}, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
|
|
27682
|
+
server.registerTool("report_artifact_read", {
|
|
27683
|
+
title: "Read Report Artifact",
|
|
27684
|
+
description: "Read back a stored report artifact by artifactId (returned by any tool whose result was too large to inline). Windowed: pass offset/maxBytes and keep reading until nextOffset is null.",
|
|
27685
|
+
inputSchema: ReportArtifactReadInputSchema,
|
|
27686
|
+
outputSchema: recordOutputSchema("report_artifact_read", ReportArtifactReadOutputSchema),
|
|
27687
|
+
annotations: liveWebToolAnnotations("Read Report Artifact")
|
|
27688
|
+
}, async (input) => {
|
|
27689
|
+
const owner = artifactOwnerId(input.artifactId);
|
|
27690
|
+
if (!owner || owner !== ownerId) {
|
|
27691
|
+
return { content: [{ type: "text", text: "Artifact not found or not owned by this caller." }], isError: true };
|
|
27692
|
+
}
|
|
27693
|
+
const window2 = await readArtifactWindow(input.artifactId, input.offset ?? 0, input.maxBytes ?? 2e4);
|
|
27694
|
+
if (!window2) {
|
|
27695
|
+
return { content: [{ type: "text", text: "Artifact not found or expired." }], isError: true };
|
|
27696
|
+
}
|
|
27697
|
+
return {
|
|
27698
|
+
content: [{ type: "text", text: window2.text }],
|
|
27699
|
+
structuredContent: { artifactId: input.artifactId, text: window2.text, totalBytes: window2.totalBytes, nextOffset: window2.nextOffset }
|
|
27700
|
+
};
|
|
27701
|
+
});
|
|
27485
27702
|
server.registerTool("rank_tracker_workflow", {
|
|
27486
27703
|
title: "Rank Tracker Blueprint Builder",
|
|
27487
27704
|
description: "Generate a build-ready database schema, cron plan, and implementation prompt for a rank tracker powered by MCP Scraper (Maps, organic, AI Overview, or PAA tracking). Local planning only \u2014 does not call the web or spend credits.",
|
|
27488
27705
|
inputSchema: RankTrackerBlueprintInputSchema,
|
|
27489
|
-
outputSchema: RankTrackerBlueprintOutputSchema,
|
|
27706
|
+
outputSchema: recordOutputSchema("rank_tracker_workflow", RankTrackerBlueprintOutputSchema),
|
|
27490
27707
|
annotations: localPlanningToolAnnotations("Rank Tracker Blueprint Builder")
|
|
27491
27708
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
27492
27709
|
server.registerTool("credits_info", {
|
|
27493
27710
|
title: "MCP Scraper Credits & Costs",
|
|
27494
27711
|
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades \u2014 balance, tool costs, concurrency limits, billing URL. Does not expose payment methods or card information.",
|
|
27495
27712
|
inputSchema: CreditsInfoInputSchema,
|
|
27496
|
-
outputSchema: CreditsInfoOutputSchema,
|
|
27713
|
+
outputSchema: recordOutputSchema("credits_info", CreditsInfoOutputSchema),
|
|
27497
27714
|
annotations: {
|
|
27498
27715
|
title: "MCP Scraper Credits & Costs",
|
|
27499
27716
|
readOnlyHint: true,
|
|
@@ -27503,16 +27720,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
27503
27720
|
}
|
|
27504
27721
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
27505
27722
|
}
|
|
27506
|
-
var import_mcp, import_node_fs8, import_node_path11;
|
|
27723
|
+
var import_mcp, import_node_fs8, import_node_path11, import_node_crypto10;
|
|
27507
27724
|
var init_paa_mcp_server = __esm({
|
|
27508
27725
|
"src/mcp/paa-mcp-server.ts"() {
|
|
27509
27726
|
"use strict";
|
|
27510
27727
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
27511
27728
|
import_node_fs8 = require("fs");
|
|
27512
27729
|
import_node_path11 = require("path");
|
|
27730
|
+
import_node_crypto10 = require("crypto");
|
|
27513
27731
|
init_version();
|
|
27514
27732
|
init_mcp_response_formatter();
|
|
27515
27733
|
init_server_instructions();
|
|
27734
|
+
init_output_schema_registry();
|
|
27735
|
+
init_report_artifact_offload();
|
|
27516
27736
|
init_mcp_tool_schemas();
|
|
27517
27737
|
init_rank_tracker_blueprint();
|
|
27518
27738
|
init_mcp_response_formatter();
|
|
@@ -28724,7 +28944,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28724
28944
|
title: "Save a Site Login to a Profile",
|
|
28725
28945
|
description: "Save a logged-in browser session so this server can reuse a site the user is signed into (ChatGPT, Claude, Reddit, any account-gated site). Returns a watch_url; the user signs in fresh (existing browser cookies are NOT imported) and cookies save to a named profile. ONE profile holds MANY logins \u2014 call again with the same profile and a different domain to stack another account. NOT for one-off scraping (use extract_url) or driving the browser (use browser_open). After sign-in, poll browser_profile_list until AUTHENTICATED, then browser_open with the profile.",
|
|
28726
28946
|
inputSchema: BrowserProfileConnectInputSchema,
|
|
28727
|
-
outputSchema: BrowserProfileConnectOutputSchema,
|
|
28947
|
+
outputSchema: recordOutputSchema("browser_profile_connect", BrowserProfileConnectOutputSchema),
|
|
28728
28948
|
annotations: annotations("Save a Site Login to a Profile")
|
|
28729
28949
|
},
|
|
28730
28950
|
async (input) => {
|
|
@@ -28782,7 +29002,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28782
29002
|
title: "List Saved Logins in a Profile",
|
|
28783
29003
|
description: "List every site login saved in a profile with its auth status (NEEDS_AUTH/AUTHENTICATED), email, and note. Use to check what's connected, or to poll a just-saved login until AUTHENTICATED. Read-only, no cost. Pass profile (or email to derive it); narrow with domain or connection_id.",
|
|
28784
29004
|
inputSchema: BrowserProfileListInputSchema,
|
|
28785
|
-
outputSchema: BrowserProfileListOutputSchema,
|
|
29005
|
+
outputSchema: recordOutputSchema("browser_profile_list", BrowserProfileListOutputSchema),
|
|
28786
29006
|
annotations: annotations("List Saved Logins in a Profile", true)
|
|
28787
29007
|
},
|
|
28788
29008
|
async (input) => {
|
|
@@ -28816,7 +29036,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28816
29036
|
title: "Open Browser Session",
|
|
28817
29037
|
description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load a session already logged into that profile's sites (set one up first with browser_profile_connect). Returns a session_id used by all other browser_* tools.",
|
|
28818
29038
|
inputSchema: BrowserOpenInputSchema,
|
|
28819
|
-
outputSchema: BrowserOpenOutputSchema,
|
|
29039
|
+
outputSchema: recordOutputSchema("browser_open", BrowserOpenOutputSchema),
|
|
28820
29040
|
annotations: annotations("Open Browser Session")
|
|
28821
29041
|
},
|
|
28822
29042
|
async (input) => {
|
|
@@ -28850,7 +29070,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28850
29070
|
title: "See Page (Screenshot + Elements)",
|
|
28851
29071
|
description: "Capture what the browser currently shows: a screenshot plus a text snapshot of interactive elements with x,y coordinates, page url/title, and visible text. Primary way to perceive the page; click elements by their listed x,y. If a Cloudflare/CAPTCHA challenge is visible, wait and screenshot again rather than clicking it.",
|
|
28852
29072
|
inputSchema: BrowserSessionInputSchema,
|
|
28853
|
-
outputSchema: BrowserScreenshotOutputSchema,
|
|
29073
|
+
outputSchema: recordOutputSchema("browser_screenshot", BrowserScreenshotOutputSchema),
|
|
28854
29074
|
annotations: annotations("See Page", true)
|
|
28855
29075
|
},
|
|
28856
29076
|
async (input) => {
|
|
@@ -28882,7 +29102,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28882
29102
|
title: "Read Page Text + Elements",
|
|
28883
29103
|
description: "Return the page url, title, visible text, and interactive elements (with x,y) without an image. Cheaper than browser_screenshot when you only need to read content or find a click target.",
|
|
28884
29104
|
inputSchema: BrowserSessionInputSchema,
|
|
28885
|
-
outputSchema: BrowserReadOutputSchema,
|
|
29105
|
+
outputSchema: recordOutputSchema("browser_read", BrowserReadOutputSchema),
|
|
28886
29106
|
annotations: annotations("Read Page", true)
|
|
28887
29107
|
},
|
|
28888
29108
|
async (input) => {
|
|
@@ -28906,7 +29126,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28906
29126
|
title: "Locate DOM Targets",
|
|
28907
29127
|
description: "Locate exact visible DOM elements or text ranges and return left/top/width/height bounds in screenshot pixels. Use before drawing annotations that must circle, box, underline, or point to a real element. Prefer CSS selectors; use text when selector is unknown.",
|
|
28908
29128
|
inputSchema: BrowserLocateInputSchema,
|
|
28909
|
-
outputSchema: BrowserLocateOutputSchema,
|
|
29129
|
+
outputSchema: recordOutputSchema("browser_locate", BrowserLocateOutputSchema),
|
|
28910
29130
|
annotations: annotations("Locate DOM Targets", true)
|
|
28911
29131
|
},
|
|
28912
29132
|
async (input) => {
|
|
@@ -28931,7 +29151,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28931
29151
|
title: "Navigate To URL",
|
|
28932
29152
|
description: "Navigate an existing browser session to a URL. Use browser_open first if no session exists; follow with browser_screenshot to see the loaded page.",
|
|
28933
29153
|
inputSchema: BrowserGotoInputSchema,
|
|
28934
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29154
|
+
outputSchema: recordOutputSchema("browser_goto", BrowserActionOutputSchema),
|
|
28935
29155
|
annotations: annotations("Navigate To URL")
|
|
28936
29156
|
},
|
|
28937
29157
|
async (input) => {
|
|
@@ -28945,7 +29165,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28945
29165
|
title: "Click",
|
|
28946
29166
|
description: "Click a visible page target using screenshot pixel coordinates. Use x/y only from the latest browser_screenshot, browser_read, or browser_locate result; do not guess coordinates.",
|
|
28947
29167
|
inputSchema: BrowserClickInputSchema,
|
|
28948
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29168
|
+
outputSchema: recordOutputSchema("browser_click", BrowserActionOutputSchema),
|
|
28949
29169
|
annotations: annotations("Click")
|
|
28950
29170
|
},
|
|
28951
29171
|
async (input) => {
|
|
@@ -28964,7 +29184,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28964
29184
|
title: "Type Text",
|
|
28965
29185
|
description: 'Type text into the currently focused browser field. Click or Tab to the field first if focus is uncertain. Use browser_press with ["Return"] to submit.',
|
|
28966
29186
|
inputSchema: BrowserTypeInputSchema,
|
|
28967
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29187
|
+
outputSchema: recordOutputSchema("browser_type", BrowserActionOutputSchema),
|
|
28968
29188
|
annotations: annotations("Type Text")
|
|
28969
29189
|
},
|
|
28970
29190
|
async (input) => {
|
|
@@ -28978,7 +29198,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28978
29198
|
title: "Scroll",
|
|
28979
29199
|
description: "Scroll the page to reveal more content. Positive delta_y scrolls down; negative scrolls up.",
|
|
28980
29200
|
inputSchema: BrowserScrollInputSchema,
|
|
28981
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29201
|
+
outputSchema: recordOutputSchema("browser_scroll", BrowserActionOutputSchema),
|
|
28982
29202
|
annotations: annotations("Scroll")
|
|
28983
29203
|
},
|
|
28984
29204
|
async (input) => {
|
|
@@ -28997,7 +29217,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
28997
29217
|
title: "Press Keys",
|
|
28998
29218
|
description: "Press keyboard keys or combinations in the active browser session \u2014 submit, Escape, Tab navigation, select-all, or shortcuts. Use browser_type for text entry.",
|
|
28999
29219
|
inputSchema: BrowserPressInputSchema,
|
|
29000
|
-
outputSchema: BrowserActionOutputSchema,
|
|
29220
|
+
outputSchema: recordOutputSchema("browser_press", BrowserActionOutputSchema),
|
|
29001
29221
|
annotations: annotations("Press Keys")
|
|
29002
29222
|
},
|
|
29003
29223
|
async (input) => {
|
|
@@ -29011,7 +29231,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29011
29231
|
title: "Start Recording",
|
|
29012
29232
|
description: "Start recording an MP4 replay of the session. Returns replay_id and a download_url. Stop with browser_replay_stop.",
|
|
29013
29233
|
inputSchema: BrowserSessionInputSchema,
|
|
29014
|
-
outputSchema: BrowserReplayStartOutputSchema,
|
|
29234
|
+
outputSchema: recordOutputSchema("browser_replay_start", BrowserReplayStartOutputSchema),
|
|
29015
29235
|
annotations: annotations("Start Recording")
|
|
29016
29236
|
},
|
|
29017
29237
|
async (input) => {
|
|
@@ -29034,7 +29254,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29034
29254
|
title: "Stop Recording",
|
|
29035
29255
|
description: "Stop a replay recording and expose its final view_url/download_url. Use browser_replay_download to save the MP4.",
|
|
29036
29256
|
inputSchema: BrowserReplayStopInputSchema,
|
|
29037
|
-
outputSchema: BrowserReplayStopOutputSchema,
|
|
29257
|
+
outputSchema: recordOutputSchema("browser_replay_stop", BrowserReplayStopOutputSchema),
|
|
29038
29258
|
annotations: annotations("Stop Recording")
|
|
29039
29259
|
},
|
|
29040
29260
|
async (input) => {
|
|
@@ -29057,7 +29277,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29057
29277
|
title: "List Replay Videos",
|
|
29058
29278
|
description: "List replay recordings for a browser session, including view_url and download_url when available.",
|
|
29059
29279
|
inputSchema: BrowserSessionInputSchema,
|
|
29060
|
-
outputSchema: BrowserListReplaysOutputSchema,
|
|
29280
|
+
outputSchema: recordOutputSchema("browser_list_replays", BrowserListReplaysOutputSchema),
|
|
29061
29281
|
annotations: annotations("List Replay Videos", true)
|
|
29062
29282
|
},
|
|
29063
29283
|
async (input) => {
|
|
@@ -29077,9 +29297,9 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29077
29297
|
"browser_replay_download",
|
|
29078
29298
|
{
|
|
29079
29299
|
title: "Download Replay MP4",
|
|
29080
|
-
description: "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
29300
|
+
description: opts.savesReportsLocally === false ? "Download a replay recording. Returns the download_url; fetch it directly (nothing is saved on this hosted endpoint). Use after browser_replay_stop or browser_list_replays." : "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
29081
29301
|
inputSchema: BrowserReplayDownloadInputSchema,
|
|
29082
|
-
outputSchema: BrowserReplayDownloadOutputSchema,
|
|
29302
|
+
outputSchema: recordOutputSchema("browser_replay_download", BrowserReplayDownloadOutputSchema),
|
|
29083
29303
|
annotations: annotations("Download Replay MP4")
|
|
29084
29304
|
},
|
|
29085
29305
|
async (input) => {
|
|
@@ -29103,7 +29323,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29103
29323
|
title: "Mark Replay Annotation",
|
|
29104
29324
|
description: "While a replay is actively recording, locate one exact DOM target and return a ready-to-use annotation with DOM bounds and replay-relative timing, instead of guessing start_seconds or rectangles. Pass the returned annotations to browser_replay_annotate after stopping the replay.",
|
|
29105
29325
|
inputSchema: BrowserReplayMarkInputSchema,
|
|
29106
|
-
outputSchema: BrowserReplayMarkOutputSchema,
|
|
29326
|
+
outputSchema: recordOutputSchema("browser_replay_mark", BrowserReplayMarkOutputSchema),
|
|
29107
29327
|
annotations: annotations("Mark Replay Annotation", true)
|
|
29108
29328
|
},
|
|
29109
29329
|
async (input) => {
|
|
@@ -29153,7 +29373,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29153
29373
|
title: "Annotate Replay MP4",
|
|
29154
29374
|
description: "Download a browser replay MP4, render visual annotations (circles/boxes/arrows/labels) over it, and save a new annotated MP4. Prefer annotations from browser_replay_mark for accurate timing; otherwise use exact bounds from browser_locate. Pass source_width/source_height if the replay video size differs from the screenshot coordinate space.",
|
|
29155
29375
|
inputSchema: BrowserReplayAnnotateInputSchema,
|
|
29156
|
-
outputSchema: BrowserReplayAnnotateOutputSchema,
|
|
29376
|
+
outputSchema: recordOutputSchema("browser_replay_annotate", BrowserReplayAnnotateOutputSchema),
|
|
29157
29377
|
annotations: annotations("Annotate Replay MP4")
|
|
29158
29378
|
},
|
|
29159
29379
|
async (input) => {
|
|
@@ -29195,7 +29415,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29195
29415
|
title: "Close Browser Session",
|
|
29196
29416
|
description: "Close and release a browser session when the task is done, to end active browser billing. Use browser_list_sessions first to recover a session_id.",
|
|
29197
29417
|
inputSchema: BrowserSessionInputSchema,
|
|
29198
|
-
outputSchema: BrowserCloseOutputSchema,
|
|
29418
|
+
outputSchema: recordOutputSchema("browser_close", BrowserCloseOutputSchema),
|
|
29199
29419
|
annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
29200
29420
|
},
|
|
29201
29421
|
async (input) => {
|
|
@@ -29216,7 +29436,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29216
29436
|
title: "List Browser Sessions",
|
|
29217
29437
|
description: "List browser sessions and their status, with a watch_url for each. Use to recover a session_id or decide which session to close.",
|
|
29218
29438
|
inputSchema: BrowserListInputSchema,
|
|
29219
|
-
outputSchema: BrowserListSessionsOutputSchema,
|
|
29439
|
+
outputSchema: recordOutputSchema("browser_list_sessions", BrowserListSessionsOutputSchema),
|
|
29220
29440
|
annotations: annotations("List Browser Sessions", true)
|
|
29221
29441
|
},
|
|
29222
29442
|
async (input) => {
|
|
@@ -29238,7 +29458,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
29238
29458
|
title: "Capture AI Search Fan-Out",
|
|
29239
29459
|
description: "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account \u2014 only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview \u2014 use harvest_paa for that.",
|
|
29240
29460
|
inputSchema: BrowserCaptureFanoutInputSchema,
|
|
29241
|
-
outputSchema: BrowserCaptureFanoutOutputSchema,
|
|
29461
|
+
outputSchema: recordOutputSchema("query_fanout_workflow", BrowserCaptureFanoutOutputSchema),
|
|
29242
29462
|
annotations: annotations("Capture AI Search Fan-Out")
|
|
29243
29463
|
},
|
|
29244
29464
|
async (input) => {
|
|
@@ -29303,6 +29523,7 @@ var init_browser_agent_mcp_server = __esm({
|
|
|
29303
29523
|
init_browser_agent_tool_schemas();
|
|
29304
29524
|
init_replay_annotator();
|
|
29305
29525
|
init_outbound_sanitize();
|
|
29526
|
+
init_output_schema_registry();
|
|
29306
29527
|
}
|
|
29307
29528
|
});
|
|
29308
29529
|
|
|
@@ -29418,9 +29639,9 @@ var init_mcp_routes = __esm({
|
|
|
29418
29639
|
enableJsonResponse: true
|
|
29419
29640
|
});
|
|
29420
29641
|
const consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
29421
|
-
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false });
|
|
29642
|
+
const server = buildPaaExtractorMcpServer(executor, { savesReportsLocally: false, ownerId: hashOwnerId(callerKey) });
|
|
29422
29643
|
registerSerpIntelligenceCaptureTools(server, executor);
|
|
29423
|
-
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl });
|
|
29644
|
+
registerBrowserAgentMcpTools(server, { baseUrl, apiKey: callerKey, consoleBaseUrl, savesReportsLocally: false });
|
|
29424
29645
|
await server.connect(transport);
|
|
29425
29646
|
return transport.handleRequest(c.req.raw);
|
|
29426
29647
|
} catch {
|
|
@@ -29541,7 +29762,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
29541
29762
|
}
|
|
29542
29763
|
async function createSessionRow(input) {
|
|
29543
29764
|
const db = getDb();
|
|
29544
|
-
const id = `bas_${(0,
|
|
29765
|
+
const id = `bas_${(0, import_node_crypto11.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
29545
29766
|
await db.execute({
|
|
29546
29767
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
29547
29768
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -29610,7 +29831,7 @@ async function recordAction(input) {
|
|
|
29610
29831
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
29611
29832
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
29612
29833
|
args: [
|
|
29613
|
-
`baa_${(0,
|
|
29834
|
+
`baa_${(0, import_node_crypto11.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
29614
29835
|
input.sessionId,
|
|
29615
29836
|
input.type,
|
|
29616
29837
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -29650,11 +29871,11 @@ async function listReplayRows(sessionId) {
|
|
|
29650
29871
|
});
|
|
29651
29872
|
return res.rows;
|
|
29652
29873
|
}
|
|
29653
|
-
var
|
|
29874
|
+
var import_node_crypto11, _ready2;
|
|
29654
29875
|
var init_browser_agent_db = __esm({
|
|
29655
29876
|
"src/api/browser-agent-db.ts"() {
|
|
29656
29877
|
"use strict";
|
|
29657
|
-
|
|
29878
|
+
import_node_crypto11 = require("crypto");
|
|
29658
29879
|
init_db();
|
|
29659
29880
|
_ready2 = false;
|
|
29660
29881
|
}
|
|
@@ -31780,7 +32001,7 @@ async function getKeys() {
|
|
|
31780
32001
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
31781
32002
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
31782
32003
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
31783
|
-
const kid = (0,
|
|
32004
|
+
const kid = (0, import_node_crypto12.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
31784
32005
|
publicJwk.kid = kid;
|
|
31785
32006
|
publicJwk.alg = "RS256";
|
|
31786
32007
|
publicJwk.use = "sig";
|
|
@@ -31959,23 +32180,23 @@ async function validateAuthRequest(p) {
|
|
|
31959
32180
|
}
|
|
31960
32181
|
function pkceMatches(verifier, challenge) {
|
|
31961
32182
|
if (!verifier) return false;
|
|
31962
|
-
const computed = (0,
|
|
32183
|
+
const computed = (0, import_node_crypto12.createHash)("sha256").update(verifier).digest("base64url");
|
|
31963
32184
|
return computed === challenge;
|
|
31964
32185
|
}
|
|
31965
32186
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
31966
32187
|
const { privateKey, kid } = await getKeys();
|
|
31967
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
32188
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto12.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
31968
32189
|
}
|
|
31969
32190
|
function tokenErrorResponse(c, error, description, status) {
|
|
31970
32191
|
return c.json({ error, error_description: description }, status);
|
|
31971
32192
|
}
|
|
31972
|
-
var import_hono17, import_cookie,
|
|
32193
|
+
var import_hono17, import_cookie, import_node_crypto12, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
31973
32194
|
var init_oauth_routes = __esm({
|
|
31974
32195
|
"src/api/oauth-routes.ts"() {
|
|
31975
32196
|
"use strict";
|
|
31976
32197
|
import_hono17 = require("hono");
|
|
31977
32198
|
import_cookie = require("hono/cookie");
|
|
31978
|
-
|
|
32199
|
+
import_node_crypto12 = require("crypto");
|
|
31979
32200
|
import_jose2 = require("jose");
|
|
31980
32201
|
init_session();
|
|
31981
32202
|
init_db();
|
|
@@ -32061,7 +32282,7 @@ var init_oauth_routes = __esm({
|
|
|
32061
32282
|
}
|
|
32062
32283
|
}
|
|
32063
32284
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
32064
|
-
const clientId = `client_${(0,
|
|
32285
|
+
const clientId = `client_${(0, import_node_crypto12.randomBytes)(16).toString("hex")}`;
|
|
32065
32286
|
await registerClient(clientId, redirectUris, clientName);
|
|
32066
32287
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
32067
32288
|
return c.json({
|
|
@@ -32114,7 +32335,7 @@ var init_oauth_routes = __esm({
|
|
|
32114
32335
|
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
32115
32336
|
if (action !== "approve") return c.text("unsupported action", 400);
|
|
32116
32337
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
32117
|
-
const code = `code_${(0,
|
|
32338
|
+
const code = `code_${(0, import_node_crypto12.randomBytes)(32).toString("base64url")}`;
|
|
32118
32339
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
32119
32340
|
await putCode({
|
|
32120
32341
|
code,
|
|
@@ -32153,7 +32374,7 @@ var init_oauth_routes = __esm({
|
|
|
32153
32374
|
const plan = user ? resolvePlan(user) : "free";
|
|
32154
32375
|
const audience = record.resource ?? RESOURCE();
|
|
32155
32376
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
32156
|
-
const refreshToken = `rt_${(0,
|
|
32377
|
+
const refreshToken = `rt_${(0, import_node_crypto12.randomBytes)(40).toString("base64url")}`;
|
|
32157
32378
|
await putRefresh({
|
|
32158
32379
|
refresh_token: refreshToken,
|
|
32159
32380
|
client_id: clientId,
|
|
@@ -32183,7 +32404,7 @@ var init_oauth_routes = __esm({
|
|
|
32183
32404
|
const plan = user ? resolvePlan(user) : "free";
|
|
32184
32405
|
const audience = record.resource ?? RESOURCE();
|
|
32185
32406
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
32186
|
-
const nextRefresh = `rt_${(0,
|
|
32407
|
+
const nextRefresh = `rt_${(0, import_node_crypto12.randomBytes)(40).toString("base64url")}`;
|
|
32187
32408
|
await rotateRefresh(refreshToken, {
|
|
32188
32409
|
refresh_token: nextRefresh,
|
|
32189
32410
|
client_id: record.client_id,
|