mcp-scraper 0.3.45 → 0.3.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +423 -170
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +444 -184
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-BRVX6RDN.js → chunk-7TFJJAPE.js} +4 -3
- package/dist/chunk-7TFJJAPE.js.map +1 -0
- package/dist/{chunk-E7BZFMWB.js → chunk-A6BAV444.js} +481 -216
- package/dist/chunk-A6BAV444.js.map +1 -0
- package/dist/chunk-NTCAVBHU.js +7 -0
- package/dist/chunk-NTCAVBHU.js.map +1 -0
- package/dist/{server-MKUN6NQZ.js → server-H6GAAEQ6.js} +61 -85
- package/dist/server-H6GAAEQ6.js.map +1 -0
- package/dist/{worker-4CVMSUZR.js → worker-WY7C45MP.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-BRVX6RDN.js.map +0 -1
- package/dist/chunk-E7BZFMWB.js.map +0 -1
- package/dist/chunk-NVTFLLEG.js +0 -7
- package/dist/chunk-NVTFLLEG.js.map +0 -1
- package/dist/server-MKUN6NQZ.js.map +0 -1
- /package/dist/{worker-4CVMSUZR.js.map → worker-WY7C45MP.js.map} +0 -0
|
@@ -24,9 +24,9 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
24
24
|
));
|
|
25
25
|
|
|
26
26
|
// bin/mcp-stdio-server.ts
|
|
27
|
-
var
|
|
28
|
-
var
|
|
29
|
-
var
|
|
27
|
+
var import_node_fs6 = require("fs");
|
|
28
|
+
var import_node_os6 = require("os");
|
|
29
|
+
var import_node_path7 = require("path");
|
|
30
30
|
var import_mcp3 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
31
31
|
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
32
32
|
|
|
@@ -474,7 +474,7 @@ render();
|
|
|
474
474
|
}
|
|
475
475
|
|
|
476
476
|
// src/version.ts
|
|
477
|
-
var PACKAGE_VERSION = "0.3.
|
|
477
|
+
var PACKAGE_VERSION = "0.3.47";
|
|
478
478
|
|
|
479
479
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
480
480
|
var import_zod = require("zod");
|
|
@@ -1136,6 +1136,14 @@ function sanitizeOutboundDiagnostics(value, parentKey = "") {
|
|
|
1136
1136
|
return value;
|
|
1137
1137
|
}
|
|
1138
1138
|
|
|
1139
|
+
// src/mcp/output-schema-registry.ts
|
|
1140
|
+
var ADVERTISE_OUTPUT_SCHEMAS = process.env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true";
|
|
1141
|
+
var OUTPUT_SCHEMAS = {};
|
|
1142
|
+
function recordOutputSchema(name, schema) {
|
|
1143
|
+
OUTPUT_SCHEMAS[name] = schema;
|
|
1144
|
+
return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1139
1147
|
// src/mcp/browser-agent-mcp-server.ts
|
|
1140
1148
|
function structuredResult(value, isError = false) {
|
|
1141
1149
|
const safe2 = sanitizeOutboundDiagnostics(value);
|
|
@@ -1301,7 +1309,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1301
1309
|
title: "Save a Site Login to a Profile",
|
|
1302
1310
|
description: "Save a logged-in browser session so this server can reuse a site the user is signed into (ChatGPT, Claude, Reddit, any account-gated site). Returns a watch_url; the user signs in fresh (existing browser cookies are NOT imported) and cookies save to a named profile. ONE profile holds MANY logins \u2014 call again with the same profile and a different domain to stack another account. NOT for one-off scraping (use extract_url) or driving the browser (use browser_open). After sign-in, poll browser_profile_list until AUTHENTICATED, then browser_open with the profile.",
|
|
1303
1311
|
inputSchema: BrowserProfileConnectInputSchema,
|
|
1304
|
-
outputSchema: BrowserProfileConnectOutputSchema,
|
|
1312
|
+
outputSchema: recordOutputSchema("browser_profile_connect", BrowserProfileConnectOutputSchema),
|
|
1305
1313
|
annotations: annotations("Save a Site Login to a Profile")
|
|
1306
1314
|
},
|
|
1307
1315
|
async (input) => {
|
|
@@ -1359,7 +1367,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1359
1367
|
title: "List Saved Logins in a Profile",
|
|
1360
1368
|
description: "List every site login saved in a profile with its auth status (NEEDS_AUTH/AUTHENTICATED), email, and note. Use to check what's connected, or to poll a just-saved login until AUTHENTICATED. Read-only, no cost. Pass profile (or email to derive it); narrow with domain or connection_id.",
|
|
1361
1369
|
inputSchema: BrowserProfileListInputSchema,
|
|
1362
|
-
outputSchema: BrowserProfileListOutputSchema,
|
|
1370
|
+
outputSchema: recordOutputSchema("browser_profile_list", BrowserProfileListOutputSchema),
|
|
1363
1371
|
annotations: annotations("List Saved Logins in a Profile", true)
|
|
1364
1372
|
},
|
|
1365
1373
|
async (input) => {
|
|
@@ -1393,7 +1401,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1393
1401
|
title: "Open Browser Session",
|
|
1394
1402
|
description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load a session already logged into that profile's sites (set one up first with browser_profile_connect). Returns a session_id used by all other browser_* tools.",
|
|
1395
1403
|
inputSchema: BrowserOpenInputSchema,
|
|
1396
|
-
outputSchema: BrowserOpenOutputSchema,
|
|
1404
|
+
outputSchema: recordOutputSchema("browser_open", BrowserOpenOutputSchema),
|
|
1397
1405
|
annotations: annotations("Open Browser Session")
|
|
1398
1406
|
},
|
|
1399
1407
|
async (input) => {
|
|
@@ -1427,7 +1435,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1427
1435
|
title: "See Page (Screenshot + Elements)",
|
|
1428
1436
|
description: "Capture what the browser currently shows: a screenshot plus a text snapshot of interactive elements with x,y coordinates, page url/title, and visible text. Primary way to perceive the page; click elements by their listed x,y. If a Cloudflare/CAPTCHA challenge is visible, wait and screenshot again rather than clicking it.",
|
|
1429
1437
|
inputSchema: BrowserSessionInputSchema,
|
|
1430
|
-
outputSchema: BrowserScreenshotOutputSchema,
|
|
1438
|
+
outputSchema: recordOutputSchema("browser_screenshot", BrowserScreenshotOutputSchema),
|
|
1431
1439
|
annotations: annotations("See Page", true)
|
|
1432
1440
|
},
|
|
1433
1441
|
async (input) => {
|
|
@@ -1459,7 +1467,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1459
1467
|
title: "Read Page Text + Elements",
|
|
1460
1468
|
description: "Return the page url, title, visible text, and interactive elements (with x,y) without an image. Cheaper than browser_screenshot when you only need to read content or find a click target.",
|
|
1461
1469
|
inputSchema: BrowserSessionInputSchema,
|
|
1462
|
-
outputSchema: BrowserReadOutputSchema,
|
|
1470
|
+
outputSchema: recordOutputSchema("browser_read", BrowserReadOutputSchema),
|
|
1463
1471
|
annotations: annotations("Read Page", true)
|
|
1464
1472
|
},
|
|
1465
1473
|
async (input) => {
|
|
@@ -1483,7 +1491,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1483
1491
|
title: "Locate DOM Targets",
|
|
1484
1492
|
description: "Locate exact visible DOM elements or text ranges and return left/top/width/height bounds in screenshot pixels. Use before drawing annotations that must circle, box, underline, or point to a real element. Prefer CSS selectors; use text when selector is unknown.",
|
|
1485
1493
|
inputSchema: BrowserLocateInputSchema,
|
|
1486
|
-
outputSchema: BrowserLocateOutputSchema,
|
|
1494
|
+
outputSchema: recordOutputSchema("browser_locate", BrowserLocateOutputSchema),
|
|
1487
1495
|
annotations: annotations("Locate DOM Targets", true)
|
|
1488
1496
|
},
|
|
1489
1497
|
async (input) => {
|
|
@@ -1508,7 +1516,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1508
1516
|
title: "Navigate To URL",
|
|
1509
1517
|
description: "Navigate an existing browser session to a URL. Use browser_open first if no session exists; follow with browser_screenshot to see the loaded page.",
|
|
1510
1518
|
inputSchema: BrowserGotoInputSchema,
|
|
1511
|
-
outputSchema: BrowserActionOutputSchema,
|
|
1519
|
+
outputSchema: recordOutputSchema("browser_goto", BrowserActionOutputSchema),
|
|
1512
1520
|
annotations: annotations("Navigate To URL")
|
|
1513
1521
|
},
|
|
1514
1522
|
async (input) => {
|
|
@@ -1522,7 +1530,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1522
1530
|
title: "Click",
|
|
1523
1531
|
description: "Click a visible page target using screenshot pixel coordinates. Use x/y only from the latest browser_screenshot, browser_read, or browser_locate result; do not guess coordinates.",
|
|
1524
1532
|
inputSchema: BrowserClickInputSchema,
|
|
1525
|
-
outputSchema: BrowserActionOutputSchema,
|
|
1533
|
+
outputSchema: recordOutputSchema("browser_click", BrowserActionOutputSchema),
|
|
1526
1534
|
annotations: annotations("Click")
|
|
1527
1535
|
},
|
|
1528
1536
|
async (input) => {
|
|
@@ -1541,7 +1549,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1541
1549
|
title: "Type Text",
|
|
1542
1550
|
description: 'Type text into the currently focused browser field. Click or Tab to the field first if focus is uncertain. Use browser_press with ["Return"] to submit.',
|
|
1543
1551
|
inputSchema: BrowserTypeInputSchema,
|
|
1544
|
-
outputSchema: BrowserActionOutputSchema,
|
|
1552
|
+
outputSchema: recordOutputSchema("browser_type", BrowserActionOutputSchema),
|
|
1545
1553
|
annotations: annotations("Type Text")
|
|
1546
1554
|
},
|
|
1547
1555
|
async (input) => {
|
|
@@ -1555,7 +1563,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1555
1563
|
title: "Scroll",
|
|
1556
1564
|
description: "Scroll the page to reveal more content. Positive delta_y scrolls down; negative scrolls up.",
|
|
1557
1565
|
inputSchema: BrowserScrollInputSchema,
|
|
1558
|
-
outputSchema: BrowserActionOutputSchema,
|
|
1566
|
+
outputSchema: recordOutputSchema("browser_scroll", BrowserActionOutputSchema),
|
|
1559
1567
|
annotations: annotations("Scroll")
|
|
1560
1568
|
},
|
|
1561
1569
|
async (input) => {
|
|
@@ -1574,7 +1582,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1574
1582
|
title: "Press Keys",
|
|
1575
1583
|
description: "Press keyboard keys or combinations in the active browser session \u2014 submit, Escape, Tab navigation, select-all, or shortcuts. Use browser_type for text entry.",
|
|
1576
1584
|
inputSchema: BrowserPressInputSchema,
|
|
1577
|
-
outputSchema: BrowserActionOutputSchema,
|
|
1585
|
+
outputSchema: recordOutputSchema("browser_press", BrowserActionOutputSchema),
|
|
1578
1586
|
annotations: annotations("Press Keys")
|
|
1579
1587
|
},
|
|
1580
1588
|
async (input) => {
|
|
@@ -1588,7 +1596,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1588
1596
|
title: "Start Recording",
|
|
1589
1597
|
description: "Start recording an MP4 replay of the session. Returns replay_id and a download_url. Stop with browser_replay_stop.",
|
|
1590
1598
|
inputSchema: BrowserSessionInputSchema,
|
|
1591
|
-
outputSchema: BrowserReplayStartOutputSchema,
|
|
1599
|
+
outputSchema: recordOutputSchema("browser_replay_start", BrowserReplayStartOutputSchema),
|
|
1592
1600
|
annotations: annotations("Start Recording")
|
|
1593
1601
|
},
|
|
1594
1602
|
async (input) => {
|
|
@@ -1611,7 +1619,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1611
1619
|
title: "Stop Recording",
|
|
1612
1620
|
description: "Stop a replay recording and expose its final view_url/download_url. Use browser_replay_download to save the MP4.",
|
|
1613
1621
|
inputSchema: BrowserReplayStopInputSchema,
|
|
1614
|
-
outputSchema: BrowserReplayStopOutputSchema,
|
|
1622
|
+
outputSchema: recordOutputSchema("browser_replay_stop", BrowserReplayStopOutputSchema),
|
|
1615
1623
|
annotations: annotations("Stop Recording")
|
|
1616
1624
|
},
|
|
1617
1625
|
async (input) => {
|
|
@@ -1634,7 +1642,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1634
1642
|
title: "List Replay Videos",
|
|
1635
1643
|
description: "List replay recordings for a browser session, including view_url and download_url when available.",
|
|
1636
1644
|
inputSchema: BrowserSessionInputSchema,
|
|
1637
|
-
outputSchema: BrowserListReplaysOutputSchema,
|
|
1645
|
+
outputSchema: recordOutputSchema("browser_list_replays", BrowserListReplaysOutputSchema),
|
|
1638
1646
|
annotations: annotations("List Replay Videos", true)
|
|
1639
1647
|
},
|
|
1640
1648
|
async (input) => {
|
|
@@ -1654,9 +1662,9 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1654
1662
|
"browser_replay_download",
|
|
1655
1663
|
{
|
|
1656
1664
|
title: "Download Replay MP4",
|
|
1657
|
-
description: "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
1665
|
+
description: opts.savesReportsLocally === false ? "Download a replay recording. Returns the download_url; fetch it directly (nothing is saved on this hosted endpoint). Use after browser_replay_stop or browser_list_replays." : "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
1658
1666
|
inputSchema: BrowserReplayDownloadInputSchema,
|
|
1659
|
-
outputSchema: BrowserReplayDownloadOutputSchema,
|
|
1667
|
+
outputSchema: recordOutputSchema("browser_replay_download", BrowserReplayDownloadOutputSchema),
|
|
1660
1668
|
annotations: annotations("Download Replay MP4")
|
|
1661
1669
|
},
|
|
1662
1670
|
async (input) => {
|
|
@@ -1680,7 +1688,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1680
1688
|
title: "Mark Replay Annotation",
|
|
1681
1689
|
description: "While a replay is actively recording, locate one exact DOM target and return a ready-to-use annotation with DOM bounds and replay-relative timing, instead of guessing start_seconds or rectangles. Pass the returned annotations to browser_replay_annotate after stopping the replay.",
|
|
1682
1690
|
inputSchema: BrowserReplayMarkInputSchema,
|
|
1683
|
-
outputSchema: BrowserReplayMarkOutputSchema,
|
|
1691
|
+
outputSchema: recordOutputSchema("browser_replay_mark", BrowserReplayMarkOutputSchema),
|
|
1684
1692
|
annotations: annotations("Mark Replay Annotation", true)
|
|
1685
1693
|
},
|
|
1686
1694
|
async (input) => {
|
|
@@ -1730,7 +1738,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1730
1738
|
title: "Annotate Replay MP4",
|
|
1731
1739
|
description: "Download a browser replay MP4, render visual annotations (circles/boxes/arrows/labels) over it, and save a new annotated MP4. Prefer annotations from browser_replay_mark for accurate timing; otherwise use exact bounds from browser_locate. Pass source_width/source_height if the replay video size differs from the screenshot coordinate space.",
|
|
1732
1740
|
inputSchema: BrowserReplayAnnotateInputSchema,
|
|
1733
|
-
outputSchema: BrowserReplayAnnotateOutputSchema,
|
|
1741
|
+
outputSchema: recordOutputSchema("browser_replay_annotate", BrowserReplayAnnotateOutputSchema),
|
|
1734
1742
|
annotations: annotations("Annotate Replay MP4")
|
|
1735
1743
|
},
|
|
1736
1744
|
async (input) => {
|
|
@@ -1772,7 +1780,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1772
1780
|
title: "Close Browser Session",
|
|
1773
1781
|
description: "Close and release a browser session when the task is done, to end active browser billing. Use browser_list_sessions first to recover a session_id.",
|
|
1774
1782
|
inputSchema: BrowserSessionInputSchema,
|
|
1775
|
-
outputSchema: BrowserCloseOutputSchema,
|
|
1783
|
+
outputSchema: recordOutputSchema("browser_close", BrowserCloseOutputSchema),
|
|
1776
1784
|
annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
1777
1785
|
},
|
|
1778
1786
|
async (input) => {
|
|
@@ -1793,7 +1801,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1793
1801
|
title: "List Browser Sessions",
|
|
1794
1802
|
description: "List browser sessions and their status, with a watch_url for each. Use to recover a session_id or decide which session to close.",
|
|
1795
1803
|
inputSchema: BrowserListInputSchema,
|
|
1796
|
-
outputSchema: BrowserListSessionsOutputSchema,
|
|
1804
|
+
outputSchema: recordOutputSchema("browser_list_sessions", BrowserListSessionsOutputSchema),
|
|
1797
1805
|
annotations: annotations("List Browser Sessions", true)
|
|
1798
1806
|
},
|
|
1799
1807
|
async (input) => {
|
|
@@ -1815,7 +1823,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1815
1823
|
title: "Capture AI Search Fan-Out",
|
|
1816
1824
|
description: "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account \u2014 only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview \u2014 use harvest_paa for that.",
|
|
1817
1825
|
inputSchema: BrowserCaptureFanoutInputSchema,
|
|
1818
|
-
outputSchema: BrowserCaptureFanoutOutputSchema,
|
|
1826
|
+
outputSchema: recordOutputSchema("query_fanout_workflow", BrowserCaptureFanoutOutputSchema),
|
|
1819
1827
|
annotations: annotations("Capture AI Search Fan-Out")
|
|
1820
1828
|
},
|
|
1821
1829
|
async (input) => {
|
|
@@ -1869,13 +1877,14 @@ function registerBrowserAgentMcpTools(server2, opts) {
|
|
|
1869
1877
|
|
|
1870
1878
|
// src/mcp/paa-mcp-server.ts
|
|
1871
1879
|
var import_mcp2 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
1872
|
-
var
|
|
1873
|
-
var
|
|
1880
|
+
var import_node_fs5 = require("fs");
|
|
1881
|
+
var import_node_path6 = require("path");
|
|
1882
|
+
var import_node_crypto2 = require("crypto");
|
|
1874
1883
|
|
|
1875
1884
|
// src/mcp/mcp-response-formatter.ts
|
|
1876
|
-
var
|
|
1877
|
-
var
|
|
1878
|
-
var
|
|
1885
|
+
var import_node_fs4 = require("fs");
|
|
1886
|
+
var import_node_os5 = require("os");
|
|
1887
|
+
var import_node_path5 = require("path");
|
|
1879
1888
|
|
|
1880
1889
|
// src/mcp/workflow-catalog.ts
|
|
1881
1890
|
var WORKFLOW_RECIPES = [
|
|
@@ -2412,7 +2421,129 @@ function renderImageSection(audit) {
|
|
|
2412
2421
|
return lines.join("\n");
|
|
2413
2422
|
}
|
|
2414
2423
|
|
|
2424
|
+
// src/mcp/report-artifact-offload.ts
|
|
2425
|
+
var import_node_crypto = require("crypto");
|
|
2426
|
+
|
|
2427
|
+
// src/api/blob-store.ts
|
|
2428
|
+
var import_node_fs3 = require("fs");
|
|
2429
|
+
var import_node_os4 = require("os");
|
|
2430
|
+
var import_node_path4 = require("path");
|
|
2431
|
+
function byteLength(data) {
|
|
2432
|
+
return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
|
|
2433
|
+
}
|
|
2434
|
+
var LocalBlobStore = class {
|
|
2435
|
+
constructor(baseDir) {
|
|
2436
|
+
this.baseDir = baseDir;
|
|
2437
|
+
}
|
|
2438
|
+
baseDir;
|
|
2439
|
+
kind = "local";
|
|
2440
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
2441
|
+
const path = (0, import_node_path4.join)(this.baseDir, "blobs", key);
|
|
2442
|
+
(0, import_node_fs3.mkdirSync)((0, import_node_path4.dirname)(path), { recursive: true });
|
|
2443
|
+
(0, import_node_fs3.writeFileSync)(path, data);
|
|
2444
|
+
return { key, url: `file://${path}`, bytes: byteLength(data), contentType };
|
|
2445
|
+
}
|
|
2446
|
+
async get(key) {
|
|
2447
|
+
const path = (0, import_node_path4.join)(this.baseDir, "blobs", key);
|
|
2448
|
+
try {
|
|
2449
|
+
const { readFileSync: readFileSync3 } = await import("fs");
|
|
2450
|
+
return readFileSync3(path);
|
|
2451
|
+
} catch {
|
|
2452
|
+
return null;
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
};
|
|
2456
|
+
function localBaseDir() {
|
|
2457
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path4.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
|
|
2458
|
+
}
|
|
2459
|
+
var cached = null;
|
|
2460
|
+
function getBlobStore() {
|
|
2461
|
+
if (cached) return cached;
|
|
2462
|
+
if (process.env.BLOB_READ_WRITE_TOKEN) {
|
|
2463
|
+
cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
|
|
2464
|
+
} else {
|
|
2465
|
+
cached = new LocalBlobStore(localBaseDir());
|
|
2466
|
+
}
|
|
2467
|
+
return cached;
|
|
2468
|
+
}
|
|
2469
|
+
var VercelBlobStore = class {
|
|
2470
|
+
constructor(token) {
|
|
2471
|
+
this.token = token;
|
|
2472
|
+
}
|
|
2473
|
+
token;
|
|
2474
|
+
kind = "vercel-blob";
|
|
2475
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
2476
|
+
const { put } = await import("@vercel/blob");
|
|
2477
|
+
const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
2478
|
+
const result = await put(key, body, {
|
|
2479
|
+
access: "public",
|
|
2480
|
+
token: this.token,
|
|
2481
|
+
contentType,
|
|
2482
|
+
addRandomSuffix: true
|
|
2483
|
+
});
|
|
2484
|
+
return { key, url: result.url, bytes: byteLength(data), contentType };
|
|
2485
|
+
}
|
|
2486
|
+
async get(key) {
|
|
2487
|
+
try {
|
|
2488
|
+
const { list } = await import("@vercel/blob");
|
|
2489
|
+
const res = await list({ prefix: key, token: this.token, limit: 1 });
|
|
2490
|
+
const match = res.blobs.find((b) => b.pathname === key || b.pathname.startsWith(key));
|
|
2491
|
+
if (!match) return null;
|
|
2492
|
+
const resp = await fetch(match.url);
|
|
2493
|
+
if (!resp.ok) return null;
|
|
2494
|
+
return Buffer.from(await resp.arrayBuffer());
|
|
2495
|
+
} catch {
|
|
2496
|
+
return null;
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
};
|
|
2500
|
+
|
|
2501
|
+
// src/mcp/report-artifact-offload.ts
|
|
2502
|
+
var REPORT_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
2503
|
+
var REPORT_BLOB_PREFIX = "mcp-reports/";
|
|
2504
|
+
var PREVIEW_CHARS = 2e3;
|
|
2505
|
+
var ARTIFACT_OFFLOAD_ENABLED = process.env.MCP_SCRAPER_ARTIFACT_OFFLOAD !== "false";
|
|
2506
|
+
async function offloadReport(toolName, ownerId, report) {
|
|
2507
|
+
const timestamp = Date.now();
|
|
2508
|
+
const random = (0, import_node_crypto.randomBytes)(6).toString("hex");
|
|
2509
|
+
const key = `${REPORT_BLOB_PREFIX}${ownerId}/${toolName}/${timestamp}-${random}.md`;
|
|
2510
|
+
const stored = await getBlobStore().put(key, report, "text/markdown");
|
|
2511
|
+
return {
|
|
2512
|
+
artifactId: stored.key,
|
|
2513
|
+
bytes: stored.bytes,
|
|
2514
|
+
expiresAt: new Date(timestamp + REPORT_BLOB_TTL_MS).toISOString(),
|
|
2515
|
+
preview: report.slice(0, PREVIEW_CHARS)
|
|
2516
|
+
};
|
|
2517
|
+
}
|
|
2518
|
+
function artifactOwnerId(artifactId) {
|
|
2519
|
+
if (!artifactId.startsWith(REPORT_BLOB_PREFIX)) return null;
|
|
2520
|
+
const rest = artifactId.slice(REPORT_BLOB_PREFIX.length);
|
|
2521
|
+
const segment = rest.split("/")[0];
|
|
2522
|
+
return segment || null;
|
|
2523
|
+
}
|
|
2524
|
+
async function readArtifactWindow(artifactId, offset, maxBytes) {
|
|
2525
|
+
const buf = await getBlobStore().get(artifactId);
|
|
2526
|
+
if (!buf) return null;
|
|
2527
|
+
const totalBytes = buf.length;
|
|
2528
|
+
const start = Math.max(0, offset);
|
|
2529
|
+
const end = Math.min(totalBytes, start + maxBytes);
|
|
2530
|
+
const slice = buf.subarray(start, end);
|
|
2531
|
+
const nextOffset = end < totalBytes ? end : null;
|
|
2532
|
+
return { text: slice.toString("utf8"), totalBytes, nextOffset };
|
|
2533
|
+
}
|
|
2534
|
+
function summaryEnvelope(executiveSummary, offloaded) {
|
|
2535
|
+
return [
|
|
2536
|
+
executiveSummary.trim(),
|
|
2537
|
+
"",
|
|
2538
|
+
"--- Full report stored as artifact ---",
|
|
2539
|
+
`artifactId: ${offloaded.artifactId} \xB7 ${offloaded.bytes} bytes \xB7 expires ${offloaded.expiresAt}`,
|
|
2540
|
+
"Read it with report_artifact_read (supports offset/maxBytes windowing)."
|
|
2541
|
+
].join("\n");
|
|
2542
|
+
}
|
|
2543
|
+
|
|
2415
2544
|
// src/mcp/mcp-response-formatter.ts
|
|
2545
|
+
var INLINE_BUDGET_BYTES = Number(process.env.MCP_SCRAPER_INLINE_BUDGET ?? 5e4);
|
|
2546
|
+
var STRUCTURED_ARRAY_CAP = 25;
|
|
2416
2547
|
var reportSavingEnabled = true;
|
|
2417
2548
|
function sanitizeVendorText(text) {
|
|
2418
2549
|
return sanitizeVendorName(
|
|
@@ -2427,16 +2558,16 @@ function reportTitle(full) {
|
|
|
2427
2558
|
return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
|
|
2428
2559
|
}
|
|
2429
2560
|
function outputBaseDir3() {
|
|
2430
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
2561
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path5.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
|
|
2431
2562
|
}
|
|
2432
2563
|
function saveFullReport(full) {
|
|
2433
2564
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
2434
2565
|
const outDir = outputBaseDir3();
|
|
2435
2566
|
try {
|
|
2436
|
-
(0,
|
|
2567
|
+
(0, import_node_fs4.mkdirSync)(outDir, { recursive: true });
|
|
2437
2568
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
2438
|
-
const file = (0,
|
|
2439
|
-
(0,
|
|
2569
|
+
const file = (0, import_node_path5.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
|
|
2570
|
+
(0, import_node_fs4.writeFileSync)(file, full, "utf8");
|
|
2440
2571
|
return file;
|
|
2441
2572
|
} catch {
|
|
2442
2573
|
return null;
|
|
@@ -2451,9 +2582,9 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
|
2451
2582
|
if (!reportSavingActive()) return null;
|
|
2452
2583
|
try {
|
|
2453
2584
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
2454
|
-
const dir = (0,
|
|
2455
|
-
const pagesDir = (0,
|
|
2456
|
-
(0,
|
|
2585
|
+
const dir = (0, import_node_path5.join)(outputBaseDir3(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
|
|
2586
|
+
const pagesDir = (0, import_node_path5.join)(dir, "pages");
|
|
2587
|
+
(0, import_node_fs4.mkdirSync)(pagesDir, { recursive: true });
|
|
2457
2588
|
const indexRows = pages.map((p, i) => {
|
|
2458
2589
|
const num = String(i + 1).padStart(4, "0");
|
|
2459
2590
|
const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
|
|
@@ -2467,7 +2598,7 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
|
2467
2598
|
"",
|
|
2468
2599
|
body || "_(no content extracted)_"
|
|
2469
2600
|
].filter(Boolean).join("\n");
|
|
2470
|
-
(0,
|
|
2601
|
+
(0, import_node_fs4.writeFileSync)((0, import_node_path5.join)(pagesDir, fname), content, "utf8");
|
|
2471
2602
|
return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
|
|
2472
2603
|
});
|
|
2473
2604
|
const dataFilesSection = seo ? [
|
|
@@ -2499,40 +2630,40 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
|
2499
2630
|
|---|-------|-----|------|
|
|
2500
2631
|
${indexRows.join("\n")}`
|
|
2501
2632
|
].filter(Boolean).join("\n");
|
|
2502
|
-
const indexFile = (0,
|
|
2503
|
-
(0,
|
|
2633
|
+
const indexFile = (0, import_node_path5.join)(dir, "index.md");
|
|
2634
|
+
(0, import_node_fs4.writeFileSync)(indexFile, index, "utf8");
|
|
2504
2635
|
let seoFiles;
|
|
2505
2636
|
if (seo) {
|
|
2506
2637
|
const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
|
|
2507
|
-
const pagesJsonl = (0,
|
|
2508
|
-
const linksJsonl = (0,
|
|
2509
|
-
const metricsJsonl = (0,
|
|
2510
|
-
const issuesFile = (0,
|
|
2511
|
-
const reportFile = (0,
|
|
2512
|
-
(0,
|
|
2513
|
-
(0,
|
|
2514
|
-
(0,
|
|
2515
|
-
(0,
|
|
2516
|
-
(0,
|
|
2638
|
+
const pagesJsonl = (0, import_node_path5.join)(dir, "pages.jsonl");
|
|
2639
|
+
const linksJsonl = (0, import_node_path5.join)(dir, "links.jsonl");
|
|
2640
|
+
const metricsJsonl = (0, import_node_path5.join)(dir, "link-metrics.jsonl");
|
|
2641
|
+
const issuesFile = (0, import_node_path5.join)(dir, "issues.json");
|
|
2642
|
+
const reportFile = (0, import_node_path5.join)(dir, "report.md");
|
|
2643
|
+
(0, import_node_fs4.writeFileSync)(pagesJsonl, toJsonl(seo.pageRows), "utf8");
|
|
2644
|
+
(0, import_node_fs4.writeFileSync)(linksJsonl, toJsonl(seo.edges), "utf8");
|
|
2645
|
+
(0, import_node_fs4.writeFileSync)(metricsJsonl, toJsonl(seo.metrics), "utf8");
|
|
2646
|
+
(0, import_node_fs4.writeFileSync)(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
|
|
2647
|
+
(0, import_node_fs4.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
|
|
2517
2648
|
|
|
2518
2649
|
${renderImageSection(imageAudit)}` : ""), "utf8");
|
|
2519
|
-
const linkReportFile = (0,
|
|
2520
|
-
const linksSummaryFile = (0,
|
|
2521
|
-
const externalDomainsFile = (0,
|
|
2522
|
-
(0,
|
|
2523
|
-
(0,
|
|
2524
|
-
(0,
|
|
2650
|
+
const linkReportFile = (0, import_node_path5.join)(dir, "link-report.md");
|
|
2651
|
+
const linksSummaryFile = (0, import_node_path5.join)(dir, "links-summary.json");
|
|
2652
|
+
const externalDomainsFile = (0, import_node_path5.join)(dir, "external-domains.json");
|
|
2653
|
+
(0, import_node_fs4.writeFileSync)(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
|
|
2654
|
+
(0, import_node_fs4.writeFileSync)(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
|
|
2655
|
+
(0, import_node_fs4.writeFileSync)(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
|
|
2525
2656
|
seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
|
|
2526
2657
|
if (imageAudit) {
|
|
2527
|
-
const imagesJsonl = (0,
|
|
2528
|
-
const imagesSummary = (0,
|
|
2529
|
-
(0,
|
|
2530
|
-
(0,
|
|
2658
|
+
const imagesJsonl = (0, import_node_path5.join)(dir, "images.jsonl");
|
|
2659
|
+
const imagesSummary = (0, import_node_path5.join)(dir, "images-summary.json");
|
|
2660
|
+
(0, import_node_fs4.writeFileSync)(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
|
|
2661
|
+
(0, import_node_fs4.writeFileSync)(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
|
|
2531
2662
|
seoFiles.push(imagesJsonl, imagesSummary);
|
|
2532
2663
|
}
|
|
2533
2664
|
if (seo.branding) {
|
|
2534
|
-
const brandingFile = (0,
|
|
2535
|
-
(0,
|
|
2665
|
+
const brandingFile = (0, import_node_path5.join)(dir, "branding.json");
|
|
2666
|
+
(0, import_node_fs4.writeFileSync)(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
|
|
2536
2667
|
seoFiles.push(brandingFile);
|
|
2537
2668
|
}
|
|
2538
2669
|
}
|
|
@@ -2545,12 +2676,12 @@ function saveUrlInventory(siteUrl, urls) {
|
|
|
2545
2676
|
if (!reportSavingActive()) return null;
|
|
2546
2677
|
try {
|
|
2547
2678
|
const outDir = outputBaseDir3();
|
|
2548
|
-
(0,
|
|
2679
|
+
(0, import_node_fs4.mkdirSync)(outDir, { recursive: true });
|
|
2549
2680
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
2550
|
-
const file = (0,
|
|
2681
|
+
const file = (0, import_node_path5.join)(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
|
|
2551
2682
|
const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
|
|
2552
2683
|
const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
|
|
2553
|
-
(0,
|
|
2684
|
+
(0, import_node_fs4.writeFileSync)(file, rows.join("\n"), "utf8");
|
|
2554
2685
|
return file;
|
|
2555
2686
|
} catch {
|
|
2556
2687
|
return null;
|
|
@@ -2559,12 +2690,12 @@ function saveUrlInventory(siteUrl, urls) {
|
|
|
2559
2690
|
function persistScreenshotLocally(base64, url) {
|
|
2560
2691
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
2561
2692
|
try {
|
|
2562
|
-
const dir = (0,
|
|
2563
|
-
(0,
|
|
2693
|
+
const dir = (0, import_node_path5.join)(outputBaseDir3(), "screenshots");
|
|
2694
|
+
(0, import_node_fs4.mkdirSync)(dir, { recursive: true });
|
|
2564
2695
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
2565
2696
|
const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
2566
|
-
const filePath = (0,
|
|
2567
|
-
(0,
|
|
2697
|
+
const filePath = (0, import_node_path5.join)(dir, `${stamp}-${slug}.png`);
|
|
2698
|
+
(0, import_node_fs4.writeFileSync)(filePath, Buffer.from(base64, "base64"));
|
|
2568
2699
|
return filePath;
|
|
2569
2700
|
} catch {
|
|
2570
2701
|
return null;
|
|
@@ -2577,6 +2708,20 @@ function oneBlock(content, diskContent) {
|
|
|
2577
2708
|
\u{1F4C4} Saved: \`${filePath}\`` : content;
|
|
2578
2709
|
return { content: [{ type: "text", text }] };
|
|
2579
2710
|
}
|
|
2711
|
+
async function maybeOffload(toolName, ctx, fullText, summaryText, structuredContent) {
|
|
2712
|
+
if (!ctx?.hosted || !ARTIFACT_OFFLOAD_ENABLED) return null;
|
|
2713
|
+
const bytes = Buffer.byteLength(fullText);
|
|
2714
|
+
if (bytes <= INLINE_BUDGET_BYTES) return null;
|
|
2715
|
+
const offloaded = await offloadReport(toolName, ctx.ownerId, fullText);
|
|
2716
|
+
return {
|
|
2717
|
+
content: [{ type: "text", text: summaryEnvelope(summaryText, offloaded) }],
|
|
2718
|
+
structuredContent: { ...structuredContent, artifact: offloaded }
|
|
2719
|
+
};
|
|
2720
|
+
}
|
|
2721
|
+
function capArray(items, cap) {
|
|
2722
|
+
if (items.length <= cap) return { items };
|
|
2723
|
+
return { items: items.slice(0, cap), truncatedCount: items.length - cap };
|
|
2724
|
+
}
|
|
2580
2725
|
function workflowRecipeTable(recipes) {
|
|
2581
2726
|
if (!recipes.length) return "";
|
|
2582
2727
|
return [
|
|
@@ -2928,7 +3073,7 @@ ${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${memSection}$
|
|
|
2928
3073
|
}
|
|
2929
3074
|
return { ...textResult, structuredContent };
|
|
2930
3075
|
}
|
|
2931
|
-
function formatMapSiteUrls(raw, input) {
|
|
3076
|
+
async function formatMapSiteUrls(raw, input, ctx) {
|
|
2932
3077
|
const parsed = parseData(raw);
|
|
2933
3078
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
2934
3079
|
const d = parsed.data;
|
|
@@ -2969,20 +3114,28 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
|
|
|
2969
3114
|
- Extract content from all pages: use \`extract_site\`
|
|
2970
3115
|
- Scrape a single page: use \`extract_url\``
|
|
2971
3116
|
].filter(Boolean).join("\n");
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
2983
|
-
durationMs: d.durationMs ?? 0
|
|
2984
|
-
}
|
|
3117
|
+
const structuredContent = {
|
|
3118
|
+
startUrl: d.startUrl ?? input.url,
|
|
3119
|
+
totalFound: d.totalFound ?? urls.length,
|
|
3120
|
+
truncated: d.truncated === true,
|
|
3121
|
+
okCount: ok.length,
|
|
3122
|
+
redirectCount: redirects.length,
|
|
3123
|
+
brokenCount: broken.length,
|
|
3124
|
+
inventoryFile: inventoryFile ?? null,
|
|
3125
|
+
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
3126
|
+
durationMs: d.durationMs ?? 0
|
|
2985
3127
|
};
|
|
3128
|
+
const summary = `# URL Map: ${input.url}
|
|
3129
|
+
**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s
|
|
3130
|
+
|
|
3131
|
+
## Summary
|
|
3132
|
+
- 2xx: ${ok.length}
|
|
3133
|
+
- 3xx: ${redirects.length}
|
|
3134
|
+
- 4xx+: ${broken.length}`;
|
|
3135
|
+
const capped = capArray(structuredContent.urls, STRUCTURED_ARRAY_CAP);
|
|
3136
|
+
const offloaded = await maybeOffload("map_site_urls", ctx, full, summary, { ...structuredContent, urls: capped.items, truncatedCount: capped.truncatedCount });
|
|
3137
|
+
if (offloaded) return offloaded;
|
|
3138
|
+
return { ...oneBlock(full), structuredContent };
|
|
2986
3139
|
}
|
|
2987
3140
|
function buildSeoExport(siteUrl, pages, branding) {
|
|
2988
3141
|
const { edges, metrics } = buildLinkGraph(pages, siteUrl);
|
|
@@ -3002,7 +3155,7 @@ function buildSeoExport(siteUrl, pages, branding) {
|
|
|
3002
3155
|
branding: branding ?? void 0
|
|
3003
3156
|
};
|
|
3004
3157
|
}
|
|
3005
|
-
function formatExtractSite(raw, input) {
|
|
3158
|
+
async function formatExtractSite(raw, input, ctx) {
|
|
3006
3159
|
const parsed = parseData(raw);
|
|
3007
3160
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
3008
3161
|
const d = parsed.data;
|
|
@@ -3037,6 +3190,36 @@ function formatExtractSite(raw, input) {
|
|
|
3037
3190
|
schemaTypes: schemaTypesOf(p)
|
|
3038
3191
|
})));
|
|
3039
3192
|
const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
|
|
3193
|
+
const summaryHeader = [
|
|
3194
|
+
`# Site Extract: ${input.url}`,
|
|
3195
|
+
durationLine,
|
|
3196
|
+
`
|
|
3197
|
+
## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
|
|
3198
|
+
| # | Title | URL | Schema |
|
|
3199
|
+
|---|-------|-----|--------|
|
|
3200
|
+
${preview}`
|
|
3201
|
+
].join("\n");
|
|
3202
|
+
if (!bulk && ctx?.hosted) {
|
|
3203
|
+
const pageDetails2 = pages.map((p, i) => {
|
|
3204
|
+
const body = (p.bodyMarkdown ?? "").trim();
|
|
3205
|
+
return [
|
|
3206
|
+
`
|
|
3207
|
+
## ${i + 1}. ${p.title ?? "Untitled"}`,
|
|
3208
|
+
`- **URL:** ${p.url}`,
|
|
3209
|
+
p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
|
|
3210
|
+
body ? `
|
|
3211
|
+
${body}` : "_(no content extracted)_"
|
|
3212
|
+
].filter(Boolean).join("\n");
|
|
3213
|
+
}).join("\n");
|
|
3214
|
+
const fullForOffload = `${summaryHeader}
|
|
3215
|
+
|
|
3216
|
+
---
|
|
3217
|
+
# Full Page Content
|
|
3218
|
+
${pageDetails2}${tips}`;
|
|
3219
|
+
const capped = capArray(structuredContent.pages, STRUCTURED_ARRAY_CAP);
|
|
3220
|
+
const offloaded = await maybeOffload("extract_site", ctx, fullForOffload, `${summaryHeader}${tips}`, { ...structuredContent, pages: capped.items, truncatedCount: capped.truncatedCount });
|
|
3221
|
+
if (offloaded) return offloaded;
|
|
3222
|
+
}
|
|
3040
3223
|
const location = bulk ? `
|
|
3041
3224
|
## \u{1F4C1} Bulk scrape saved
|
|
3042
3225
|
- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
|
|
@@ -3089,7 +3272,7 @@ ${body}` : "_(no content extracted)_"
|
|
|
3089
3272
|
${pageDetails}${tips}`;
|
|
3090
3273
|
return { ...oneBlock(full, diskReport), structuredContent };
|
|
3091
3274
|
}
|
|
3092
|
-
async function formatAuditSite(raw, input) {
|
|
3275
|
+
async function formatAuditSite(raw, input, ctx) {
|
|
3093
3276
|
const parsed = parseData(raw);
|
|
3094
3277
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
3095
3278
|
const d = parsed.data;
|
|
@@ -3120,6 +3303,24 @@ async function formatAuditSite(raw, input) {
|
|
|
3120
3303
|
images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
|
|
3121
3304
|
links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
|
|
3122
3305
|
};
|
|
3306
|
+
const summary = [
|
|
3307
|
+
`# Technical SEO Audit: ${input.url}`,
|
|
3308
|
+
durationLine,
|
|
3309
|
+
`
|
|
3310
|
+
## Top issues
|
|
3311
|
+
${topIssues || "_none found_"}`,
|
|
3312
|
+
`
|
|
3313
|
+
## Links
|
|
3314
|
+
${linkLine}`,
|
|
3315
|
+
`
|
|
3316
|
+
## Images
|
|
3317
|
+
${imgLine}`
|
|
3318
|
+
].join("\n");
|
|
3319
|
+
if (!bulk && ctx?.hosted) {
|
|
3320
|
+
const fullForOffload = [summary, "\n---\n# Full Audit Report", seo.reportMd, renderImageSection(imageAudit), renderLinkReport(seo.linkReport)].join("\n");
|
|
3321
|
+
const offloaded = await maybeOffload("audit_site", ctx, fullForOffload, summary, structuredContent);
|
|
3322
|
+
if (offloaded) return offloaded;
|
|
3323
|
+
}
|
|
3123
3324
|
const location = bulk ? `
|
|
3124
3325
|
## \u{1F4C1} Technical audit saved
|
|
3125
3326
|
- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
|
|
@@ -3890,7 +4091,7 @@ ${rows}`,
|
|
|
3890
4091
|
}
|
|
3891
4092
|
};
|
|
3892
4093
|
}
|
|
3893
|
-
function formatDirectoryWorkflow(raw, input) {
|
|
4094
|
+
async function formatDirectoryWorkflow(raw, input, ctx) {
|
|
3894
4095
|
const parsed = parseData(raw);
|
|
3895
4096
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
3896
4097
|
const d = parsed.data;
|
|
@@ -3942,26 +4143,29 @@ ${businessRows}` : null,
|
|
|
3942
4143
|
durationMs != null ? `
|
|
3943
4144
|
*Completed in ${(durationMs / 1e3).toFixed(1)}s*` : null
|
|
3944
4145
|
].filter(Boolean).join("\n");
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
cities,
|
|
3962
|
-
durationMs: durationMs ?? 0
|
|
3963
|
-
}
|
|
4146
|
+
const structuredContent = {
|
|
4147
|
+
query: d.query,
|
|
4148
|
+
state: d.state,
|
|
4149
|
+
minPopulation: d.minPopulation,
|
|
4150
|
+
populationYear: d.populationYear,
|
|
4151
|
+
maxResultsPerCity: d.maxResultsPerCity,
|
|
4152
|
+
concurrency: d.concurrency,
|
|
4153
|
+
censusSourceUrl: d.censusSourceUrl,
|
|
4154
|
+
usZipsSourcePath: d.usZipsSourcePath ?? null,
|
|
4155
|
+
warnings,
|
|
4156
|
+
extractedAt: d.extractedAt,
|
|
4157
|
+
selectedCityCount: d.selectedCityCount,
|
|
4158
|
+
totalResultCount,
|
|
4159
|
+
csvPath,
|
|
4160
|
+
cities,
|
|
4161
|
+
durationMs: durationMs ?? 0
|
|
3964
4162
|
};
|
|
4163
|
+
const summary = `# Directory Workflow: ${input.query}
|
|
4164
|
+
**Markets:** ${cities.length} \xB7 **Maps results:** ${totalResultCount} \xB7 **State:** ${d.state ?? input.state ?? "US"}`;
|
|
4165
|
+
const capped = capArray(cities, STRUCTURED_ARRAY_CAP);
|
|
4166
|
+
const offloaded = await maybeOffload("directory_workflow", ctx, full, summary, { ...structuredContent, cities: capped.items, truncatedCount: capped.truncatedCount });
|
|
4167
|
+
if (offloaded) return offloaded;
|
|
4168
|
+
return { ...oneBlock(full), structuredContent };
|
|
3965
4169
|
}
|
|
3966
4170
|
function formatMapsPlaceIntel(raw, input) {
|
|
3967
4171
|
const parsed = parseData(raw);
|
|
@@ -4483,9 +4687,13 @@ ${rows}` : ""
|
|
|
4483
4687
|
}
|
|
4484
4688
|
|
|
4485
4689
|
// src/mcp/server-instructions.ts
|
|
4486
|
-
|
|
4690
|
+
function serverInstructions(savesReportsLocally) {
|
|
4691
|
+
const reportLine = savesReportsLocally ? "All report-producing tools also save a full Markdown report to disk." : "On this hosted endpoint, small reports return inline; large reports are stored as artifacts \u2014 read them back with report_artifact_read.";
|
|
4692
|
+
return `
|
|
4487
4693
|
# MCP Scraper
|
|
4488
4694
|
|
|
4695
|
+
${reportLine}
|
|
4696
|
+
|
|
4489
4697
|
Scrapes and analyzes web, search, maps, social, and site data. **Pick a tool by NAME from the map
|
|
4490
4698
|
below, then load its schema before calling.** Where a tool needs a value another tool produces, the
|
|
4491
4699
|
seam is noted so you can chain them.
|
|
@@ -4569,9 +4777,11 @@ Multi-step orchestrations \u2014 prefer these over hand-chaining primitives when
|
|
|
4569
4777
|
sites. It returns a saved folder/artifact plus a summary, not the full content inline.
|
|
4570
4778
|
- Browser sessions and media transcription cost credits and take longer \u2014 prefer the cheapest tool that
|
|
4571
4779
|
answers the question (plain search/extract before browser agents).
|
|
4572
|
-
- Large results are saved to disk or an artifact and returned as a summary plus a path
|
|
4573
|
-
full detail rather than expecting the whole payload inline.
|
|
4780
|
+
- Large results are saved to disk or an artifact and returned as a summary plus a path or artifactId;
|
|
4781
|
+
read it back for full detail rather than expecting the whole payload inline.
|
|
4574
4782
|
`.trim();
|
|
4783
|
+
}
|
|
4784
|
+
var SERVER_INSTRUCTIONS = serverInstructions(true);
|
|
4575
4785
|
|
|
4576
4786
|
// src/mcp/mcp-tool-schemas.ts
|
|
4577
4787
|
var import_zod3 = require("zod");
|
|
@@ -4744,7 +4954,7 @@ var RedditThreadInputSchema = {
|
|
|
4744
4954
|
var VideoFrameAnalysisInputSchema = {
|
|
4745
4955
|
sourceUrl: import_zod3.z.string().min(1).describe("A DIRECT video file URL (.mp4/.webm/.mov). Not a YouTube/Facebook/Instagram page URL \u2014 resolve those to a direct media URL first. Videos up to 30 minutes are supported."),
|
|
4746
4956
|
intervalS: import_zod3.z.number().min(1).max(30).optional().describe("Preferred seconds between sampled frames (1-30, default 2). Automatically widened for long videos so the whole duration is covered within the frame budget."),
|
|
4747
|
-
maxFrames: import_zod3.z.number().int().min(1).max(
|
|
4957
|
+
maxFrames: import_zod3.z.number().int().min(1).max(480).optional().describe("Max frames analyzed (<=480, default 120). $1 per 120 frames requested \u2014 120=$1 \u2026 480=$4 \u2014 automatically refunded down if the video cannot use them (minimum 1s between frames). Frames are spread evenly across the whole video."),
|
|
4748
4958
|
detail: import_zod3.z.enum(["fast", "standard", "deep"]).optional().describe("Analysis depth. Default standard."),
|
|
4749
4959
|
vault: import_zod3.z.string().min(1).optional().describe('Memory vault to save the finished breakdown into. Default "Library".')
|
|
4750
4960
|
};
|
|
@@ -4819,12 +5029,18 @@ var DirectoryWorkflowInputSchema = {
|
|
|
4819
5029
|
maxResultsPerCity: import_zod3.z.number().int().min(1).max(50).default(50).describe("Google Maps candidates to collect per city."),
|
|
4820
5030
|
concurrency: import_zod3.z.number().int().min(1).max(5).default(5).describe("City Maps searches to run in parallel."),
|
|
4821
5031
|
includeZipGroups: import_zod3.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available (MCP_SCRAPER_USZIPS_CSV_PATH or usZipsCsvPath)."),
|
|
4822
|
-
usZipsCsvPath: import_zod3.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
5032
|
+
usZipsCsvPath: import_zod3.z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead. For ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the server, or pass this in local/test mode."),
|
|
4823
5033
|
saveCsv: import_zod3.z.boolean().default(true).describe("Save a directory-ready CSV of results to the MCP Scraper output directory and return its path."),
|
|
4824
5034
|
proxyMode: import_zod3.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting per city search. Defaults to location (city/ZIP-group residential targeting); configured forces the service proxy; none is local debugging only."),
|
|
4825
5035
|
proxyZip: import_zod3.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting; normally omitted."),
|
|
4826
5036
|
debug: import_zod3.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
4827
5037
|
};
|
|
5038
|
+
var ArtifactPointerOutputSchema = import_zod3.z.object({
|
|
5039
|
+
artifactId: import_zod3.z.string(),
|
|
5040
|
+
bytes: import_zod3.z.number().int().min(0),
|
|
5041
|
+
expiresAt: import_zod3.z.string(),
|
|
5042
|
+
preview: import_zod3.z.string()
|
|
5043
|
+
});
|
|
4828
5044
|
var RankTrackerModeSchema = import_zod3.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
4829
5045
|
var RankTrackerBlueprintInputSchema = {
|
|
4830
5046
|
projectName: import_zod3.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
@@ -4938,7 +5154,9 @@ var DirectoryWorkflowOutputSchema = {
|
|
|
4938
5154
|
attempts: import_zod3.z.array(MapsSearchAttemptOutput),
|
|
4939
5155
|
results: import_zod3.z.array(DirectoryMapsBusinessOutput)
|
|
4940
5156
|
})),
|
|
4941
|
-
durationMs: import_zod3.z.number().int().min(0)
|
|
5157
|
+
durationMs: import_zod3.z.number().int().min(0),
|
|
5158
|
+
truncatedCount: import_zod3.z.number().int().min(0).optional(),
|
|
5159
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
4942
5160
|
};
|
|
4943
5161
|
var RankTrackerToolPlanOutput = import_zod3.z.object({
|
|
4944
5162
|
tool: import_zod3.z.string(),
|
|
@@ -5058,7 +5276,9 @@ var ExtractSiteOutputSchema = {
|
|
|
5058
5276
|
title: NullableString2,
|
|
5059
5277
|
schemaTypes: import_zod3.z.array(import_zod3.z.string())
|
|
5060
5278
|
})),
|
|
5061
|
-
durationMs: import_zod3.z.number().min(0)
|
|
5279
|
+
durationMs: import_zod3.z.number().min(0),
|
|
5280
|
+
truncatedCount: import_zod3.z.number().int().min(0).optional(),
|
|
5281
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
5062
5282
|
};
|
|
5063
5283
|
var AuditSiteOutputSchema = {
|
|
5064
5284
|
url: import_zod3.z.string(),
|
|
@@ -5078,7 +5298,8 @@ var AuditSiteOutputSchema = {
|
|
|
5078
5298
|
orphans: import_zod3.z.number().int().min(0),
|
|
5079
5299
|
brokenInternal: import_zod3.z.number().int().min(0),
|
|
5080
5300
|
externalDomains: import_zod3.z.number().int().min(0)
|
|
5081
|
-
})
|
|
5301
|
+
}),
|
|
5302
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
5082
5303
|
};
|
|
5083
5304
|
var MapsPlaceIntelOutputSchema = {
|
|
5084
5305
|
name: import_zod3.z.string(),
|
|
@@ -5150,7 +5371,9 @@ var MapSiteUrlsOutputSchema = {
|
|
|
5150
5371
|
url: import_zod3.z.string(),
|
|
5151
5372
|
status: import_zod3.z.number().int().nullable()
|
|
5152
5373
|
})),
|
|
5153
|
-
durationMs: import_zod3.z.number().min(0)
|
|
5374
|
+
durationMs: import_zod3.z.number().min(0),
|
|
5375
|
+
truncatedCount: import_zod3.z.number().int().min(0).optional(),
|
|
5376
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
5154
5377
|
};
|
|
5155
5378
|
var YoutubeHarvestOutputSchema = {
|
|
5156
5379
|
mode: import_zod3.z.string(),
|
|
@@ -5194,7 +5417,12 @@ var VideoFrameAnalysisStatusOutputSchema = {
|
|
|
5194
5417
|
frameCount: import_zod3.z.number().int().nullable().optional(),
|
|
5195
5418
|
artifactPath: NullableString2,
|
|
5196
5419
|
report: NullableString2,
|
|
5197
|
-
error: NullableString2
|
|
5420
|
+
error: NullableString2,
|
|
5421
|
+
reconciliation: import_zod3.z.object({
|
|
5422
|
+
billedMc: import_zod3.z.number().int(),
|
|
5423
|
+
refundedMc: import_zod3.z.number().int(),
|
|
5424
|
+
effectiveFrames: import_zod3.z.number().int().nullable()
|
|
5425
|
+
}).nullable().optional()
|
|
5198
5426
|
};
|
|
5199
5427
|
var RedditThreadOutputSchema = {
|
|
5200
5428
|
sourceUrl: NullableString2,
|
|
@@ -5582,6 +5810,17 @@ var CaptureSerpPageSnapshotsInputSchema = {
|
|
|
5582
5810
|
timeoutMs: import_zod3.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
5583
5811
|
debug: import_zod3.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
5584
5812
|
};
|
|
5813
|
+
var ReportArtifactReadInputSchema = {
|
|
5814
|
+
artifactId: import_zod3.z.string().min(1).describe("Artifact id returned inline by a tool whose result was too large to inline. Use only a returned artifactId; do not construct one yourself."),
|
|
5815
|
+
offset: import_zod3.z.number().int().min(0).default(0).describe("Byte offset to start reading from. Pass the previous call's nextOffset to continue."),
|
|
5816
|
+
maxBytes: import_zod3.z.number().int().min(1e3).max(1e5).default(2e4).describe("Maximum bytes of artifact text to return in this window.")
|
|
5817
|
+
};
|
|
5818
|
+
var ReportArtifactReadOutputSchema = {
|
|
5819
|
+
artifactId: import_zod3.z.string(),
|
|
5820
|
+
text: import_zod3.z.string(),
|
|
5821
|
+
totalBytes: import_zod3.z.number().int().min(0),
|
|
5822
|
+
nextOffset: import_zod3.z.number().int().min(0).nullable()
|
|
5823
|
+
};
|
|
5585
5824
|
|
|
5586
5825
|
// src/mcp/rank-tracker-blueprint.ts
|
|
5587
5826
|
var DEFAULT_MODES = ["maps", "organic", "ai_overview", "paa"];
|
|
@@ -5919,16 +6158,16 @@ function liveWebToolAnnotations(title) {
|
|
|
5919
6158
|
function registerSerpIntelligenceCaptureTools(server2, executor) {
|
|
5920
6159
|
server2.registerTool("capture_serp_snapshot", {
|
|
5921
6160
|
title: "SERP Intelligence Snapshot",
|
|
5922
|
-
description: "Capture a structured SERP Intelligence Google
|
|
6161
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
5923
6162
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
5924
|
-
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
6163
|
+
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
5925
6164
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
5926
6165
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
5927
6166
|
server2.registerTool("capture_serp_page_snapshots", {
|
|
5928
6167
|
title: "SERP Intelligence Page Snapshots",
|
|
5929
|
-
description: "Capture public ranking
|
|
6168
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
|
|
5930
6169
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
5931
|
-
outputSchema: CaptureSerpPageSnapshotsOutputSchema,
|
|
6170
|
+
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
5932
6171
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
5933
6172
|
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
5934
6173
|
}
|
|
@@ -5944,7 +6183,7 @@ function localPlanningToolAnnotations(title) {
|
|
|
5944
6183
|
function listSavedReports() {
|
|
5945
6184
|
try {
|
|
5946
6185
|
const dir = outputBaseDir3();
|
|
5947
|
-
return (0,
|
|
6186
|
+
return (0, import_node_fs5.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs5.statSync)((0, import_node_path6.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
5948
6187
|
} catch {
|
|
5949
6188
|
return [];
|
|
5950
6189
|
}
|
|
@@ -5968,233 +6207,254 @@ function registerSavedReportResources(server2) {
|
|
|
5968
6207
|
},
|
|
5969
6208
|
async (uri, variables) => {
|
|
5970
6209
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
5971
|
-
const filename = (0,
|
|
6210
|
+
const filename = (0, import_node_path6.basename)(decodeURIComponent(String(requested ?? "")));
|
|
5972
6211
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
5973
|
-
const text = (0,
|
|
6212
|
+
const text = (0, import_node_fs5.readFileSync)((0, import_node_path6.join)(outputBaseDir3(), filename), "utf8");
|
|
5974
6213
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
5975
6214
|
}
|
|
5976
6215
|
);
|
|
5977
6216
|
}
|
|
5978
6217
|
function registerPaaExtractorMcpTools(server2, executor, options = {}) {
|
|
5979
6218
|
const savesReports = options.savesReportsLocally !== false;
|
|
5980
|
-
const
|
|
5981
|
-
const
|
|
6219
|
+
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
6220
|
+
const ownerId = options.ownerId ?? "local";
|
|
6221
|
+
const ctx = { hosted: !savesReports, ownerId };
|
|
5982
6222
|
if (savesReports) registerSavedReportResources(server2);
|
|
5983
6223
|
server2.registerTool("harvest_paa", {
|
|
5984
6224
|
title: "Google PAA + SERP Harvest",
|
|
5985
|
-
description:
|
|
6225
|
+
description: "Best default tool for Google search research: People Also Ask questions with answers/sources, organic SERP, local pack, entity IDs, and AI Overview. Split topic from location; leave proxyMode unset. Warn the user before maxQuestions above 100 \u2014 deep harvests can run several minutes with no interim progress, billed per extracted question.",
|
|
5986
6226
|
inputSchema: HarvestPaaInputSchema,
|
|
5987
|
-
outputSchema: HarvestPaaOutputSchema,
|
|
6227
|
+
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
5988
6228
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
5989
6229
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
5990
6230
|
server2.registerTool("search_serp", {
|
|
5991
6231
|
title: "Google SERP Lookup",
|
|
5992
|
-
description:
|
|
6232
|
+
description: "Fast Google SERP lookup without PAA expansion \u2014 rankings, organic results, local pack, positions. Split topic from location; leave proxyMode unset.",
|
|
5993
6233
|
inputSchema: SearchSerpInputSchema,
|
|
5994
|
-
outputSchema: SearchSerpOutputSchema,
|
|
6234
|
+
outputSchema: recordOutputSchema("search_serp", SearchSerpOutputSchema),
|
|
5995
6235
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
5996
6236
|
}, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
|
|
5997
6237
|
server2.registerTool("extract_url", {
|
|
5998
6238
|
title: "Single URL Extract",
|
|
5999
|
-
description:
|
|
6239
|
+
description: "Extract structured data from one public URL: content, schema, headings, metadata, screenshots, branding, or media assets. Set depositToVault:true to save the full page into the user's MCP Memory vault server-side (not returned to chat).",
|
|
6000
6240
|
inputSchema: ExtractUrlInputSchema,
|
|
6001
|
-
outputSchema: ExtractUrlOutputSchema,
|
|
6241
|
+
outputSchema: recordOutputSchema("extract_url", ExtractUrlOutputSchema),
|
|
6002
6242
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
6003
6243
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
6004
6244
|
server2.registerTool("map_site_urls", {
|
|
6005
6245
|
title: "Site URL Map",
|
|
6006
|
-
description:
|
|
6246
|
+
description: `Map/crawl a public website for a sitemap, URL inventory, or broken-link scan. Returns internal URLs with HTTP status; ${fileBehavior("maps over 500 URLs are written to a local CSV file instead of inlined.", "large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
6007
6247
|
inputSchema: MapSiteUrlsInputSchema,
|
|
6008
|
-
outputSchema: MapSiteUrlsOutputSchema,
|
|
6248
|
+
outputSchema: recordOutputSchema("map_site_urls", MapSiteUrlsOutputSchema),
|
|
6009
6249
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
6010
|
-
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
6250
|
+
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input, ctx));
|
|
6011
6251
|
server2.registerTool("extract_site", {
|
|
6012
6252
|
title: "Multi-Page Site Content Crawl",
|
|
6013
|
-
description:
|
|
6253
|
+
description: `Crawl a public website and return page CONTENT (Markdown) across multiple pages. ${fileBehavior("Bulk crawls over 25 pages are saved as per-page Markdown files in a local folder instead of inlined.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Content only \u2014 for a technical SEO audit use audit_site instead.`,
|
|
6014
6254
|
inputSchema: ExtractSiteInputSchema,
|
|
6015
|
-
outputSchema: ExtractSiteOutputSchema,
|
|
6255
|
+
outputSchema: recordOutputSchema("extract_site", ExtractSiteOutputSchema),
|
|
6016
6256
|
annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
|
|
6017
|
-
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
6257
|
+
}, async (input) => formatExtractSite(await executor.extractSite(input), input, ctx));
|
|
6018
6258
|
server2.registerTool("audit_site", {
|
|
6019
6259
|
title: "Technical SEO Audit",
|
|
6020
|
-
description:
|
|
6260
|
+
description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. ${fileBehavior("Writes a folder of analysis files plus per-page content, and returns a summary plus the folder path.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Use extract_site instead for plain page content.`,
|
|
6021
6261
|
inputSchema: AuditSiteInputSchema,
|
|
6022
|
-
outputSchema: AuditSiteOutputSchema,
|
|
6262
|
+
outputSchema: recordOutputSchema("audit_site", AuditSiteOutputSchema),
|
|
6023
6263
|
annotations: liveWebToolAnnotations("Technical SEO Audit")
|
|
6024
|
-
}, async (input) => formatAuditSite(await executor.auditSite(input), input));
|
|
6264
|
+
}, async (input) => formatAuditSite(await executor.auditSite(input), input, ctx));
|
|
6025
6265
|
server2.registerTool("youtube_harvest", {
|
|
6026
6266
|
title: "YouTube Video Harvest",
|
|
6027
|
-
description:
|
|
6267
|
+
description: 'Harvest YouTube video metadata by topic search or channel library. Use mode "search" for keyword/topic requests, mode "channel" for @handles/channel IDs/URLs. Returns titles, views, durations, and videoIds.',
|
|
6028
6268
|
inputSchema: YoutubeHarvestInputSchema,
|
|
6029
|
-
outputSchema: YoutubeHarvestOutputSchema,
|
|
6269
|
+
outputSchema: recordOutputSchema("youtube_harvest", YoutubeHarvestOutputSchema),
|
|
6030
6270
|
annotations: liveWebToolAnnotations("YouTube Video Harvest")
|
|
6031
6271
|
}, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
|
|
6032
6272
|
server2.registerTool("youtube_transcribe", {
|
|
6033
6273
|
title: "YouTube Transcription",
|
|
6034
|
-
description:
|
|
6274
|
+
description: "Fetch and transcribe captions from a YouTube video. Pass videoId from youtube_harvest, or a url the user pasted. Returns full transcript, timestamped chunks, and word count.",
|
|
6035
6275
|
inputSchema: YoutubeTranscribeInputSchema,
|
|
6036
|
-
outputSchema: YoutubeTranscribeOutputSchema,
|
|
6276
|
+
outputSchema: recordOutputSchema("youtube_transcribe", YoutubeTranscribeOutputSchema),
|
|
6037
6277
|
annotations: liveWebToolAnnotations("YouTube Transcription")
|
|
6038
6278
|
}, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
|
|
6039
6279
|
server2.registerTool("facebook_page_intel", {
|
|
6040
6280
|
title: "Facebook Advertiser Ad Intel",
|
|
6041
|
-
description:
|
|
6281
|
+
description: "Harvest ads from a Facebook advertiser: copy, creative angles, CTAs, and direct video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name.",
|
|
6042
6282
|
inputSchema: FacebookPageIntelInputSchema,
|
|
6043
|
-
outputSchema: FacebookPageIntelOutputSchema,
|
|
6283
|
+
outputSchema: recordOutputSchema("facebook_page_intel", FacebookPageIntelOutputSchema),
|
|
6044
6284
|
annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
|
|
6045
6285
|
}, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
|
|
6046
6286
|
server2.registerTool("facebook_ad_search", {
|
|
6047
6287
|
title: "Facebook Ad Library Search",
|
|
6048
|
-
description:
|
|
6288
|
+
description: "Search Facebook Ad Library to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs.",
|
|
6049
6289
|
inputSchema: FacebookAdSearchInputSchema,
|
|
6050
|
-
outputSchema: FacebookAdSearchOutputSchema,
|
|
6290
|
+
outputSchema: recordOutputSchema("facebook_ad_search", FacebookAdSearchOutputSchema),
|
|
6051
6291
|
annotations: liveWebToolAnnotations("Facebook Ad Library Search")
|
|
6052
6292
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
6053
6293
|
server2.registerTool("reddit_thread", {
|
|
6054
6294
|
title: "Reddit Thread + Comments",
|
|
6055
|
-
description:
|
|
6295
|
+
description: "Capture a Reddit post and its comment tree from a reddit.com thread URL \u2014 comments, opinions, audience voice. Handles Reddit's bot protection automatically; pass maxComments to cap the list.",
|
|
6056
6296
|
inputSchema: RedditThreadInputSchema,
|
|
6057
|
-
outputSchema: RedditThreadOutputSchema,
|
|
6297
|
+
outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
|
|
6058
6298
|
annotations: liveWebToolAnnotations("Reddit Thread + Comments")
|
|
6059
6299
|
}, async (input) => formatRedditThread(await executor.redditThread(input), input));
|
|
6060
6300
|
server2.registerTool("video_frame_analysis", {
|
|
6061
6301
|
title: "Video Breakdown (frame-by-frame + transcript)",
|
|
6062
|
-
description: "Produce a deep frame-by-frame + transcript breakdown of a video \u2014 pacing, hook, visual style, and how to replicate it. Pass a DIRECT video file URL (.mp4/.webm/.mov), not a page URL.
|
|
6302
|
+
description: "Produce a deep frame-by-frame + transcript breakdown of a video \u2014 pacing, hook, visual style, and how to replicate it. Pass a DIRECT video file URL (.mp4/.webm/.mov), not a page URL. Costs $1 per 120 frames requested (max 480 = $4; refunded down if the video can't use them; refunded fully on failure): returns a runId immediately; poll video_frame_analysis_status until done. Videos up to 30 minutes.",
|
|
6063
6303
|
inputSchema: VideoFrameAnalysisInputSchema,
|
|
6064
|
-
outputSchema: VideoFrameAnalysisOutputSchema,
|
|
6304
|
+
outputSchema: recordOutputSchema("video_frame_analysis", VideoFrameAnalysisOutputSchema),
|
|
6065
6305
|
annotations: { title: "Video Breakdown", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
|
|
6066
6306
|
}, async (input) => executor.videoFrameAnalysis(input));
|
|
6067
6307
|
server2.registerTool("video_frame_analysis_status", {
|
|
6068
6308
|
title: "Video Breakdown Status",
|
|
6069
|
-
description: 'Check progress of a video breakdown started with video_frame_analysis, using its runId. Free to call. When status is "done" it returns the full report and vault path; stop polling on "done" or "failed".',
|
|
6309
|
+
description: 'Check progress of a video breakdown started with video_frame_analysis, using its runId. Free to call. When status is "done" it returns the full report and vault path; stop polling on "done" or "failed". Reports the billed tier reconciliation when done.',
|
|
6070
6310
|
inputSchema: VideoFrameAnalysisStatusInputSchema,
|
|
6071
|
-
outputSchema: VideoFrameAnalysisStatusOutputSchema,
|
|
6311
|
+
outputSchema: recordOutputSchema("video_frame_analysis_status", VideoFrameAnalysisStatusOutputSchema),
|
|
6072
6312
|
annotations: { title: "Video Breakdown Status", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
6073
6313
|
}, async (input) => executor.videoFrameAnalysisStatus(input));
|
|
6074
6314
|
server2.registerTool("facebook_ad_transcribe", {
|
|
6075
6315
|
title: "Facebook Ad Transcription",
|
|
6076
6316
|
description: "Transcribe audio from a Facebook ad video CDN URL returned by facebook_page_intel. Use only that direct videoUrl value \u2014 do not pass public Facebook post/reel/share URLs (use facebook_video_transcribe for those).",
|
|
6077
6317
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
6078
|
-
outputSchema: FacebookAdTranscribeOutputSchema,
|
|
6318
|
+
outputSchema: recordOutputSchema("facebook_ad_transcribe", FacebookAdTranscribeOutputSchema),
|
|
6079
6319
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
6080
6320
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
6081
6321
|
server2.registerTool("google_ads_search", {
|
|
6082
6322
|
title: "Google Ads Transparency Search",
|
|
6083
|
-
description:
|
|
6323
|
+
description: "Search the Google Ads Transparency Center to find advertisers by domain or brand name. Returns advertisers with advertiser ID and approximate ad count, to pass to google_ads_page_intel.",
|
|
6084
6324
|
inputSchema: GoogleAdsSearchInputSchema,
|
|
6085
|
-
outputSchema: GoogleAdsSearchOutputSchema,
|
|
6325
|
+
outputSchema: recordOutputSchema("google_ads_search", GoogleAdsSearchOutputSchema),
|
|
6086
6326
|
annotations: liveWebToolAnnotations("Google Ads Transparency Search")
|
|
6087
6327
|
}, async (input) => formatGoogleAdsSearch(await executor.googleAdsSearch(input), input));
|
|
6088
6328
|
server2.registerTool("google_ads_page_intel", {
|
|
6089
6329
|
title: "Google Ads Advertiser Intel",
|
|
6090
|
-
description:
|
|
6330
|
+
description: "Harvest an advertiser's ad creatives from the Google Ads Transparency Center: format, image URLs, and \u2014 for video ads \u2014 a YouTube video ID or direct video URL. Accepts advertiserId (from google_ads_search) or a domain.",
|
|
6091
6331
|
inputSchema: GoogleAdsPageIntelInputSchema,
|
|
6092
|
-
outputSchema: GoogleAdsPageIntelOutputSchema,
|
|
6332
|
+
outputSchema: recordOutputSchema("google_ads_page_intel", GoogleAdsPageIntelOutputSchema),
|
|
6093
6333
|
annotations: liveWebToolAnnotations("Google Ads Advertiser Intel")
|
|
6094
6334
|
}, async (input) => formatGoogleAdsPageIntel(await executor.googleAdsPageIntel(input), input));
|
|
6095
6335
|
server2.registerTool("google_ads_transcribe", {
|
|
6096
6336
|
title: "Google Ad Video Transcription",
|
|
6097
6337
|
description: "Transcribe audio from a Google video ad's direct videoUrl (a googlevideo.com playback URL) returned by google_ads_page_intel. For YouTube-hosted ads, use youtube_transcribe with the returned youtubeVideoId instead.",
|
|
6098
6338
|
inputSchema: GoogleAdsTranscribeInputSchema,
|
|
6099
|
-
outputSchema: GoogleAdsTranscribeOutputSchema,
|
|
6339
|
+
outputSchema: recordOutputSchema("google_ads_transcribe", GoogleAdsTranscribeOutputSchema),
|
|
6100
6340
|
annotations: liveWebToolAnnotations("Google Ad Video Transcription")
|
|
6101
6341
|
}, async (input) => formatGoogleAdsTranscribe(await executor.googleAdsTranscribe(input), input));
|
|
6102
6342
|
server2.registerTool("facebook_video_transcribe", {
|
|
6103
6343
|
title: "Facebook Organic Video Transcription",
|
|
6104
|
-
description:
|
|
6344
|
+
description: "Transcribe audio from an organic Facebook reel/video/post/share URL (including fb.watch). Renders the page, selects the best public CDN MP4, and returns the transcript plus resolved video metadata.",
|
|
6105
6345
|
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
6106
|
-
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
6346
|
+
outputSchema: recordOutputSchema("facebook_video_transcribe", FacebookVideoTranscribeOutputSchema),
|
|
6107
6347
|
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
6108
6348
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
6109
6349
|
server2.registerTool("instagram_profile_content", {
|
|
6110
6350
|
title: "Instagram Profile Content Discovery",
|
|
6111
|
-
description:
|
|
6351
|
+
description: "Discover Instagram profile grid content links (posts/reels/tv) for a handle or profile URL, for later selection with instagram_media_download. Returns profile stats and collected URLs.",
|
|
6112
6352
|
inputSchema: InstagramProfileContentInputSchema,
|
|
6113
|
-
outputSchema: InstagramProfileContentOutputSchema,
|
|
6353
|
+
outputSchema: recordOutputSchema("instagram_profile_content", InstagramProfileContentOutputSchema),
|
|
6114
6354
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
6115
6355
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
6116
6356
|
server2.registerTool("instagram_media_download", {
|
|
6117
6357
|
title: "Instagram Post/Reel Media Download",
|
|
6118
|
-
description:
|
|
6358
|
+
description: "Extract and download media from one Instagram post, reel, or tv URL \u2014 image, caption, video/audio tracks, optional muxed MP4, or transcript. Selects the best video/audio track pair and muxes when ffmpeg is available.",
|
|
6119
6359
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
6120
|
-
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
6360
|
+
outputSchema: recordOutputSchema("instagram_media_download", InstagramMediaDownloadOutputSchema),
|
|
6121
6361
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
6122
6362
|
}, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
|
|
6123
6363
|
server2.registerTool("maps_place_intel", {
|
|
6124
6364
|
title: "Google Maps Business Profile Details",
|
|
6125
|
-
description:
|
|
6365
|
+
description: "Extract Google Maps business intelligence for one known/named business: rating, reviews, category, address, phone, hours, entity IDs. Not for category searches or multi-business prospect lists \u2014 use maps_search for those. Split business name from location.",
|
|
6126
6366
|
inputSchema: MapsPlaceIntelInputSchema,
|
|
6127
|
-
outputSchema: MapsPlaceIntelOutputSchema,
|
|
6367
|
+
outputSchema: recordOutputSchema("maps_place_intel", MapsPlaceIntelOutputSchema),
|
|
6128
6368
|
annotations: liveWebToolAnnotations("Google Maps Business Profile Details")
|
|
6129
6369
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
6130
6370
|
server2.registerTool("maps_search", {
|
|
6131
6371
|
title: "Google Maps Business Search",
|
|
6132
|
-
description:
|
|
6372
|
+
description: "Search Google Maps for multiple businesses by category, niche, or local market \u2014 leads, prospects, competitors, or beyond the 3-pack. Leave proxyMode unset. Returns up to 50 candidates (default 10) with names, place URLs, CIDs, ratings.",
|
|
6133
6373
|
inputSchema: MapsSearchInputSchema,
|
|
6134
|
-
outputSchema: MapsSearchOutputSchema,
|
|
6374
|
+
outputSchema: recordOutputSchema("maps_search", MapsSearchOutputSchema),
|
|
6135
6375
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
6136
6376
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
6137
6377
|
server2.registerTool("directory_workflow", {
|
|
6138
6378
|
title: "Directory Workflow: Markets + Maps",
|
|
6139
|
-
description:
|
|
6379
|
+
description: `Build directory/prospecting datasets: selects US city markets from Census population data, optionally joins configured ZIP groups, then runs Google Maps business searches per city in parallel. Use for "all cities over 100k population in a state" or market+Maps workflows. ${fileBehavior("Saves a CSV of results per city.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
6140
6380
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
6141
|
-
outputSchema: DirectoryWorkflowOutputSchema,
|
|
6381
|
+
outputSchema: recordOutputSchema("directory_workflow", DirectoryWorkflowOutputSchema),
|
|
6142
6382
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
6143
|
-
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input));
|
|
6383
|
+
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input, ctx));
|
|
6144
6384
|
server2.registerTool("workflow_list", {
|
|
6145
6385
|
title: "Workflow Catalog",
|
|
6146
6386
|
description: "List MCP Scraper higher-level workflows and recipes \u2014 market analysis, ICP research, CRO audits, competitive positioning, content gap briefs, AI search visibility, and more. Returns runnable workflow ids plus tool-chain guidance.",
|
|
6147
6387
|
inputSchema: WorkflowListInputSchema,
|
|
6148
|
-
outputSchema: WorkflowListOutputSchema,
|
|
6388
|
+
outputSchema: recordOutputSchema("workflow_list", WorkflowListOutputSchema),
|
|
6149
6389
|
annotations: localPlanningToolAnnotations("Workflow Catalog")
|
|
6150
6390
|
}, async (input) => formatWorkflowList(await executor.workflowList(input), input));
|
|
6151
6391
|
server2.registerTool("workflow_suggest", {
|
|
6152
6392
|
title: "Workflow Intent Router",
|
|
6153
6393
|
description: "Route a high-level business/research goal (market analysis, ICP research, CRO audit, competitor comparison, content gap brief, AI search visibility, etc) to the right MCP Scraper workflow/tool chain. Free; tells you what to run next.",
|
|
6154
6394
|
inputSchema: WorkflowSuggestInputSchema,
|
|
6155
|
-
outputSchema: WorkflowSuggestOutputSchema,
|
|
6395
|
+
outputSchema: recordOutputSchema("workflow_suggest", WorkflowSuggestOutputSchema),
|
|
6156
6396
|
annotations: localPlanningToolAnnotations("Workflow Intent Router")
|
|
6157
6397
|
}, async (input) => formatWorkflowSuggest(input));
|
|
6158
6398
|
server2.registerTool("workflow_run", {
|
|
6159
6399
|
title: "Run Workflow",
|
|
6160
|
-
description:
|
|
6400
|
+
description: "Start a higher-level MCP Scraper workflow (directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language). Use after workflow_suggest or workflow_list. Stepwise workflows return runId + nextStep \u2014 call workflow_step with the runId until done is true.",
|
|
6161
6401
|
inputSchema: WorkflowRunInputSchema,
|
|
6162
|
-
outputSchema: WorkflowRunOutputSchema,
|
|
6402
|
+
outputSchema: recordOutputSchema("workflow_run", WorkflowRunOutputSchema),
|
|
6163
6403
|
annotations: liveWebToolAnnotations("Run Workflow")
|
|
6164
6404
|
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
6165
6405
|
server2.registerTool("workflow_step", {
|
|
6166
6406
|
title: "Advance Workflow Step",
|
|
6167
|
-
description:
|
|
6407
|
+
description: "Run the next leg of a stepwise workflow started with workflow_run. Pass the runId; each call executes one step and returns its output plus nextStep. Keep calling with the same runId until done is true.",
|
|
6168
6408
|
inputSchema: WorkflowStepInputSchema,
|
|
6169
|
-
outputSchema: WorkflowStepOutputSchema,
|
|
6409
|
+
outputSchema: recordOutputSchema("workflow_step", WorkflowStepOutputSchema),
|
|
6170
6410
|
annotations: liveWebToolAnnotations("Advance Workflow Step")
|
|
6171
6411
|
}, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
|
|
6172
6412
|
server2.registerTool("workflow_status", {
|
|
6173
6413
|
title: "Workflow Status",
|
|
6174
6414
|
description: "Fetch a hosted workflow run by id and list its current status and artifacts, to re-open a run or recover artifact ids. Use only a runId returned by workflow_run/workflow_step/workflow_status.",
|
|
6175
6415
|
inputSchema: WorkflowStatusInputSchema,
|
|
6176
|
-
outputSchema: WorkflowStatusOutputSchema,
|
|
6416
|
+
outputSchema: recordOutputSchema("workflow_status", WorkflowStatusOutputSchema),
|
|
6177
6417
|
annotations: liveWebToolAnnotations("Workflow Status")
|
|
6178
6418
|
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
6179
6419
|
server2.registerTool("workflow_artifact_read", {
|
|
6180
6420
|
title: "Read Workflow Artifact",
|
|
6181
6421
|
description: "Read a workflow artifact back into context by run id and artifact id, so final deliverables are grounded in generated evidence rather than memory. Use workflow_status first when artifact ids are unknown. Use maxBytes to limit large artifacts.",
|
|
6182
6422
|
inputSchema: WorkflowArtifactReadInputSchema,
|
|
6183
|
-
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
6423
|
+
outputSchema: recordOutputSchema("workflow_artifact_read", WorkflowArtifactReadOutputSchema),
|
|
6184
6424
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
6185
6425
|
}, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
|
|
6426
|
+
server2.registerTool("report_artifact_read", {
|
|
6427
|
+
title: "Read Report Artifact",
|
|
6428
|
+
description: "Read back a stored report artifact by artifactId (returned by any tool whose result was too large to inline). Windowed: pass offset/maxBytes and keep reading until nextOffset is null.",
|
|
6429
|
+
inputSchema: ReportArtifactReadInputSchema,
|
|
6430
|
+
outputSchema: recordOutputSchema("report_artifact_read", ReportArtifactReadOutputSchema),
|
|
6431
|
+
annotations: liveWebToolAnnotations("Read Report Artifact")
|
|
6432
|
+
}, async (input) => {
|
|
6433
|
+
const owner = artifactOwnerId(input.artifactId);
|
|
6434
|
+
if (!owner || owner !== ownerId) {
|
|
6435
|
+
return { content: [{ type: "text", text: "Artifact not found or not owned by this caller." }], isError: true };
|
|
6436
|
+
}
|
|
6437
|
+
const window = await readArtifactWindow(input.artifactId, input.offset ?? 0, input.maxBytes ?? 2e4);
|
|
6438
|
+
if (!window) {
|
|
6439
|
+
return { content: [{ type: "text", text: "Artifact not found or expired." }], isError: true };
|
|
6440
|
+
}
|
|
6441
|
+
return {
|
|
6442
|
+
content: [{ type: "text", text: window.text }],
|
|
6443
|
+
structuredContent: { artifactId: input.artifactId, text: window.text, totalBytes: window.totalBytes, nextOffset: window.nextOffset }
|
|
6444
|
+
};
|
|
6445
|
+
});
|
|
6186
6446
|
server2.registerTool("rank_tracker_workflow", {
|
|
6187
6447
|
title: "Rank Tracker Blueprint Builder",
|
|
6188
6448
|
description: "Generate a build-ready database schema, cron plan, and implementation prompt for a rank tracker powered by MCP Scraper (Maps, organic, AI Overview, or PAA tracking). Local planning only \u2014 does not call the web or spend credits.",
|
|
6189
6449
|
inputSchema: RankTrackerBlueprintInputSchema,
|
|
6190
|
-
outputSchema: RankTrackerBlueprintOutputSchema,
|
|
6450
|
+
outputSchema: recordOutputSchema("rank_tracker_workflow", RankTrackerBlueprintOutputSchema),
|
|
6191
6451
|
annotations: localPlanningToolAnnotations("Rank Tracker Blueprint Builder")
|
|
6192
6452
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
6193
6453
|
server2.registerTool("credits_info", {
|
|
6194
6454
|
title: "MCP Scraper Credits & Costs",
|
|
6195
6455
|
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades \u2014 balance, tool costs, concurrency limits, billing URL. Does not expose payment methods or card information.",
|
|
6196
6456
|
inputSchema: CreditsInfoInputSchema,
|
|
6197
|
-
outputSchema: CreditsInfoOutputSchema,
|
|
6457
|
+
outputSchema: recordOutputSchema("credits_info", CreditsInfoOutputSchema),
|
|
6198
6458
|
annotations: {
|
|
6199
6459
|
title: "MCP Scraper Credits & Costs",
|
|
6200
6460
|
readOnlyHint: true,
|
|
@@ -6319,10 +6579,10 @@ if (!forceStdio && (interactiveTerminal || wantsHelp)) {
|
|
|
6319
6579
|
}
|
|
6320
6580
|
function readApiKeyFile() {
|
|
6321
6581
|
const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
|
|
6322
|
-
const paths = [explicitPath, (0,
|
|
6582
|
+
const paths = [explicitPath, (0, import_node_path7.join)((0, import_node_os6.homedir)(), ".mcp-scraper-key")].filter(Boolean);
|
|
6323
6583
|
for (const path of paths) {
|
|
6324
6584
|
try {
|
|
6325
|
-
const value = (0,
|
|
6585
|
+
const value = (0, import_node_fs6.readFileSync)(path, "utf8").trim();
|
|
6326
6586
|
if (value) return value;
|
|
6327
6587
|
} catch {
|
|
6328
6588
|
}
|