mcp-scraper 0.3.15 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +290 -23
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +253 -19
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -4
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +251 -17
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-WQAIOY2D.js → chunk-7XARJHS6.js} +254 -18
- package/dist/chunk-7XARJHS6.js.map +1 -0
- package/dist/{chunk-7Y6JDBML.js → chunk-ACIUCZ27.js} +3 -3
- package/dist/chunk-ACIUCZ27.js.map +1 -0
- package/dist/{chunk-WJ2LWHPA.js → chunk-ISZWGIWL.js} +2 -2
- package/dist/chunk-SOMBZK3V.js +7 -0
- package/dist/chunk-SOMBZK3V.js.map +1 -0
- package/dist/{server-UOQ2JIUQ.js → server-LPVBSE2E.js} +35 -8
- package/dist/server-LPVBSE2E.js.map +1 -0
- package/docs/mcp-tool-manifest.generated.json +45 -10
- package/package.json +1 -1
- package/dist/chunk-2MX5WOII.js +0 -7
- package/dist/chunk-2MX5WOII.js.map +0 -1
- package/dist/chunk-7Y6JDBML.js.map +0 -1
- package/dist/chunk-WQAIOY2D.js.map +0 -1
- package/dist/server-UOQ2JIUQ.js.map +0 -1
- /package/dist/{chunk-WJ2LWHPA.js.map → chunk-ISZWGIWL.js.map} +0 -0
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
import {
|
|
3
3
|
HttpMcpToolExecutor,
|
|
4
4
|
buildPaaExtractorMcpServer
|
|
5
|
-
} from "../chunk-
|
|
5
|
+
} from "../chunk-7XARJHS6.js";
|
|
6
6
|
import "../chunk-M2S27J6Z.js";
|
|
7
7
|
import "../chunk-WN7PBKMV.js";
|
|
8
8
|
import "../chunk-LFATOGDF.js";
|
|
9
|
-
import "../chunk-
|
|
9
|
+
import "../chunk-SOMBZK3V.js";
|
|
10
10
|
|
|
11
11
|
// bin/mcp-stdio-server.ts
|
|
12
12
|
import { readFileSync } from "fs";
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from "./chunk-LFATOGDF.js";
|
|
12
12
|
import {
|
|
13
13
|
PACKAGE_VERSION
|
|
14
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-SOMBZK3V.js";
|
|
15
15
|
|
|
16
16
|
// src/harvest-timeout.ts
|
|
17
17
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
@@ -364,6 +364,137 @@ _Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1
|
|
|
364
364
|
].join("\n");
|
|
365
365
|
}
|
|
366
366
|
|
|
367
|
+
// src/api/image-audit.ts
|
|
368
|
+
var OVER_BYTES = 100 * 1024;
|
|
369
|
+
var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
|
|
370
|
+
function formatBytes(n) {
|
|
371
|
+
if (n == null) return null;
|
|
372
|
+
if (n < 1024) return `${n} B`;
|
|
373
|
+
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
|
374
|
+
return `${(n / (1024 * 1024)).toFixed(2)} MB`;
|
|
375
|
+
}
|
|
376
|
+
var decodeEntities = (u) => u.replace(/&/g, "&").replace(/�?38;/g, "&").replace(/&/gi, "&").trim();
|
|
377
|
+
var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
|
|
378
|
+
var formatOf = (ct, url) => {
|
|
379
|
+
if (ct) {
|
|
380
|
+
const m = ct.split(";")[0].trim().toLowerCase();
|
|
381
|
+
if (m.startsWith("image/")) return m.replace("image/", "");
|
|
382
|
+
}
|
|
383
|
+
return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
|
|
384
|
+
};
|
|
385
|
+
function collectUrls(pages) {
|
|
386
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
387
|
+
for (const p of pages) for (const raw of p.imageLinks || []) {
|
|
388
|
+
const u = decodeEntities(raw);
|
|
389
|
+
if (!/^https?:\/\//i.test(u)) continue;
|
|
390
|
+
if (/[{}]|%7[bd]/i.test(u)) continue;
|
|
391
|
+
const k = dedupKey(u);
|
|
392
|
+
const prev = byKey.get(k);
|
|
393
|
+
if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
|
|
394
|
+
}
|
|
395
|
+
return [...byKey.values()];
|
|
396
|
+
}
|
|
397
|
+
async function sizeAndType(url, timeoutMs) {
|
|
398
|
+
const ctrl = new AbortController();
|
|
399
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
400
|
+
const read = (res) => ({
|
|
401
|
+
len: res.headers.get("content-length"),
|
|
402
|
+
cr: res.headers.get("content-range"),
|
|
403
|
+
ct: res.headers.get("content-type"),
|
|
404
|
+
status: res.status
|
|
405
|
+
});
|
|
406
|
+
try {
|
|
407
|
+
let bytes = null;
|
|
408
|
+
let ct = null;
|
|
409
|
+
let status = null;
|
|
410
|
+
try {
|
|
411
|
+
const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
|
|
412
|
+
status = h.status;
|
|
413
|
+
ct = h.ct;
|
|
414
|
+
if (h.len) bytes = Number(h.len);
|
|
415
|
+
} catch {
|
|
416
|
+
}
|
|
417
|
+
if (bytes == null) {
|
|
418
|
+
const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
|
|
419
|
+
const r = read(g);
|
|
420
|
+
status = status ?? r.status;
|
|
421
|
+
ct = ct ?? r.ct;
|
|
422
|
+
if (r.cr && r.cr.includes("/")) {
|
|
423
|
+
const tail = r.cr.split("/")[1];
|
|
424
|
+
if (tail && tail !== "*") bytes = Number(tail);
|
|
425
|
+
} else if (r.len) bytes = Number(r.len);
|
|
426
|
+
try {
|
|
427
|
+
await g.body?.cancel();
|
|
428
|
+
} catch {
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
|
|
432
|
+
} catch (e) {
|
|
433
|
+
return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
|
|
434
|
+
} finally {
|
|
435
|
+
clearTimeout(t);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
async function pool(items, n, fn) {
|
|
439
|
+
const out = new Array(items.length);
|
|
440
|
+
let i = 0;
|
|
441
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
|
|
442
|
+
while (i < items.length) {
|
|
443
|
+
const idx = i++;
|
|
444
|
+
out[idx] = await fn(items[idx]);
|
|
445
|
+
}
|
|
446
|
+
}));
|
|
447
|
+
return out;
|
|
448
|
+
}
|
|
449
|
+
async function auditImages(pages, opts = {}) {
|
|
450
|
+
const concurrency = opts.concurrency ?? 12;
|
|
451
|
+
const timeoutMs = opts.timeoutMs ?? 12e3;
|
|
452
|
+
const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
|
|
453
|
+
const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
|
|
454
|
+
const rows = heads.map((r) => {
|
|
455
|
+
const format = formatOf(r.contentType, r.url);
|
|
456
|
+
return {
|
|
457
|
+
...r,
|
|
458
|
+
size: formatBytes(r.bytes),
|
|
459
|
+
format,
|
|
460
|
+
over100kb: r.bytes != null && r.bytes > OVER_BYTES,
|
|
461
|
+
legacyFormat: format !== "unknown" && !MODERN.has(format)
|
|
462
|
+
};
|
|
463
|
+
});
|
|
464
|
+
const sized = rows.filter((r) => r.bytes != null);
|
|
465
|
+
const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
|
|
466
|
+
const formatCounts = {};
|
|
467
|
+
for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
|
|
468
|
+
return {
|
|
469
|
+
rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
|
|
470
|
+
summary: {
|
|
471
|
+
unique: rows.length,
|
|
472
|
+
sized: sized.length,
|
|
473
|
+
totalBytes,
|
|
474
|
+
totalSize: formatBytes(totalBytes) ?? "0 B",
|
|
475
|
+
avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
|
|
476
|
+
over100kb: rows.filter((r) => r.over100kb).length,
|
|
477
|
+
legacyFormat: rows.filter((r) => r.legacyFormat).length,
|
|
478
|
+
formatCounts
|
|
479
|
+
}
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function renderImageSection(audit) {
|
|
483
|
+
const s = audit.summary;
|
|
484
|
+
const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join(" ");
|
|
485
|
+
const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
|
|
486
|
+
const lines = [
|
|
487
|
+
`## Images`,
|
|
488
|
+
`**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
|
|
489
|
+
`Formats: ${fmt}`
|
|
490
|
+
];
|
|
491
|
+
if (heaviest.length) {
|
|
492
|
+
lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
|
|
493
|
+
for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
|
|
494
|
+
}
|
|
495
|
+
return lines.join("\n");
|
|
496
|
+
}
|
|
497
|
+
|
|
367
498
|
// src/mcp/mcp-response-formatter.ts
|
|
368
499
|
var reportSavingEnabled = true;
|
|
369
500
|
function configureReportSaving(enabled) {
|
|
@@ -402,7 +533,7 @@ var BULK_URL_THRESHOLD = 500;
|
|
|
402
533
|
function reportSavingActive() {
|
|
403
534
|
return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
|
|
404
535
|
}
|
|
405
|
-
function saveBulkSite(siteUrl, pages, seo) {
|
|
536
|
+
function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
406
537
|
if (!reportSavingActive()) return null;
|
|
407
538
|
try {
|
|
408
539
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
@@ -425,15 +556,32 @@ function saveBulkSite(siteUrl, pages, seo) {
|
|
|
425
556
|
writeFileSync(join(pagesDir, fname), content, "utf8");
|
|
426
557
|
return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
|
|
427
558
|
});
|
|
559
|
+
const dataFilesSection = seo ? [
|
|
560
|
+
`
|
|
561
|
+
## Data files (in this folder)`,
|
|
562
|
+
`- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
|
|
563
|
+
`- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
|
|
564
|
+
`- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
|
|
565
|
+
`- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
|
|
566
|
+
`- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
|
|
567
|
+
...imageAudit ? [
|
|
568
|
+
`- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
|
|
569
|
+
`- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
|
|
570
|
+
] : [],
|
|
571
|
+
...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
|
|
572
|
+
].join("\n") : "";
|
|
428
573
|
const index = [
|
|
429
574
|
`# Site Extract: ${siteUrl}`,
|
|
430
|
-
`**${pages.length} pages**
|
|
575
|
+
`**${pages.length} pages** crawled. Everything is in this folder.`,
|
|
576
|
+
`- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
|
|
577
|
+
seo ? `- SEO crawl data: the data files listed below.` : "",
|
|
578
|
+
dataFilesSection,
|
|
431
579
|
`
|
|
432
580
|
## Pages
|
|
433
581
|
| # | Title | URL | File |
|
|
434
582
|
|---|-------|-----|------|
|
|
435
583
|
${indexRows.join("\n")}`
|
|
436
|
-
].join("\n");
|
|
584
|
+
].filter(Boolean).join("\n");
|
|
437
585
|
const indexFile = join(dir, "index.md");
|
|
438
586
|
writeFileSync(indexFile, index, "utf8");
|
|
439
587
|
let seoFiles;
|
|
@@ -448,8 +596,17 @@ ${indexRows.join("\n")}`
|
|
|
448
596
|
writeFileSync(linksJsonl, toJsonl(seo.edges), "utf8");
|
|
449
597
|
writeFileSync(metricsJsonl, toJsonl(seo.metrics), "utf8");
|
|
450
598
|
writeFileSync(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
|
|
451
|
-
writeFileSync(reportFile, seo.reportMd
|
|
599
|
+
writeFileSync(reportFile, seo.reportMd + (imageAudit ? `
|
|
600
|
+
|
|
601
|
+
${renderImageSection(imageAudit)}` : ""), "utf8");
|
|
452
602
|
seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
|
|
603
|
+
if (imageAudit) {
|
|
604
|
+
const imagesJsonl = join(dir, "images.jsonl");
|
|
605
|
+
const imagesSummary = join(dir, "images-summary.json");
|
|
606
|
+
writeFileSync(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
|
|
607
|
+
writeFileSync(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
|
|
608
|
+
seoFiles.push(imagesJsonl, imagesSummary);
|
|
609
|
+
}
|
|
453
610
|
if (seo.branding) {
|
|
454
611
|
const brandingFile = join(dir, "branding.json");
|
|
455
612
|
writeFileSync(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
|
|
@@ -924,24 +1081,23 @@ function formatExtractSite(raw, input) {
|
|
|
924
1081
|
durationMs: d.durationMs ?? 0
|
|
925
1082
|
};
|
|
926
1083
|
if (pages.length > BULK_PAGE_THRESHOLD) {
|
|
927
|
-
const seo = buildSeoExport(input.url, pages, d.branding);
|
|
928
1084
|
const bulk = saveBulkSite(input.url, pages.map((p) => ({
|
|
929
1085
|
url: String(p.url ?? ""),
|
|
930
1086
|
title: p.title ?? null,
|
|
931
1087
|
bodyMarkdown: p.bodyMarkdown ?? null,
|
|
932
1088
|
metaDescription: p.metaDescription ?? null,
|
|
933
1089
|
schemaTypes: schemaTypesOf(p)
|
|
934
|
-
}))
|
|
1090
|
+
})));
|
|
935
1091
|
const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
|
|
936
|
-
const seoLine = bulk?.seoFiles?.length ? `
|
|
937
|
-
- **SEO data:** \`report.md\` (issues summary), \`issues.json\` (findings + offender URLs), \`pages.jsonl\` (per-page fields), \`links.jsonl\` (link edges), \`link-metrics.jsonl\` (inlinks / crawl depth / orphans)` : "";
|
|
938
1092
|
const location = bulk ? `
|
|
939
1093
|
## \u{1F4C1} Bulk scrape saved
|
|
940
|
-
- **Folder:** \`${bulk.dir}\`
|
|
941
|
-
- **
|
|
942
|
-
- **
|
|
1094
|
+
- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
|
|
1095
|
+
- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
|
|
1096
|
+
- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
|
|
1097
|
+
|
|
1098
|
+
The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
|
|
943
1099
|
|
|
944
|
-
|
|
1100
|
+
\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
|
|
945
1101
|
## \u26A0\uFE0F Bulk scrape not saved
|
|
946
1102
|
Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
|
|
947
1103
|
const full2 = [
|
|
@@ -985,6 +1141,55 @@ ${body}` : "_(no content extracted)_"
|
|
|
985
1141
|
${pageDetails}${tips}`;
|
|
986
1142
|
return { ...oneBlock(full, diskReport), structuredContent };
|
|
987
1143
|
}
|
|
1144
|
+
async function formatAuditSite(raw, input) {
|
|
1145
|
+
const parsed = parseData(raw);
|
|
1146
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1147
|
+
const d = parsed.data;
|
|
1148
|
+
const pages = d.pages ?? [];
|
|
1149
|
+
const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
|
|
1150
|
+
const seo = buildSeoExport(input.url, pages, d.branding);
|
|
1151
|
+
const imageAudit = await auditImages(pages, { concurrency: 12 });
|
|
1152
|
+
const bulk = saveBulkSite(input.url, pages.map((p) => ({
|
|
1153
|
+
url: String(p.url ?? ""),
|
|
1154
|
+
title: p.title ?? null,
|
|
1155
|
+
bodyMarkdown: p.bodyMarkdown ?? null,
|
|
1156
|
+
metaDescription: p.metaDescription ?? null,
|
|
1157
|
+
schemaTypes: schemaTypesOf(p)
|
|
1158
|
+
})), seo, imageAudit);
|
|
1159
|
+
const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
|
|
1160
|
+
const img = imageAudit.summary;
|
|
1161
|
+
const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
|
|
1162
|
+
const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
|
|
1163
|
+
const structuredContent = {
|
|
1164
|
+
url: input.url,
|
|
1165
|
+
pageCount: pages.length,
|
|
1166
|
+
durationMs: d.durationMs ?? 0,
|
|
1167
|
+
bulkFolder: bulk?.dir ?? null,
|
|
1168
|
+
issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
|
|
1169
|
+
images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
|
|
1170
|
+
};
|
|
1171
|
+
const location = bulk ? `
|
|
1172
|
+
## \u{1F4C1} Technical audit saved
|
|
1173
|
+
- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
|
|
1174
|
+
- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
|
|
1175
|
+
- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
|
|
1176
|
+
|
|
1177
|
+
Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
|
|
1178
|
+
## \u26A0\uFE0F Audit not saved
|
|
1179
|
+
Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
|
|
1180
|
+
const full = [
|
|
1181
|
+
`# Technical SEO Audit: ${input.url}`,
|
|
1182
|
+
durationLine,
|
|
1183
|
+
location,
|
|
1184
|
+
`
|
|
1185
|
+
## Top issues
|
|
1186
|
+
${topIssues || "_none found_"}`,
|
|
1187
|
+
`
|
|
1188
|
+
## Images
|
|
1189
|
+
${imgLine}`
|
|
1190
|
+
].join("\n");
|
|
1191
|
+
return { content: [{ type: "text", text: full }], structuredContent };
|
|
1192
|
+
}
|
|
988
1193
|
function formatYoutubeHarvest(raw, input) {
|
|
989
1194
|
const parsed = parseData(raw);
|
|
990
1195
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -2187,12 +2392,18 @@ var MapSiteUrlsInputSchema = {
|
|
|
2187
2392
|
maxUrls: z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
|
|
2188
2393
|
};
|
|
2189
2394
|
var ExtractSiteInputSchema = {
|
|
2190
|
-
url: z.string().url().describe("Public website URL or domain to
|
|
2395
|
+
url: z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
|
|
2191
2396
|
maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
|
|
2192
2397
|
rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
|
|
2193
2398
|
rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
|
|
2194
2399
|
formats: z.array(z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
|
|
2195
2400
|
};
|
|
2401
|
+
var AuditSiteInputSchema = {
|
|
2402
|
+
url: z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
|
|
2403
|
+
maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
|
|
2404
|
+
rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
|
|
2405
|
+
rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
|
|
2406
|
+
};
|
|
2196
2407
|
var YoutubeHarvestInputSchema = {
|
|
2197
2408
|
mode: z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
2198
2409
|
query: z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
@@ -2493,6 +2704,19 @@ var ExtractSiteOutputSchema = {
|
|
|
2493
2704
|
})),
|
|
2494
2705
|
durationMs: z.number().min(0)
|
|
2495
2706
|
};
|
|
2707
|
+
var AuditSiteOutputSchema = {
|
|
2708
|
+
url: z.string(),
|
|
2709
|
+
pageCount: z.number().int().min(0),
|
|
2710
|
+
durationMs: z.number().min(0),
|
|
2711
|
+
bulkFolder: z.string().nullable(),
|
|
2712
|
+
issues: z.record(z.string(), z.number()),
|
|
2713
|
+
images: z.object({
|
|
2714
|
+
unique: z.number().int().min(0),
|
|
2715
|
+
totalBytes: z.number().min(0),
|
|
2716
|
+
over100kb: z.number().int().min(0),
|
|
2717
|
+
legacyFormat: z.number().int().min(0)
|
|
2718
|
+
})
|
|
2719
|
+
};
|
|
2496
2720
|
var MapsPlaceIntelOutputSchema = {
|
|
2497
2721
|
name: z.string(),
|
|
2498
2722
|
rating: NullableString,
|
|
@@ -3334,12 +3558,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
3334
3558
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
3335
3559
|
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
3336
3560
|
server.registerTool("extract_site", {
|
|
3337
|
-
title: "Multi-Page Site
|
|
3338
|
-
description: withReportNote("
|
|
3561
|
+
title: "Multi-Page Site Content Crawl",
|
|
3562
|
+
description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
|
|
3339
3563
|
inputSchema: ExtractSiteInputSchema,
|
|
3340
3564
|
outputSchema: ExtractSiteOutputSchema,
|
|
3341
|
-
annotations: liveWebToolAnnotations("Multi-Page Site
|
|
3565
|
+
annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
|
|
3342
3566
|
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
3567
|
+
server.registerTool("audit_site", {
|
|
3568
|
+
title: "Technical SEO Audit",
|
|
3569
|
+
description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
|
|
3570
|
+
inputSchema: AuditSiteInputSchema,
|
|
3571
|
+
outputSchema: AuditSiteOutputSchema,
|
|
3572
|
+
annotations: liveWebToolAnnotations("Technical SEO Audit")
|
|
3573
|
+
}, async (input) => formatAuditSite(await executor.auditSite(input), input));
|
|
3343
3574
|
server.registerTool("youtube_harvest", {
|
|
3344
3575
|
title: "YouTube Video Harvest",
|
|
3345
3576
|
description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
|
|
@@ -3656,6 +3887,9 @@ var HttpMcpToolExecutor = class {
|
|
|
3656
3887
|
extractSite(input) {
|
|
3657
3888
|
return this.call("/extract-site", input);
|
|
3658
3889
|
}
|
|
3890
|
+
auditSite(input) {
|
|
3891
|
+
return this.call("/extract-site", input);
|
|
3892
|
+
}
|
|
3659
3893
|
youtubeHarvest(input) {
|
|
3660
3894
|
return this.call("/youtube/harvest", input);
|
|
3661
3895
|
}
|
|
@@ -3746,6 +3980,8 @@ export {
|
|
|
3746
3980
|
buildLinkGraph,
|
|
3747
3981
|
computeIssues,
|
|
3748
3982
|
renderIssueReport,
|
|
3983
|
+
auditImages,
|
|
3984
|
+
renderImageSection,
|
|
3749
3985
|
configureReportSaving,
|
|
3750
3986
|
outputBaseDir,
|
|
3751
3987
|
formatCaptureSerpSnapshot,
|
|
@@ -3760,4 +3996,4 @@ export {
|
|
|
3760
3996
|
registerPaaExtractorMcpTools,
|
|
3761
3997
|
HttpMcpToolExecutor
|
|
3762
3998
|
};
|
|
3763
|
-
//# sourceMappingURL=chunk-
|
|
3999
|
+
//# sourceMappingURL=chunk-7XARJHS6.js.map
|