mcp-scraper 0.3.15 → 0.3.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/bin/api-server.cjs +290 -23
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +1 -1
  4. package/dist/bin/browser-agent-stdio-server.cjs +1 -1
  5. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  6. package/dist/bin/browser-agent-stdio-server.js +2 -2
  7. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  8. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.js +1 -1
  10. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +253 -19
  11. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -4
  13. package/dist/bin/mcp-scraper-install.cjs +3 -3
  14. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-install.js +2 -2
  16. package/dist/bin/mcp-stdio-server.cjs +251 -17
  17. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  18. package/dist/bin/mcp-stdio-server.js +2 -2
  19. package/dist/{chunk-WQAIOY2D.js → chunk-7XARJHS6.js} +254 -18
  20. package/dist/chunk-7XARJHS6.js.map +1 -0
  21. package/dist/{chunk-7Y6JDBML.js → chunk-ACIUCZ27.js} +3 -3
  22. package/dist/chunk-ACIUCZ27.js.map +1 -0
  23. package/dist/{chunk-WJ2LWHPA.js → chunk-ISZWGIWL.js} +2 -2
  24. package/dist/chunk-SOMBZK3V.js +7 -0
  25. package/dist/chunk-SOMBZK3V.js.map +1 -0
  26. package/dist/{server-UOQ2JIUQ.js → server-LPVBSE2E.js} +35 -8
  27. package/dist/server-LPVBSE2E.js.map +1 -0
  28. package/docs/mcp-tool-manifest.generated.json +45 -10
  29. package/package.json +1 -1
  30. package/dist/chunk-2MX5WOII.js +0 -7
  31. package/dist/chunk-2MX5WOII.js.map +0 -1
  32. package/dist/chunk-7Y6JDBML.js.map +0 -1
  33. package/dist/chunk-WQAIOY2D.js.map +0 -1
  34. package/dist/server-UOQ2JIUQ.js.map +0 -1
  35. /package/dist/{chunk-WJ2LWHPA.js.map → chunk-ISZWGIWL.js.map} +0 -0
@@ -2,11 +2,11 @@
2
2
  import {
3
3
  HttpMcpToolExecutor,
4
4
  buildPaaExtractorMcpServer
5
- } from "../chunk-WQAIOY2D.js";
5
+ } from "../chunk-7XARJHS6.js";
6
6
  import "../chunk-M2S27J6Z.js";
7
7
  import "../chunk-WN7PBKMV.js";
8
8
  import "../chunk-LFATOGDF.js";
9
- import "../chunk-2MX5WOII.js";
9
+ import "../chunk-SOMBZK3V.js";
10
10
 
11
11
  // bin/mcp-stdio-server.ts
12
12
  import { readFileSync } from "fs";
@@ -11,7 +11,7 @@ import {
11
11
  } from "./chunk-LFATOGDF.js";
12
12
  import {
13
13
  PACKAGE_VERSION
14
- } from "./chunk-2MX5WOII.js";
14
+ } from "./chunk-SOMBZK3V.js";
15
15
 
16
16
  // src/harvest-timeout.ts
17
17
  var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -364,6 +364,137 @@ _Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1
364
364
  ].join("\n");
365
365
  }
366
366
 
367
+ // src/api/image-audit.ts
368
+ var OVER_BYTES = 100 * 1024;
369
+ var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
370
+ function formatBytes(n) {
371
+ if (n == null) return null;
372
+ if (n < 1024) return `${n} B`;
373
+ if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
374
+ return `${(n / (1024 * 1024)).toFixed(2)} MB`;
375
+ }
376
+ var decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
377
+ var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
378
+ var formatOf = (ct, url) => {
379
+ if (ct) {
380
+ const m = ct.split(";")[0].trim().toLowerCase();
381
+ if (m.startsWith("image/")) return m.replace("image/", "");
382
+ }
383
+ return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
384
+ };
385
+ function collectUrls(pages) {
386
+ const byKey = /* @__PURE__ */ new Map();
387
+ for (const p of pages) for (const raw of p.imageLinks || []) {
388
+ const u = decodeEntities(raw);
389
+ if (!/^https?:\/\//i.test(u)) continue;
390
+ if (/[{}]|%7[bd]/i.test(u)) continue;
391
+ const k = dedupKey(u);
392
+ const prev = byKey.get(k);
393
+ if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
394
+ }
395
+ return [...byKey.values()];
396
+ }
397
+ async function sizeAndType(url, timeoutMs) {
398
+ const ctrl = new AbortController();
399
+ const t = setTimeout(() => ctrl.abort(), timeoutMs);
400
+ const read = (res) => ({
401
+ len: res.headers.get("content-length"),
402
+ cr: res.headers.get("content-range"),
403
+ ct: res.headers.get("content-type"),
404
+ status: res.status
405
+ });
406
+ try {
407
+ let bytes = null;
408
+ let ct = null;
409
+ let status = null;
410
+ try {
411
+ const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
412
+ status = h.status;
413
+ ct = h.ct;
414
+ if (h.len) bytes = Number(h.len);
415
+ } catch {
416
+ }
417
+ if (bytes == null) {
418
+ const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
419
+ const r = read(g);
420
+ status = status ?? r.status;
421
+ ct = ct ?? r.ct;
422
+ if (r.cr && r.cr.includes("/")) {
423
+ const tail = r.cr.split("/")[1];
424
+ if (tail && tail !== "*") bytes = Number(tail);
425
+ } else if (r.len) bytes = Number(r.len);
426
+ try {
427
+ await g.body?.cancel();
428
+ } catch {
429
+ }
430
+ }
431
+ return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
432
+ } catch (e) {
433
+ return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
434
+ } finally {
435
+ clearTimeout(t);
436
+ }
437
+ }
438
+ async function pool(items, n, fn) {
439
+ const out = new Array(items.length);
440
+ let i = 0;
441
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
442
+ while (i < items.length) {
443
+ const idx = i++;
444
+ out[idx] = await fn(items[idx]);
445
+ }
446
+ }));
447
+ return out;
448
+ }
449
+ async function auditImages(pages, opts = {}) {
450
+ const concurrency = opts.concurrency ?? 12;
451
+ const timeoutMs = opts.timeoutMs ?? 12e3;
452
+ const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
453
+ const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
454
+ const rows = heads.map((r) => {
455
+ const format = formatOf(r.contentType, r.url);
456
+ return {
457
+ ...r,
458
+ size: formatBytes(r.bytes),
459
+ format,
460
+ over100kb: r.bytes != null && r.bytes > OVER_BYTES,
461
+ legacyFormat: format !== "unknown" && !MODERN.has(format)
462
+ };
463
+ });
464
+ const sized = rows.filter((r) => r.bytes != null);
465
+ const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
466
+ const formatCounts = {};
467
+ for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
468
+ return {
469
+ rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
470
+ summary: {
471
+ unique: rows.length,
472
+ sized: sized.length,
473
+ totalBytes,
474
+ totalSize: formatBytes(totalBytes) ?? "0 B",
475
+ avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
476
+ over100kb: rows.filter((r) => r.over100kb).length,
477
+ legacyFormat: rows.filter((r) => r.legacyFormat).length,
478
+ formatCounts
479
+ }
480
+ };
481
+ }
482
+ function renderImageSection(audit) {
483
+ const s = audit.summary;
484
+ const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join(" ");
485
+ const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
486
+ const lines = [
487
+ `## Images`,
488
+ `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
489
+ `Formats: ${fmt}`
490
+ ];
491
+ if (heaviest.length) {
492
+ lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
493
+ for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
494
+ }
495
+ return lines.join("\n");
496
+ }
497
+
367
498
  // src/mcp/mcp-response-formatter.ts
368
499
  var reportSavingEnabled = true;
369
500
  function configureReportSaving(enabled) {
@@ -402,7 +533,7 @@ var BULK_URL_THRESHOLD = 500;
402
533
  function reportSavingActive() {
403
534
  return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
404
535
  }
405
- function saveBulkSite(siteUrl, pages, seo) {
536
+ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
406
537
  if (!reportSavingActive()) return null;
407
538
  try {
408
539
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -425,15 +556,32 @@ function saveBulkSite(siteUrl, pages, seo) {
425
556
  writeFileSync(join(pagesDir, fname), content, "utf8");
426
557
  return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
427
558
  });
559
+ const dataFilesSection = seo ? [
560
+ `
561
+ ## Data files (in this folder)`,
562
+ `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
563
+ `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
564
+ `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
565
+ `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
566
+ `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
567
+ ...imageAudit ? [
568
+ `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
569
+ `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
570
+ ] : [],
571
+ ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
572
+ ].join("\n") : "";
428
573
  const index = [
429
574
  `# Site Extract: ${siteUrl}`,
430
- `**${pages.length} pages** \u2014 each saved as an individual Markdown file under \`pages/\`.`,
575
+ `**${pages.length} pages** crawled. Everything is in this folder.`,
576
+ `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
577
+ seo ? `- SEO crawl data: the data files listed below.` : "",
578
+ dataFilesSection,
431
579
  `
432
580
  ## Pages
433
581
  | # | Title | URL | File |
434
582
  |---|-------|-----|------|
435
583
  ${indexRows.join("\n")}`
436
- ].join("\n");
584
+ ].filter(Boolean).join("\n");
437
585
  const indexFile = join(dir, "index.md");
438
586
  writeFileSync(indexFile, index, "utf8");
439
587
  let seoFiles;
@@ -448,8 +596,17 @@ ${indexRows.join("\n")}`
448
596
  writeFileSync(linksJsonl, toJsonl(seo.edges), "utf8");
449
597
  writeFileSync(metricsJsonl, toJsonl(seo.metrics), "utf8");
450
598
  writeFileSync(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
451
- writeFileSync(reportFile, seo.reportMd, "utf8");
599
+ writeFileSync(reportFile, seo.reportMd + (imageAudit ? `
600
+
601
+ ${renderImageSection(imageAudit)}` : ""), "utf8");
452
602
  seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
603
+ if (imageAudit) {
604
+ const imagesJsonl = join(dir, "images.jsonl");
605
+ const imagesSummary = join(dir, "images-summary.json");
606
+ writeFileSync(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
607
+ writeFileSync(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
608
+ seoFiles.push(imagesJsonl, imagesSummary);
609
+ }
453
610
  if (seo.branding) {
454
611
  const brandingFile = join(dir, "branding.json");
455
612
  writeFileSync(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
@@ -924,24 +1081,23 @@ function formatExtractSite(raw, input) {
924
1081
  durationMs: d.durationMs ?? 0
925
1082
  };
926
1083
  if (pages.length > BULK_PAGE_THRESHOLD) {
927
- const seo = buildSeoExport(input.url, pages, d.branding);
928
1084
  const bulk = saveBulkSite(input.url, pages.map((p) => ({
929
1085
  url: String(p.url ?? ""),
930
1086
  title: p.title ?? null,
931
1087
  bodyMarkdown: p.bodyMarkdown ?? null,
932
1088
  metaDescription: p.metaDescription ?? null,
933
1089
  schemaTypes: schemaTypesOf(p)
934
- })), seo);
1090
+ })));
935
1091
  const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
936
- const seoLine = bulk?.seoFiles?.length ? `
937
- - **SEO data:** \`report.md\` (issues summary), \`issues.json\` (findings + offender URLs), \`pages.jsonl\` (per-page fields), \`links.jsonl\` (link edges), \`link-metrics.jsonl\` (inlinks / crawl depth / orphans)` : "";
938
1092
  const location = bulk ? `
939
1093
  ## \u{1F4C1} Bulk scrape saved
940
- - **Folder:** \`${bulk.dir}\`
941
- - **Index:** \`${bulk.indexFile}\`
942
- - **Files:** ${bulk.fileCount} page files under \`pages/\`${seoLine}
1094
+ - **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
1095
+ - **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
1096
+ - **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
1097
+
1098
+ The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
943
1099
 
944
- Read \`index.md\` for the full page list, open files in \`pages/\` for full content, or read \`pages.jsonl\` / \`link-metrics.jsonl\` for the SEO crawl data. Page bodies are not inlined here to keep the context window clear.` : `
1100
+ \u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
945
1101
  ## \u26A0\uFE0F Bulk scrape not saved
946
1102
  Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
947
1103
  const full2 = [
@@ -985,6 +1141,55 @@ ${body}` : "_(no content extracted)_"
985
1141
  ${pageDetails}${tips}`;
986
1142
  return { ...oneBlock(full, diskReport), structuredContent };
987
1143
  }
1144
+ async function formatAuditSite(raw, input) {
1145
+ const parsed = parseData(raw);
1146
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
1147
+ const d = parsed.data;
1148
+ const pages = d.pages ?? [];
1149
+ const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
1150
+ const seo = buildSeoExport(input.url, pages, d.branding);
1151
+ const imageAudit = await auditImages(pages, { concurrency: 12 });
1152
+ const bulk = saveBulkSite(input.url, pages.map((p) => ({
1153
+ url: String(p.url ?? ""),
1154
+ title: p.title ?? null,
1155
+ bodyMarkdown: p.bodyMarkdown ?? null,
1156
+ metaDescription: p.metaDescription ?? null,
1157
+ schemaTypes: schemaTypesOf(p)
1158
+ })), seo, imageAudit);
1159
+ const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
1160
+ const img = imageAudit.summary;
1161
+ const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
1162
+ const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
1163
+ const structuredContent = {
1164
+ url: input.url,
1165
+ pageCount: pages.length,
1166
+ durationMs: d.durationMs ?? 0,
1167
+ bulkFolder: bulk?.dir ?? null,
1168
+ issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
1169
+ images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
1170
+ };
1171
+ const location = bulk ? `
1172
+ ## \u{1F4C1} Technical audit saved
1173
+ - **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
1174
+ - **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
1175
+ - **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
1176
+
1177
+ Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
1178
+ ## \u26A0\uFE0F Audit not saved
1179
+ Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
1180
+ const full = [
1181
+ `# Technical SEO Audit: ${input.url}`,
1182
+ durationLine,
1183
+ location,
1184
+ `
1185
+ ## Top issues
1186
+ ${topIssues || "_none found_"}`,
1187
+ `
1188
+ ## Images
1189
+ ${imgLine}`
1190
+ ].join("\n");
1191
+ return { content: [{ type: "text", text: full }], structuredContent };
1192
+ }
988
1193
  function formatYoutubeHarvest(raw, input) {
989
1194
  const parsed = parseData(raw);
990
1195
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -2187,12 +2392,18 @@ var MapSiteUrlsInputSchema = {
2187
2392
  maxUrls: z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
2188
2393
  };
2189
2394
  var ExtractSiteInputSchema = {
2190
- url: z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
2395
+ url: z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
2191
2396
  maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
2192
2397
  rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
2193
2398
  rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
2194
2399
  formats: z.array(z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
2195
2400
  };
2401
+ var AuditSiteInputSchema = {
2402
+ url: z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
2403
+ maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
2404
+ rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
2405
+ rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
2406
+ };
2196
2407
  var YoutubeHarvestInputSchema = {
2197
2408
  mode: z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
2198
2409
  query: z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
@@ -2493,6 +2704,19 @@ var ExtractSiteOutputSchema = {
2493
2704
  })),
2494
2705
  durationMs: z.number().min(0)
2495
2706
  };
2707
+ var AuditSiteOutputSchema = {
2708
+ url: z.string(),
2709
+ pageCount: z.number().int().min(0),
2710
+ durationMs: z.number().min(0),
2711
+ bulkFolder: z.string().nullable(),
2712
+ issues: z.record(z.string(), z.number()),
2713
+ images: z.object({
2714
+ unique: z.number().int().min(0),
2715
+ totalBytes: z.number().min(0),
2716
+ over100kb: z.number().int().min(0),
2717
+ legacyFormat: z.number().int().min(0)
2718
+ })
2719
+ };
2496
2720
  var MapsPlaceIntelOutputSchema = {
2497
2721
  name: z.string(),
2498
2722
  rating: NullableString,
@@ -3334,12 +3558,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
3334
3558
  annotations: liveWebToolAnnotations("Site URL Map")
3335
3559
  }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
3336
3560
  server.registerTool("extract_site", {
3337
- title: "Multi-Page Site Extract",
3338
- description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all page content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
3561
+ title: "Multi-Page Site Content Crawl",
3562
+ description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
3339
3563
  inputSchema: ExtractSiteInputSchema,
3340
3564
  outputSchema: ExtractSiteOutputSchema,
3341
- annotations: liveWebToolAnnotations("Multi-Page Site Extract")
3565
+ annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
3342
3566
  }, async (input) => formatExtractSite(await executor.extractSite(input), input));
3567
+ server.registerTool("audit_site", {
3568
+ title: "Technical SEO Audit",
3569
+ description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
3570
+ inputSchema: AuditSiteInputSchema,
3571
+ outputSchema: AuditSiteOutputSchema,
3572
+ annotations: liveWebToolAnnotations("Technical SEO Audit")
3573
+ }, async (input) => formatAuditSite(await executor.auditSite(input), input));
3343
3574
  server.registerTool("youtube_harvest", {
3344
3575
  title: "YouTube Video Harvest",
3345
3576
  description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -3656,6 +3887,9 @@ var HttpMcpToolExecutor = class {
3656
3887
  extractSite(input) {
3657
3888
  return this.call("/extract-site", input);
3658
3889
  }
3890
+ auditSite(input) {
3891
+ return this.call("/extract-site", input);
3892
+ }
3659
3893
  youtubeHarvest(input) {
3660
3894
  return this.call("/youtube/harvest", input);
3661
3895
  }
@@ -3746,6 +3980,8 @@ export {
3746
3980
  buildLinkGraph,
3747
3981
  computeIssues,
3748
3982
  renderIssueReport,
3983
+ auditImages,
3984
+ renderImageSection,
3749
3985
  configureReportSaving,
3750
3986
  outputBaseDir,
3751
3987
  formatCaptureSerpSnapshot,
@@ -3760,4 +3996,4 @@ export {
3760
3996
  registerPaaExtractorMcpTools,
3761
3997
  HttpMcpToolExecutor
3762
3998
  };
3763
- //# sourceMappingURL=chunk-WQAIOY2D.js.map
3999
+ //# sourceMappingURL=chunk-7XARJHS6.js.map