mcp-scraper 0.3.15 → 0.3.17

package/dist/bin/mcp-scraper-cli.js

@@ -10,7 +10,7 @@ import {
 import "../chunk-LFATOGDF.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-2MX5WOII.js";
+} from "../chunk-PVWWY5NV.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-combined-stdio-server.cjs

@@ -232,6 +232,9 @@ var HttpMcpToolExecutor = class {
   extractSite(input) {
     return this.call("/extract-site", input);
   }
+  auditSite(input) {
+    return this.call("/extract-site", input);
+  }
   youtubeHarvest(input) {
     return this.call("/youtube/harvest", input);
   }
@@ -453,7 +456,7 @@ render();
 }
 
 // src/version.ts
-var PACKAGE_VERSION = "0.3.15";
+var PACKAGE_VERSION = "0.3.17";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -1967,6 +1970,99 @@ function buildLinkGraph(pages, startUrl) {
   return { edges, metrics };
 }
 
+// src/api/seo-link-report.ts
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function buildLinkReport(edges, metrics, siteUrl) {
+  let siteReg = "";
+  try {
+    siteReg = registrableDomain(new URL(siteUrl).hostname);
+  } catch {
+    siteReg = "";
+  }
+  const internalEdges = edges.filter((e) => e.internal);
+  const pages = metrics.length;
+  const orphans = metrics.filter((m) => m.orphan).length;
+  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
+  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
+  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
+  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+  for (const m of metrics) {
+    if (m.inlinks === 0) distribution.zero++;
+    else if (m.inlinks <= 2) distribution.oneToTwo++;
+    else if (m.inlinks <= 10) distribution.threeToTen++;
+    else distribution.elevenPlus++;
+  }
+  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
+  const domMap = /* @__PURE__ */ new Map();
+  let externalTotal = 0;
+  for (const e of edges) {
+    let domain;
+    try {
+      domain = registrableDomain(new URL(e.to).hostname);
+    } catch {
+      continue;
+    }
+    if (!domain || domain === siteReg) continue;
+    externalTotal++;
+    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+    d.links++;
+    if (e.nofollow) d.nofollow++;
+    d.pages.add(e.from);
+    domMap.set(domain, d);
+  }
+  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+  const round = (n) => Math.round(n * 10) / 10;
+  return {
+    summary: {
+      internal: {
+        totalLinks: internalEdges.length,
+        pages,
+        orphans,
+        brokenInternal,
+        avgInlinks: pages ? round(sumInlinks / pages) : 0,
+        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
+        distribution,
+        topByInlinks
+      },
+      external: {
+        totalLinks: externalTotal,
+        uniqueDomains: externalDomains.length,
+        topDomains: externalDomains.slice(0, 20)
+      }
+    },
+    externalDomains
+  };
+}
+function renderLinkReport(r) {
+  const s = r.summary;
+  const lines = [
+    `## Link analysis`,
+    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
+    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
+    ``,
+    `### Inlink distribution`,
+    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
+    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
+    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
+    `- 11+: ${s.internal.distribution.elevenPlus}`,
+    ``,
+    `### Top 20 internal pages by inlinks`,
+    `| inlinks | out (int/ext) | URL |`,
+    `|---|---|---|`,
+    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
+    ``,
+    `### Top 20 external domains by links`,
+    `| links | nofollow | from pages | domain |`,
+    `|---|---|---|---|`,
+    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
+  ];
+  return lines.join("\n");
+}
+
 // src/api/seo-issues.ts
 var THIN_WORDS = 200;
 var TITLE_MAX = 60;
@@ -2075,6 +2171,137 @@ _Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1
   ].join("\n");
 }
 
+// src/api/image-audit.ts
+var OVER_BYTES = 100 * 1024;
+var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
+function formatBytes(n) {
+  if (n == null) return null;
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
+}
+var decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
+var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
+var formatOf = (ct, url) => {
+  if (ct) {
+    const m = ct.split(";")[0].trim().toLowerCase();
+    if (m.startsWith("image/")) return m.replace("image/", "");
+  }
+  return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
+};
+function collectUrls(pages) {
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of pages) for (const raw of p.imageLinks || []) {
+    const u = decodeEntities(raw);
+    if (!/^https?:\/\//i.test(u)) continue;
+    if (/[{}]|%7[bd]/i.test(u)) continue;
+    const k = dedupKey(u);
+    const prev = byKey.get(k);
+    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
+  }
+  return [...byKey.values()];
+}
+async function sizeAndType(url, timeoutMs) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  const read = (res) => ({
+    len: res.headers.get("content-length"),
+    cr: res.headers.get("content-range"),
+    ct: res.headers.get("content-type"),
+    status: res.status
+  });
+  try {
+    let bytes = null;
+    let ct = null;
+    let status = null;
+    try {
+      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
+      status = h.status;
+      ct = h.ct;
+      if (h.len) bytes = Number(h.len);
+    } catch {
+    }
+    if (bytes == null) {
+      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
+      const r = read(g);
+      status = status ?? r.status;
+      ct = ct ?? r.ct;
+      if (r.cr && r.cr.includes("/")) {
+        const tail = r.cr.split("/")[1];
+        if (tail && tail !== "*") bytes = Number(tail);
+      } else if (r.len) bytes = Number(r.len);
+      try {
+        await g.body?.cancel();
+      } catch {
+      }
+    }
+    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
+  } catch (e) {
+    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+async function pool(items, n, fn) {
+  const out = new Array(items.length);
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) {
+      const idx = i++;
+      out[idx] = await fn(items[idx]);
+    }
+  }));
+  return out;
+}
+async function auditImages(pages, opts = {}) {
+  const concurrency = opts.concurrency ?? 12;
+  const timeoutMs = opts.timeoutMs ?? 12e3;
+  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
+  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
+  const rows = heads.map((r) => {
+    const format = formatOf(r.contentType, r.url);
+    return {
+      ...r,
+      size: formatBytes(r.bytes),
+      format,
+      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
+      legacyFormat: format !== "unknown" && !MODERN.has(format)
+    };
+  });
+  const sized = rows.filter((r) => r.bytes != null);
+  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
+  const formatCounts = {};
+  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
+  return {
+    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
+    summary: {
+      unique: rows.length,
+      sized: sized.length,
+      totalBytes,
+      totalSize: formatBytes(totalBytes) ?? "0 B",
+      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
+      over100kb: rows.filter((r) => r.over100kb).length,
+      legacyFormat: rows.filter((r) => r.legacyFormat).length,
+      formatCounts
+    }
+  };
+}
+function renderImageSection(audit) {
+  const s = audit.summary;
+  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
+  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
+  const lines = [
+    `## Images`,
+    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
+    `Formats: ${fmt}`
+  ];
+  if (heaviest.length) {
+    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
+    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
+  }
+  return lines.join("\n");
+}
+
 // src/mcp/mcp-response-formatter.ts
 var reportSavingEnabled = true;
 function sanitizeVendorText(text) {
@@ -2110,7 +2337,7 @@ var BULK_URL_THRESHOLD = 500;
 function reportSavingActive() {
   return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
 }
-function saveBulkSite(siteUrl, pages, seo) {
+function saveBulkSite(siteUrl, pages, seo, imageAudit) {
   if (!reportSavingActive()) return null;
   try {
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -2133,15 +2360,35 @@ function saveBulkSite(siteUrl, pages, seo) {
       (0, import_node_fs3.writeFileSync)((0, import_node_path4.join)(pagesDir, fname), content, "utf8");
       return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
     });
+    const dataFilesSection = seo ? [
+      `
+## Data files (in this folder)`,
+      `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
+      `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
+      `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
+      `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
+      `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      `- \`link-report.md\` \u2014 link analysis: top internal pages by inlinks, top external domains, distribution`,
+      `- \`links-summary.json\` \u2014 link totals + top-20 internal pages + top-20 external domains`,
+      `- \`external-domains.json\` \u2014 every external domain linked to, ordered by link count`,
+      ...imageAudit ? [
+        `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
+        `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
+      ] : [],
+      ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
+    ].join("\n") : "";
     const index = [
       `# Site Extract: ${siteUrl}`,
-      `**${pages.length} pages** \u2014 each saved as an individual Markdown file under \`pages/\`.`,
+      `**${pages.length} pages** crawled. Everything is in this folder.`,
+      `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
+      seo ? `- SEO crawl data: the data files listed below.` : "",
+      dataFilesSection,
       `
 ## Pages
 | # | Title | URL | File |
 |---|-------|-----|------|
 ${indexRows.join("\n")}`
-    ].join("\n");
+    ].filter(Boolean).join("\n");
     const indexFile = (0, import_node_path4.join)(dir, "index.md");
     (0, import_node_fs3.writeFileSync)(indexFile, index, "utf8");
     let seoFiles;
@@ -2156,8 +2403,23 @@ ${indexRows.join("\n")}`
       (0, import_node_fs3.writeFileSync)(linksJsonl, toJsonl(seo.edges), "utf8");
       (0, import_node_fs3.writeFileSync)(metricsJsonl, toJsonl(seo.metrics), "utf8");
       (0, import_node_fs3.writeFileSync)(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
-      (0, import_node_fs3.writeFileSync)(reportFile, seo.reportMd, "utf8");
-      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      (0, import_node_fs3.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
+
+${renderImageSection(imageAudit)}` : ""), "utf8");
+      const linkReportFile = (0, import_node_path4.join)(dir, "link-report.md");
+      const linksSummaryFile = (0, import_node_path4.join)(dir, "links-summary.json");
+      const externalDomainsFile = (0, import_node_path4.join)(dir, "external-domains.json");
+      (0, import_node_fs3.writeFileSync)(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
+      (0, import_node_fs3.writeFileSync)(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
+      (0, import_node_fs3.writeFileSync)(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
+      if (imageAudit) {
+        const imagesJsonl = (0, import_node_path4.join)(dir, "images.jsonl");
+        const imagesSummary = (0, import_node_path4.join)(dir, "images-summary.json");
+        (0, import_node_fs3.writeFileSync)(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
+        (0, import_node_fs3.writeFileSync)(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
+        seoFiles.push(imagesJsonl, imagesSummary);
+      }
       if (seo.branding) {
         const brandingFile = (0, import_node_path4.join)(dir, "branding.json");
         (0, import_node_fs3.writeFileSync)(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
@@ -2602,6 +2864,7 @@ function buildSeoExport(siteUrl, pages, branding) {
     metrics: [...metrics.values()],
     issues,
     reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    linkReport: buildLinkReport(edges, [...metrics.values()], siteUrl),
     branding: branding ?? void 0
   };
 }
@@ -2632,24 +2895,23 @@ function formatExtractSite(raw, input) {
     durationMs: d.durationMs ?? 0
   };
   if (pages.length > BULK_PAGE_THRESHOLD) {
-    const seo = buildSeoExport(input.url, pages, d.branding);
     const bulk = saveBulkSite(input.url, pages.map((p) => ({
       url: String(p.url ?? ""),
       title: p.title ?? null,
       bodyMarkdown: p.bodyMarkdown ?? null,
       metaDescription: p.metaDescription ?? null,
       schemaTypes: schemaTypesOf(p)
-    })), seo);
+    })));
     const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
-    const seoLine = bulk?.seoFiles?.length ? `
-- **SEO data:** \`report.md\` (issues summary), \`issues.json\` (findings + offender URLs), \`pages.jsonl\` (per-page fields), \`links.jsonl\` (link edges), \`link-metrics.jsonl\` (inlinks / crawl depth / orphans)` : "";
     const location = bulk ? `
 ## \u{1F4C1} Bulk scrape saved
-- **Folder:** \`${bulk.dir}\`
-- **Index:** \`${bulk.indexFile}\`
-- **Files:** ${bulk.fileCount} page files under \`pages/\`${seoLine}
+- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
+- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
+- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
 
-Read \`index.md\` for the full page list, open files in \`pages/\` for full content, or read \`pages.jsonl\` / \`link-metrics.jsonl\` for the SEO crawl data. Page bodies are not inlined here to keep the context window clear.` : `
+The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
+
+\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
 ## \u26A0\uFE0F Bulk scrape not saved
 Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
     const full2 = [
@@ -2693,6 +2955,62 @@ ${body}` : "_(no content extracted)_"
 ${pageDetails}${tips}`;
   return { ...oneBlock(full, diskReport), structuredContent };
 }
+async function formatAuditSite(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const pages = d.pages ?? [];
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const seo = buildSeoExport(input.url, pages, d.branding);
+  const imageAudit = await auditImages(pages, { concurrency: 12 });
+  const bulk = saveBulkSite(input.url, pages.map((p) => ({
+    url: String(p.url ?? ""),
+    title: p.title ?? null,
+    bodyMarkdown: p.bodyMarkdown ?? null,
+    metaDescription: p.metaDescription ?? null,
+    schemaTypes: schemaTypesOf(p)
+  })), seo, imageAudit);
+  const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
+  const img = imageAudit.summary;
+  const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const lr = seo.linkReport.summary;
+  const topExt = lr.external.topDomains.slice(0, 3).map((dm) => `${dm.domain} (${dm.links})`).join(", ") || "\u2014";
+  const linkLine = `${lr.internal.totalLinks} internal / ${lr.external.totalLinks} external links \xB7 ${lr.internal.orphans} orphans \xB7 ${lr.internal.brokenInternal} broken \xB7 avg ${lr.internal.avgInlinks} inlinks/page \xB7 top external: ${topExt}`;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    durationMs: d.durationMs ?? 0,
+    bulkFolder: bulk?.dir ?? null,
+    issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
+    links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
+  };
+  const location = bulk ? `
+## \u{1F4C1} Technical audit saved
+- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
+- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
+- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
+
+Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
+## \u26A0\uFE0F Audit not saved
+Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
+  const full = [
+    `# Technical SEO Audit: ${input.url}`,
+    durationLine,
+    location,
+    `
+## Top issues
+${topIssues || "_none found_"}`,
+    `
+## Links
+${linkLine}`,
+    `
+## Images
+${imgLine}`
+  ].join("\n");
+  return { content: [{ type: "text", text: full }], structuredContent };
+}
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -3932,12 +4250,18 @@ var MapSiteUrlsInputSchema = {
   maxUrls: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
 };
 var ExtractSiteInputSchema = {
-  url: import_zod3.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
+  url: import_zod3.z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
   maxPages: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
   rotateProxies: import_zod3.z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
   rotateProxyEvery: import_zod3.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
   formats: import_zod3.z.array(import_zod3.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
 };
+var AuditSiteInputSchema = {
+  url: import_zod3.z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
+  maxPages: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
+  rotateProxies: import_zod3.z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+  rotateProxyEvery: import_zod3.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
+};
 var YoutubeHarvestInputSchema = {
   mode: import_zod3.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
   query: import_zod3.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
@@ -4238,6 +4562,26 @@ var ExtractSiteOutputSchema = {
   })),
   durationMs: import_zod3.z.number().min(0)
 };
+var AuditSiteOutputSchema = {
+  url: import_zod3.z.string(),
+  pageCount: import_zod3.z.number().int().min(0),
+  durationMs: import_zod3.z.number().min(0),
+  bulkFolder: import_zod3.z.string().nullable(),
+  issues: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.number()),
+  images: import_zod3.z.object({
+    unique: import_zod3.z.number().int().min(0),
+    totalBytes: import_zod3.z.number().min(0),
+    over100kb: import_zod3.z.number().int().min(0),
+    legacyFormat: import_zod3.z.number().int().min(0)
+  }),
+  links: import_zod3.z.object({
+    internal: import_zod3.z.number().int().min(0),
+    external: import_zod3.z.number().int().min(0),
+    orphans: import_zod3.z.number().int().min(0),
+    brokenInternal: import_zod3.z.number().int().min(0),
+    externalDomains: import_zod3.z.number().int().min(0)
+  })
+};
 var MapsPlaceIntelOutputSchema = {
   name: import_zod3.z.string(),
   rating: NullableString2,
@@ -5074,12 +5418,19 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
     annotations: liveWebToolAnnotations("Site URL Map")
   }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
   server2.registerTool("extract_site", {
-    title: "Multi-Page Site Extract",
-    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all page content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
+    title: "Multi-Page Site Content Crawl",
+    description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
     inputSchema: ExtractSiteInputSchema,
     outputSchema: ExtractSiteOutputSchema,
-    annotations: liveWebToolAnnotations("Multi-Page Site Extract")
+    annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
+  server2.registerTool("audit_site", {
+    title: "Technical SEO Audit",
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph (inlinks/orphans/crawl-depth), a link analysis (top internal pages by inlinks, top external domains by link count, broken/orphan counts), an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, link-report.md, links-summary.json, external-domains.json, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    inputSchema: AuditSiteInputSchema,
+    outputSchema: AuditSiteOutputSchema,
+    annotations: liveWebToolAnnotations("Technical SEO Audit")
+  }, async (input) => formatAuditSite(await executor.auditSite(input), input));
   server2.registerTool("youtube_harvest", {
     title: "YouTube Video Harvest",
     description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -5273,9 +5624,9 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(44 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(45 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
-    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
+    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
     toolRow("build", ["rank_tracker_workflow", "cron plan", "database prompt"], color),
     toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
     toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_profile_status", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),