mcp-scraper 0.3.15 → 0.3.17

package/dist/bin/api-server.cjs

@@ -9054,6 +9054,104 @@ var init_seo_link_graph = __esm({
   }
 });
 
+// src/api/seo-link-report.ts
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function buildLinkReport(edges, metrics, siteUrl) {
+  let siteReg = "";
+  try {
+    siteReg = registrableDomain(new URL(siteUrl).hostname);
+  } catch {
+    siteReg = "";
+  }
+  const internalEdges = edges.filter((e) => e.internal);
+  const pages = metrics.length;
+  const orphans = metrics.filter((m) => m.orphan).length;
+  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
+  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
+  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
+  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+  for (const m of metrics) {
+    if (m.inlinks === 0) distribution.zero++;
+    else if (m.inlinks <= 2) distribution.oneToTwo++;
+    else if (m.inlinks <= 10) distribution.threeToTen++;
+    else distribution.elevenPlus++;
+  }
+  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
+  const domMap = /* @__PURE__ */ new Map();
+  let externalTotal = 0;
+  for (const e of edges) {
+    let domain;
+    try {
+      domain = registrableDomain(new URL(e.to).hostname);
+    } catch {
+      continue;
+    }
+    if (!domain || domain === siteReg) continue;
+    externalTotal++;
+    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+    d.links++;
+    if (e.nofollow) d.nofollow++;
+    d.pages.add(e.from);
+    domMap.set(domain, d);
+  }
+  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+  const round = (n) => Math.round(n * 10) / 10;
+  return {
+    summary: {
+      internal: {
+        totalLinks: internalEdges.length,
+        pages,
+        orphans,
+        brokenInternal,
+        avgInlinks: pages ? round(sumInlinks / pages) : 0,
+        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
+        distribution,
+        topByInlinks
+      },
+      external: {
+        totalLinks: externalTotal,
+        uniqueDomains: externalDomains.length,
+        topDomains: externalDomains.slice(0, 20)
+      }
+    },
+    externalDomains
+  };
+}
+function renderLinkReport(r) {
+  const s = r.summary;
+  const lines = [
+    `## Link analysis`,
+    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
+    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
+    ``,
+    `### Inlink distribution`,
+    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
+    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
+    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
+    `- 11+: ${s.internal.distribution.elevenPlus}`,
+    ``,
+    `### Top 20 internal pages by inlinks`,
+    `| inlinks | out (int/ext) | URL |`,
+    `|---|---|---|`,
+    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
+    ``,
+    `### Top 20 external domains by links`,
+    `| links | nofollow | from pages | domain |`,
+    `|---|---|---|---|`,
+    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
+  ];
+  return lines.join("\n");
+}
+var init_seo_link_report = __esm({
+  "src/api/seo-link-report.ts"() {
+    "use strict";
+  }
+});
+
 // src/api/seo-issues.ts
 function dupes(pages, key) {
   const groups = /* @__PURE__ */ new Map();
@@ -9168,6 +9266,143 @@ var init_seo_issues = __esm({
   }
 });
 
+// src/api/image-audit.ts
+function formatBytes(n) {
+  if (n == null) return null;
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
+}
+function collectUrls(pages) {
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of pages) for (const raw of p.imageLinks || []) {
+    const u = decodeEntities(raw);
+    if (!/^https?:\/\//i.test(u)) continue;
+    if (/[{}]|%7[bd]/i.test(u)) continue;
+    const k = dedupKey(u);
+    const prev = byKey.get(k);
+    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
+  }
+  return [...byKey.values()];
+}
+async function sizeAndType(url, timeoutMs) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  const read = (res) => ({
+    len: res.headers.get("content-length"),
+    cr: res.headers.get("content-range"),
+    ct: res.headers.get("content-type"),
+    status: res.status
+  });
+  try {
+    let bytes = null;
+    let ct = null;
+    let status = null;
+    try {
+      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
+      status = h.status;
+      ct = h.ct;
+      if (h.len) bytes = Number(h.len);
+    } catch {
+    }
+    if (bytes == null) {
+      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
+      const r = read(g);
+      status = status ?? r.status;
+      ct = ct ?? r.ct;
+      if (r.cr && r.cr.includes("/")) {
+        const tail = r.cr.split("/")[1];
+        if (tail && tail !== "*") bytes = Number(tail);
+      } else if (r.len) bytes = Number(r.len);
+      try {
+        await g.body?.cancel();
+      } catch {
+      }
+    }
+    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
+  } catch (e) {
+    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+async function pool(items, n, fn) {
+  const out = new Array(items.length);
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) {
+      const idx = i++;
+      out[idx] = await fn(items[idx]);
+    }
+  }));
+  return out;
+}
+async function auditImages(pages, opts = {}) {
+  const concurrency = opts.concurrency ?? 12;
+  const timeoutMs = opts.timeoutMs ?? 12e3;
+  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
+  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
+  const rows = heads.map((r) => {
+    const format = formatOf(r.contentType, r.url);
+    return {
+      ...r,
+      size: formatBytes(r.bytes),
+      format,
+      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
+      legacyFormat: format !== "unknown" && !MODERN.has(format)
+    };
+  });
+  const sized = rows.filter((r) => r.bytes != null);
+  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
+  const formatCounts = {};
+  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
+  return {
+    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
+    summary: {
+      unique: rows.length,
+      sized: sized.length,
+      totalBytes,
+      totalSize: formatBytes(totalBytes) ?? "0 B",
+      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
+      over100kb: rows.filter((r) => r.over100kb).length,
+      legacyFormat: rows.filter((r) => r.legacyFormat).length,
+      formatCounts
+    }
+  };
+}
+function renderImageSection(audit) {
+  const s = audit.summary;
+  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
+  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
+  const lines = [
+    `## Images`,
+    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
+    `Formats: ${fmt}`
+  ];
+  if (heaviest.length) {
+    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
+    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
+  }
+  return lines.join("\n");
+}
+var OVER_BYTES, MODERN, decodeEntities, dedupKey, formatOf;
+var init_image_audit = __esm({
+  "src/api/image-audit.ts"() {
+    "use strict";
+    OVER_BYTES = 100 * 1024;
+    MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
+    decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
+    dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
+    formatOf = (ct, url) => {
+      if (ct) {
+        const m = ct.split(";")[0].trim().toLowerCase();
+        if (m.startsWith("image/")) return m.replace("image/", "");
+      }
+      return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
+    };
+  }
+});
+
 // src/api/blob-store.ts
 function byteLength(data) {
   return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
@@ -9582,10 +9817,14 @@ function chunk(items, size) {
   for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
   return out;
 }
-function buildArtifacts(job, pages, branding) {
+function buildArtifacts(job, pages, branding, imageAudit) {
   const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
   const issues = computeIssues(pages, metrics);
-  const reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  const linkReport = buildLinkReport(edges, [...metrics.values()], job.startUrl);
+  let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  if (imageAudit) reportMd += `
+
+${renderImageSection(imageAudit)}`;
   const pageRows = pages.map((p) => {
     const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
     const m = metrics.get(p.url);
@@ -9597,8 +9836,15 @@ function buildArtifacts(job, pages, branding) {
     { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
     { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
     { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
-    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
+    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" },
+    { name: "link-report.md", body: renderLinkReport(linkReport), type: "text/markdown" },
+    { name: "links-summary.json", body: JSON.stringify(linkReport.summary, null, 2), type: "application/json" },
+    { name: "external-domains.json", body: JSON.stringify(linkReport.externalDomains, null, 2), type: "application/json" }
   ];
+  if (imageAudit) {
+    files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
+    files.push({ name: "images-summary.json", body: JSON.stringify(imageAudit.summary, null, 2), type: "application/json" });
+  }
   if (branding) files.push({ name: "branding.json", body: JSON.stringify(branding, null, 2), type: "application/json" });
   return files;
 }
@@ -9609,7 +9855,9 @@ var init_site_extract = __esm({
     init_client();
     init_site_extractor();
     init_seo_link_graph();
+    init_seo_link_report();
     init_seo_issues();
+    init_image_audit();
     init_blob_store();
     init_screenshot();
     init_browser_service_env();
@@ -9657,10 +9905,14 @@ var init_site_extract = __esm({
           });
         }
         const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
+        const imageAudit = Array.isArray(job.options.formats) && job.options.formats.includes("images") ? await step.run("image-audit", async () => {
+          const pages = await getExtractedPages(jobId);
+          return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+        }) : null;
         const artifacts = await step.run("finalize", async () => {
           const pages = await getExtractedPages(jobId);
           const store = getBlobStore();
-          const files = buildArtifacts(job, pages, branding);
+          const files = buildArtifacts(job, pages, branding, imageAudit);
           const stored = [];
           for (const f of files) stored.push(await store.put(`extract/${jobId}/${f.name}`, f.body, f.type));
           await completeExtractJob(jobId, stored);
@@ -9935,7 +10187,7 @@ var init_catalog_seed = __esm({
               kind: "simple",
               endpoint: "/extract-site",
               mcpName: "extract_site",
-              tagline: "Map + bulk scrape + SEO report",
+              tagline: "Map + bulk scrape (content)",
               params: [
                 URL_PARAM,
                 { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
@@ -9943,7 +10195,21 @@ var init_catalog_seed = __esm({
                 { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true },
                 { key: "background", label: "Run in background", type: "boolean", default: true, advanced: true }
               ],
-              examples: ["https://steenshoney.com/ (full-site SEO crawl)"]
+              examples: ["https://steenshoney.com/ (full-site content crawl)"]
+            },
+            {
+              key: "audit_site",
+              label: "Audit",
+              kind: "simple",
+              endpoint: "/extract-site",
+              mcpName: "audit_site",
+              tagline: "Technical SEO audit (Screaming-Frog style)",
+              params: [
+                URL_PARAM,
+                { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
+                { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true }
+              ],
+              examples: ["https://steenshoney.com/ (issues + link graph + image audit)"]
             }
           ]
         },
@@ -16037,7 +16303,7 @@ function saveFullReport(full) {
 function reportSavingActive() {
   return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
 }
-function saveBulkSite(siteUrl, pages, seo) {
+function saveBulkSite(siteUrl, pages, seo, imageAudit) {
   if (!reportSavingActive()) return null;
   try {
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -16060,15 +16326,35 @@ function saveBulkSite(siteUrl, pages, seo) {
       (0, import_node_fs5.writeFileSync)((0, import_node_path7.join)(pagesDir, fname), content, "utf8");
       return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
     });
+    const dataFilesSection = seo ? [
+      `
+## Data files (in this folder)`,
+      `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
+      `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
+      `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
+      `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
+      `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      `- \`link-report.md\` \u2014 link analysis: top internal pages by inlinks, top external domains, distribution`,
+      `- \`links-summary.json\` \u2014 link totals + top-20 internal pages + top-20 external domains`,
+      `- \`external-domains.json\` \u2014 every external domain linked to, ordered by link count`,
+      ...imageAudit ? [
+        `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
+        `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
+      ] : [],
+      ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
+    ].join("\n") : "";
     const index = [
       `# Site Extract: ${siteUrl}`,
-      `**${pages.length} pages** \u2014 each saved as an individual Markdown file under \`pages/\`.`,
+      `**${pages.length} pages** crawled. Everything is in this folder.`,
+      `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
+      seo ? `- SEO crawl data: the data files listed below.` : "",
+      dataFilesSection,
       `
 ## Pages
 | # | Title | URL | File |
 |---|-------|-----|------|
 ${indexRows.join("\n")}`
-    ].join("\n");
+    ].filter(Boolean).join("\n");
     const indexFile = (0, import_node_path7.join)(dir, "index.md");
     (0, import_node_fs5.writeFileSync)(indexFile, index, "utf8");
     let seoFiles;
@@ -16083,8 +16369,23 @@ ${indexRows.join("\n")}`
       (0, import_node_fs5.writeFileSync)(linksJsonl, toJsonl(seo.edges), "utf8");
       (0, import_node_fs5.writeFileSync)(metricsJsonl, toJsonl(seo.metrics), "utf8");
       (0, import_node_fs5.writeFileSync)(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
-      (0, import_node_fs5.writeFileSync)(reportFile, seo.reportMd, "utf8");
-      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      (0, import_node_fs5.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
+
+${renderImageSection(imageAudit)}` : ""), "utf8");
+      const linkReportFile = (0, import_node_path7.join)(dir, "link-report.md");
+      const linksSummaryFile = (0, import_node_path7.join)(dir, "links-summary.json");
+      const externalDomainsFile = (0, import_node_path7.join)(dir, "external-domains.json");
+      (0, import_node_fs5.writeFileSync)(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
+      (0, import_node_fs5.writeFileSync)(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
+      (0, import_node_fs5.writeFileSync)(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
+      if (imageAudit) {
+        const imagesJsonl = (0, import_node_path7.join)(dir, "images.jsonl");
+        const imagesSummary = (0, import_node_path7.join)(dir, "images-summary.json");
+        (0, import_node_fs5.writeFileSync)(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
+        (0, import_node_fs5.writeFileSync)(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
+        seoFiles.push(imagesJsonl, imagesSummary);
+      }
       if (seo.branding) {
         const brandingFile = (0, import_node_path7.join)(dir, "branding.json");
         (0, import_node_fs5.writeFileSync)(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
@@ -16529,6 +16830,7 @@ function buildSeoExport(siteUrl, pages, branding) {
     metrics: [...metrics.values()],
     issues,
     reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    linkReport: buildLinkReport(edges, [...metrics.values()], siteUrl),
     branding: branding ?? void 0
   };
 }
@@ -16559,24 +16861,23 @@ function formatExtractSite(raw, input) {
     durationMs: d.durationMs ?? 0
   };
   if (pages.length > BULK_PAGE_THRESHOLD) {
-    const seo = buildSeoExport(input.url, pages, d.branding);
     const bulk = saveBulkSite(input.url, pages.map((p) => ({
       url: String(p.url ?? ""),
       title: p.title ?? null,
       bodyMarkdown: p.bodyMarkdown ?? null,
       metaDescription: p.metaDescription ?? null,
       schemaTypes: schemaTypesOf(p)
-    })), seo);
+    })));
     const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
-    const seoLine = bulk?.seoFiles?.length ? `
-- **SEO data:** \`report.md\` (issues summary), \`issues.json\` (findings + offender URLs), \`pages.jsonl\` (per-page fields), \`links.jsonl\` (link edges), \`link-metrics.jsonl\` (inlinks / crawl depth / orphans)` : "";
     const location2 = bulk ? `
 ## \u{1F4C1} Bulk scrape saved
-- **Folder:** \`${bulk.dir}\`
-- **Index:** \`${bulk.indexFile}\`
-- **Files:** ${bulk.fileCount} page files under \`pages/\`${seoLine}
+- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
+- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
+- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
+
+The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
 
-Read \`index.md\` for the full page list, open files in \`pages/\` for full content, or read \`pages.jsonl\` / \`link-metrics.jsonl\` for the SEO crawl data. Page bodies are not inlined here to keep the context window clear.` : `
+\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
 ## \u26A0\uFE0F Bulk scrape not saved
 Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
     const full2 = [
@@ -16620,6 +16921,62 @@ ${body}` : "_(no content extracted)_"
 ${pageDetails}${tips}`;
   return { ...oneBlock(full, diskReport), structuredContent };
 }
+async function formatAuditSite(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const pages = d.pages ?? [];
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const seo = buildSeoExport(input.url, pages, d.branding);
+  const imageAudit = await auditImages(pages, { concurrency: 12 });
+  const bulk = saveBulkSite(input.url, pages.map((p) => ({
+    url: String(p.url ?? ""),
+    title: p.title ?? null,
+    bodyMarkdown: p.bodyMarkdown ?? null,
+    metaDescription: p.metaDescription ?? null,
+    schemaTypes: schemaTypesOf(p)
+  })), seo, imageAudit);
+  const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
+  const img = imageAudit.summary;
+  const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const lr = seo.linkReport.summary;
+  const topExt = lr.external.topDomains.slice(0, 3).map((dm) => `${dm.domain} (${dm.links})`).join(", ") || "\u2014";
+  const linkLine = `${lr.internal.totalLinks} internal / ${lr.external.totalLinks} external links \xB7 ${lr.internal.orphans} orphans \xB7 ${lr.internal.brokenInternal} broken \xB7 avg ${lr.internal.avgInlinks} inlinks/page \xB7 top external: ${topExt}`;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    durationMs: d.durationMs ?? 0,
+    bulkFolder: bulk?.dir ?? null,
+    issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
+    links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
+  };
+  const location2 = bulk ? `
+## \u{1F4C1} Technical audit saved
+- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
+- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
+- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
+
+Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
+## \u26A0\uFE0F Audit not saved
+Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
+  const full = [
+    `# Technical SEO Audit: ${input.url}`,
+    durationLine,
+    location2,
+    `
+## Top issues
+${topIssues || "_none found_"}`,
+    `
+## Links
+${linkLine}`,
+    `
+## Images
+${imgLine}`
+  ].join("\n");
+  return { content: [{ type: "text", text: full }], structuredContent };
+}
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -17804,7 +18161,9 @@ var init_mcp_response_formatter = __esm({
     init_errors();
     init_workflow_catalog();
     init_seo_link_graph();
+    init_seo_link_report();
     init_seo_issues();
+    init_image_audit();
     reportSavingEnabled = true;
     BULK_PAGE_THRESHOLD = 25;
     BULK_URL_THRESHOLD = 500;
@@ -23339,7 +23698,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.15";
+    PACKAGE_VERSION = "0.3.17";
   }
 });
 
@@ -23377,7 +23736,7 @@ NOTES
 });
 
 // src/mcp/mcp-tool-schemas.ts
-var import_zod27, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
+var import_zod27, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
 var init_mcp_tool_schemas = __esm({
   "src/mcp/mcp-tool-schemas.ts"() {
     "use strict";
@@ -23408,12 +23767,18 @@ var init_mcp_tool_schemas = __esm({
       maxUrls: import_zod27.z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
     };
     ExtractSiteInputSchema = {
-      url: import_zod27.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
+      url: import_zod27.z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
       maxPages: import_zod27.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
       rotateProxies: import_zod27.z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
       rotateProxyEvery: import_zod27.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
       formats: import_zod27.z.array(import_zod27.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
     };
+    AuditSiteInputSchema = {
+      url: import_zod27.z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
+      maxPages: import_zod27.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
+      rotateProxies: import_zod27.z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+      rotateProxyEvery: import_zod27.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
+    };
     YoutubeHarvestInputSchema = {
       mode: import_zod27.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
       query: import_zod27.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
@@ -23714,6 +24079,26 @@ var init_mcp_tool_schemas = __esm({
       })),
       durationMs: import_zod27.z.number().min(0)
     };
+    AuditSiteOutputSchema = {
+      url: import_zod27.z.string(),
+      pageCount: import_zod27.z.number().int().min(0),
+      durationMs: import_zod27.z.number().min(0),
+      bulkFolder: import_zod27.z.string().nullable(),
+      issues: import_zod27.z.record(import_zod27.z.string(), import_zod27.z.number()),
+      images: import_zod27.z.object({
+        unique: import_zod27.z.number().int().min(0),
+        totalBytes: import_zod27.z.number().min(0),
+        over100kb: import_zod27.z.number().int().min(0),
+        legacyFormat: import_zod27.z.number().int().min(0)
+      }),
+      links: import_zod27.z.object({
+        internal: import_zod27.z.number().int().min(0),
+        external: import_zod27.z.number().int().min(0),
+        orphans: import_zod27.z.number().int().min(0),
+        brokenInternal: import_zod27.z.number().int().min(0),
+        externalDomains: import_zod27.z.number().int().min(0)
+      })
+    };
     MapsPlaceIntelOutputSchema = {
       name: import_zod27.z.string(),
       rating: NullableString,
@@ -24563,12 +24948,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     annotations: liveWebToolAnnotations("Site URL Map")
   }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
   server.registerTool("extract_site", {
-    title: "Multi-Page Site Extract",
-    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all page content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
+    title: "Multi-Page Site Content Crawl",
+    description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
     inputSchema: ExtractSiteInputSchema,
     outputSchema: ExtractSiteOutputSchema,
-    annotations: liveWebToolAnnotations("Multi-Page Site Extract")
+    annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
+  server.registerTool("audit_site", {
+    title: "Technical SEO Audit",
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph (inlinks/orphans/crawl-depth), a link analysis (top internal pages by inlinks, top external domains by link count, broken/orphan counts), an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, link-report.md, links-summary.json, external-domains.json, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    inputSchema: AuditSiteInputSchema,
+    outputSchema: AuditSiteOutputSchema,
+    annotations: liveWebToolAnnotations("Technical SEO Audit")
+  }, async (input) => formatAuditSite(await executor.auditSite(input), input));
   server.registerTool("youtube_harvest", {
     title: "YouTube Video Harvest",
     description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -24906,6 +25298,9 @@ var init_http_mcp_tool_executor = __esm({
       extractSite(input) {
         return this.call("/extract-site", input);
       }
+      auditSite(input) {
+        return this.call("/extract-site", input);
+      }
       youtubeHarvest(input) {
         return this.call("/youtube/harvest", input);
       }