mcp-scraper 0.3.15 → 0.3.17

package/dist/bin/mcp-stdio-server.js

@@ -2,11 +2,11 @@
 import {
   HttpMcpToolExecutor,
   buildPaaExtractorMcpServer
-} from "../chunk-WQAIOY2D.js";
+} from "../chunk-N33YEY6W.js";
 import "../chunk-M2S27J6Z.js";
 import "../chunk-WN7PBKMV.js";
 import "../chunk-LFATOGDF.js";
-import "../chunk-2MX5WOII.js";
+import "../chunk-PVWWY5NV.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";

package/dist/{chunk-7Y6JDBML.js → chunk-ACIUCZ27.js}

@@ -50,9 +50,9 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(44 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(45 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
-    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
+    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
     toolRow("build", ["rank_tracker_workflow", "cron plan", "database prompt"], color),
     toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
     toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_profile_status", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
@@ -100,4 +100,4 @@ function renderInstallTerminal(options) {
 export {
   renderInstallTerminal
 };
-//# sourceMappingURL=chunk-7Y6JDBML.js.map
+//# sourceMappingURL=chunk-ACIUCZ27.js.map

package/dist/chunk-ACIUCZ27.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(45 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download'], color),\n    toolRow('browser', ['browser_open', 'browser_profile_onboard', 'browser_profile_status', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper-combined in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Customer auth setup: run browser_profile_onboard, send the watch_url, let the user sign in, then call browser_profile_status until AUTHENTICATED.',\n    'Reconnect setup: if a saved profile shows login controls later, run browser_profile_onboard again with the same profile name.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper-combined so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_onboard with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_status, then browser_open with the returned profile.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper-combined prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,0LAA0L,QAAQ,KAAK;AAAA,IAChN;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,kBAAkB,SAAS,KAAK,CAAC;AAAA,IAClF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,0BAA0B,GAAG,KAAK;AAAA,IAC9N,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,0BAA0B,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IAC1N,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}

package/dist/{chunk-WQAIOY2D.js → chunk-N33YEY6W.js}

@@ -11,7 +11,7 @@ import {
 } from "./chunk-LFATOGDF.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2MX5WOII.js";
+} from "./chunk-PVWWY5NV.js";
 
 // src/harvest-timeout.ts
 var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -256,6 +256,99 @@ function buildLinkGraph(pages, startUrl) {
   return { edges, metrics };
 }
 
+// src/api/seo-link-report.ts
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function buildLinkReport(edges, metrics, siteUrl) {
+  let siteReg = "";
+  try {
+    siteReg = registrableDomain(new URL(siteUrl).hostname);
+  } catch {
+    siteReg = "";
+  }
+  const internalEdges = edges.filter((e) => e.internal);
+  const pages = metrics.length;
+  const orphans = metrics.filter((m) => m.orphan).length;
+  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
+  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
+  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
+  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+  for (const m of metrics) {
+    if (m.inlinks === 0) distribution.zero++;
+    else if (m.inlinks <= 2) distribution.oneToTwo++;
+    else if (m.inlinks <= 10) distribution.threeToTen++;
+    else distribution.elevenPlus++;
+  }
+  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
+  const domMap = /* @__PURE__ */ new Map();
+  let externalTotal = 0;
+  for (const e of edges) {
+    let domain;
+    try {
+      domain = registrableDomain(new URL(e.to).hostname);
+    } catch {
+      continue;
+    }
+    if (!domain || domain === siteReg) continue;
+    externalTotal++;
+    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+    d.links++;
+    if (e.nofollow) d.nofollow++;
+    d.pages.add(e.from);
+    domMap.set(domain, d);
+  }
+  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+  const round = (n) => Math.round(n * 10) / 10;
+  return {
+    summary: {
+      internal: {
+        totalLinks: internalEdges.length,
+        pages,
+        orphans,
+        brokenInternal,
+        avgInlinks: pages ? round(sumInlinks / pages) : 0,
+        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
+        distribution,
+        topByInlinks
+      },
+      external: {
+        totalLinks: externalTotal,
+        uniqueDomains: externalDomains.length,
+        topDomains: externalDomains.slice(0, 20)
+      }
+    },
+    externalDomains
+  };
+}
+function renderLinkReport(r) {
+  const s = r.summary;
+  const lines = [
+    `## Link analysis`,
+    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
+    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
+    ``,
+    `### Inlink distribution`,
+    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
+    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
+    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
+    `- 11+: ${s.internal.distribution.elevenPlus}`,
+    ``,
+    `### Top 20 internal pages by inlinks`,
+    `| inlinks | out (int/ext) | URL |`,
+    `|---|---|---|`,
+    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
+    ``,
+    `### Top 20 external domains by links`,
+    `| links | nofollow | from pages | domain |`,
+    `|---|---|---|---|`,
+    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
+  ];
+  return lines.join("\n");
+}
+
 // src/api/seo-issues.ts
 var THIN_WORDS = 200;
 var TITLE_MAX = 60;
@@ -364,6 +457,137 @@ _Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1
   ].join("\n");
 }
 
+// src/api/image-audit.ts
+var OVER_BYTES = 100 * 1024;
+var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
+function formatBytes(n) {
+  if (n == null) return null;
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
+}
+var decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
+var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
+var formatOf = (ct, url) => {
+  if (ct) {
+    const m = ct.split(";")[0].trim().toLowerCase();
+    if (m.startsWith("image/")) return m.replace("image/", "");
+  }
+  return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
+};
+function collectUrls(pages) {
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of pages) for (const raw of p.imageLinks || []) {
+    const u = decodeEntities(raw);
+    if (!/^https?:\/\//i.test(u)) continue;
+    if (/[{}]|%7[bd]/i.test(u)) continue;
+    const k = dedupKey(u);
+    const prev = byKey.get(k);
+    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
+  }
+  return [...byKey.values()];
+}
+async function sizeAndType(url, timeoutMs) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  const read = (res) => ({
+    len: res.headers.get("content-length"),
+    cr: res.headers.get("content-range"),
+    ct: res.headers.get("content-type"),
+    status: res.status
+  });
+  try {
+    let bytes = null;
+    let ct = null;
+    let status = null;
+    try {
+      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
+      status = h.status;
+      ct = h.ct;
+      if (h.len) bytes = Number(h.len);
+    } catch {
+    }
+    if (bytes == null) {
+      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
+      const r = read(g);
+      status = status ?? r.status;
+      ct = ct ?? r.ct;
+      if (r.cr && r.cr.includes("/")) {
+        const tail = r.cr.split("/")[1];
+        if (tail && tail !== "*") bytes = Number(tail);
+      } else if (r.len) bytes = Number(r.len);
+      try {
+        await g.body?.cancel();
+      } catch {
+      }
+    }
+    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
+  } catch (e) {
+    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+async function pool(items, n, fn) {
+  const out = new Array(items.length);
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) {
+      const idx = i++;
+      out[idx] = await fn(items[idx]);
+    }
+  }));
+  return out;
+}
+async function auditImages(pages, opts = {}) {
+  const concurrency = opts.concurrency ?? 12;
+  const timeoutMs = opts.timeoutMs ?? 12e3;
+  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
+  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
+  const rows = heads.map((r) => {
+    const format = formatOf(r.contentType, r.url);
+    return {
+      ...r,
+      size: formatBytes(r.bytes),
+      format,
+      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
+      legacyFormat: format !== "unknown" && !MODERN.has(format)
+    };
+  });
+  const sized = rows.filter((r) => r.bytes != null);
+  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
+  const formatCounts = {};
+  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
+  return {
+    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
+    summary: {
+      unique: rows.length,
+      sized: sized.length,
+      totalBytes,
+      totalSize: formatBytes(totalBytes) ?? "0 B",
+      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
+      over100kb: rows.filter((r) => r.over100kb).length,
+      legacyFormat: rows.filter((r) => r.legacyFormat).length,
+      formatCounts
+    }
+  };
+}
+function renderImageSection(audit) {
+  const s = audit.summary;
+  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
+  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
+  const lines = [
+    `## Images`,
+    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
+    `Formats: ${fmt}`
+  ];
+  if (heaviest.length) {
+    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
+    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
+  }
+  return lines.join("\n");
+}
+
 // src/mcp/mcp-response-formatter.ts
 var reportSavingEnabled = true;
 function configureReportSaving(enabled) {
@@ -402,7 +626,7 @@ var BULK_URL_THRESHOLD = 500;
 function reportSavingActive() {
   return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
 }
-function saveBulkSite(siteUrl, pages, seo) {
+function saveBulkSite(siteUrl, pages, seo, imageAudit) {
   if (!reportSavingActive()) return null;
   try {
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -425,15 +649,35 @@ function saveBulkSite(siteUrl, pages, seo) {
       writeFileSync(join(pagesDir, fname), content, "utf8");
       return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
     });
+    const dataFilesSection = seo ? [
+      `
+## Data files (in this folder)`,
+      `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
+      `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
+      `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
+      `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
+      `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      `- \`link-report.md\` \u2014 link analysis: top internal pages by inlinks, top external domains, distribution`,
+      `- \`links-summary.json\` \u2014 link totals + top-20 internal pages + top-20 external domains`,
+      `- \`external-domains.json\` \u2014 every external domain linked to, ordered by link count`,
+      ...imageAudit ? [
+        `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
+        `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
+      ] : [],
+      ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
+    ].join("\n") : "";
     const index = [
       `# Site Extract: ${siteUrl}`,
-      `**${pages.length} pages** \u2014 each saved as an individual Markdown file under \`pages/\`.`,
+      `**${pages.length} pages** crawled. Everything is in this folder.`,
+      `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
+      seo ? `- SEO crawl data: the data files listed below.` : "",
+      dataFilesSection,
       `
 ## Pages
 | # | Title | URL | File |
 |---|-------|-----|------|
 ${indexRows.join("\n")}`
-    ].join("\n");
+    ].filter(Boolean).join("\n");
     const indexFile = join(dir, "index.md");
     writeFileSync(indexFile, index, "utf8");
     let seoFiles;
@@ -448,8 +692,23 @@ ${indexRows.join("\n")}`
       writeFileSync(linksJsonl, toJsonl(seo.edges), "utf8");
       writeFileSync(metricsJsonl, toJsonl(seo.metrics), "utf8");
       writeFileSync(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
-      writeFileSync(reportFile, seo.reportMd, "utf8");
-      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      writeFileSync(reportFile, seo.reportMd + (imageAudit ? `
+
+${renderImageSection(imageAudit)}` : ""), "utf8");
+      const linkReportFile = join(dir, "link-report.md");
+      const linksSummaryFile = join(dir, "links-summary.json");
+      const externalDomainsFile = join(dir, "external-domains.json");
+      writeFileSync(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
+      writeFileSync(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
+      writeFileSync(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
+      if (imageAudit) {
+        const imagesJsonl = join(dir, "images.jsonl");
+        const imagesSummary = join(dir, "images-summary.json");
+        writeFileSync(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
+        writeFileSync(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
+        seoFiles.push(imagesJsonl, imagesSummary);
+      }
       if (seo.branding) {
         const brandingFile = join(dir, "branding.json");
         writeFileSync(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
@@ -894,6 +1153,7 @@ function buildSeoExport(siteUrl, pages, branding) {
     metrics: [...metrics.values()],
     issues,
     reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    linkReport: buildLinkReport(edges, [...metrics.values()], siteUrl),
     branding: branding ?? void 0
   };
 }
@@ -924,24 +1184,23 @@ function formatExtractSite(raw, input) {
     durationMs: d.durationMs ?? 0
   };
   if (pages.length > BULK_PAGE_THRESHOLD) {
-    const seo = buildSeoExport(input.url, pages, d.branding);
     const bulk = saveBulkSite(input.url, pages.map((p) => ({
       url: String(p.url ?? ""),
       title: p.title ?? null,
       bodyMarkdown: p.bodyMarkdown ?? null,
       metaDescription: p.metaDescription ?? null,
       schemaTypes: schemaTypesOf(p)
-    })), seo);
+    })));
     const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
-    const seoLine = bulk?.seoFiles?.length ? `
-- **SEO data:** \`report.md\` (issues summary), \`issues.json\` (findings + offender URLs), \`pages.jsonl\` (per-page fields), \`links.jsonl\` (link edges), \`link-metrics.jsonl\` (inlinks / crawl depth / orphans)` : "";
     const location = bulk ? `
 ## \u{1F4C1} Bulk scrape saved
-- **Folder:** \`${bulk.dir}\`
-- **Index:** \`${bulk.indexFile}\`
-- **Files:** ${bulk.fileCount} page files under \`pages/\`${seoLine}
+- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
+- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
+- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
 
-Read \`index.md\` for the full page list, open files in \`pages/\` for full content, or read \`pages.jsonl\` / \`link-metrics.jsonl\` for the SEO crawl data. Page bodies are not inlined here to keep the context window clear.` : `
+The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
+
+\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
 ## \u26A0\uFE0F Bulk scrape not saved
 Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
     const full2 = [
@@ -985,6 +1244,62 @@ ${body}` : "_(no content extracted)_"
 ${pageDetails}${tips}`;
   return { ...oneBlock(full, diskReport), structuredContent };
 }
+async function formatAuditSite(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const pages = d.pages ?? [];
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const seo = buildSeoExport(input.url, pages, d.branding);
+  const imageAudit = await auditImages(pages, { concurrency: 12 });
+  const bulk = saveBulkSite(input.url, pages.map((p) => ({
+    url: String(p.url ?? ""),
+    title: p.title ?? null,
+    bodyMarkdown: p.bodyMarkdown ?? null,
+    metaDescription: p.metaDescription ?? null,
+    schemaTypes: schemaTypesOf(p)
+  })), seo, imageAudit);
+  const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
+  const img = imageAudit.summary;
+  const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const lr = seo.linkReport.summary;
+  const topExt = lr.external.topDomains.slice(0, 3).map((dm) => `${dm.domain} (${dm.links})`).join(", ") || "\u2014";
+  const linkLine = `${lr.internal.totalLinks} internal / ${lr.external.totalLinks} external links \xB7 ${lr.internal.orphans} orphans \xB7 ${lr.internal.brokenInternal} broken \xB7 avg ${lr.internal.avgInlinks} inlinks/page \xB7 top external: ${topExt}`;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    durationMs: d.durationMs ?? 0,
+    bulkFolder: bulk?.dir ?? null,
+    issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
+    links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
+  };
+  const location = bulk ? `
+## \u{1F4C1} Technical audit saved
+- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
+- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
+- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
+
+Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
+## \u26A0\uFE0F Audit not saved
+Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
+  const full = [
+    `# Technical SEO Audit: ${input.url}`,
+    durationLine,
+    location,
+    `
+## Top issues
+${topIssues || "_none found_"}`,
+    `
+## Links
+${linkLine}`,
+    `
+## Images
+${imgLine}`
+  ].join("\n");
+  return { content: [{ type: "text", text: full }], structuredContent };
+}
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -2187,12 +2502,18 @@ var MapSiteUrlsInputSchema = {
   maxUrls: z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
 };
 var ExtractSiteInputSchema = {
-  url: z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
+  url: z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
   maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
   rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
   rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
   formats: z.array(z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
 };
+var AuditSiteInputSchema = {
+  url: z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
+  maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
+  rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+  rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
+};
 var YoutubeHarvestInputSchema = {
   mode: z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
   query: z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
@@ -2493,6 +2814,26 @@ var ExtractSiteOutputSchema = {
   })),
   durationMs: z.number().min(0)
 };
+var AuditSiteOutputSchema = {
+  url: z.string(),
+  pageCount: z.number().int().min(0),
+  durationMs: z.number().min(0),
+  bulkFolder: z.string().nullable(),
+  issues: z.record(z.string(), z.number()),
+  images: z.object({
+    unique: z.number().int().min(0),
+    totalBytes: z.number().min(0),
+    over100kb: z.number().int().min(0),
+    legacyFormat: z.number().int().min(0)
+  }),
+  links: z.object({
+    internal: z.number().int().min(0),
+    external: z.number().int().min(0),
+    orphans: z.number().int().min(0),
+    brokenInternal: z.number().int().min(0),
+    externalDomains: z.number().int().min(0)
+  })
+};
 var MapsPlaceIntelOutputSchema = {
   name: z.string(),
   rating: NullableString,
@@ -3334,12 +3675,19 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     annotations: liveWebToolAnnotations("Site URL Map")
   }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
   server.registerTool("extract_site", {
-    title: "Multi-Page Site Extract",
-    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all page content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
+    title: "Multi-Page Site Content Crawl",
+    description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
     inputSchema: ExtractSiteInputSchema,
     outputSchema: ExtractSiteOutputSchema,
-    annotations: liveWebToolAnnotations("Multi-Page Site Extract")
+    annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
+  server.registerTool("audit_site", {
+    title: "Technical SEO Audit",
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph (inlinks/orphans/crawl-depth), a link analysis (top internal pages by inlinks, top external domains by link count, broken/orphan counts), an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, link-report.md, links-summary.json, external-domains.json, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    inputSchema: AuditSiteInputSchema,
+    outputSchema: AuditSiteOutputSchema,
+    annotations: liveWebToolAnnotations("Technical SEO Audit")
+  }, async (input) => formatAuditSite(await executor.auditSite(input), input));
   server.registerTool("youtube_harvest", {
     title: "YouTube Video Harvest",
     description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -3656,6 +4004,9 @@ var HttpMcpToolExecutor = class {
   extractSite(input) {
     return this.call("/extract-site", input);
   }
+  auditSite(input) {
+    return this.call("/extract-site", input);
+  }
   youtubeHarvest(input) {
     return this.call("/youtube/harvest", input);
   }
@@ -3744,8 +4095,12 @@ var HttpMcpToolExecutor = class {
 export {
   harvestTimeoutBudget,
   buildLinkGraph,
+  buildLinkReport,
+  renderLinkReport,
   computeIssues,
   renderIssueReport,
+  auditImages,
+  renderImageSection,
   configureReportSaving,
   outputBaseDir,
   formatCaptureSerpSnapshot,
@@ -3760,4 +4115,4 @@ export {
   registerPaaExtractorMcpTools,
   HttpMcpToolExecutor
 };
-//# sourceMappingURL=chunk-WQAIOY2D.js.map
+//# sourceMappingURL=chunk-N33YEY6W.js.map