mcp-scraper 0.3.14 → 0.3.16

package/dist/bin/mcp-scraper-combined-stdio-server.cjs

@@ -232,6 +232,9 @@ var HttpMcpToolExecutor = class {
   extractSite(input) {
     return this.call("/extract-site", input);
   }
+  auditSite(input) {
+    return this.call("/extract-site", input);
+  }
   youtubeHarvest(input) {
     return this.call("/youtube/harvest", input);
   }
@@ -453,7 +456,7 @@ render();
 }
 
 // src/version.ts
-var PACKAGE_VERSION = "0.3.14";
+var PACKAGE_VERSION = "0.3.16";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -604,7 +607,7 @@ var FanoutSourceOutput = import_zod.z.object({
 });
 var BrowserCaptureFanoutOutputSchema = {
   ...BrowserBaseOutput,
-  tool: import_zod.z.literal("browser_capture_fanout"),
+  tool: import_zod.z.literal("query_fanout_workflow"),
   platform: import_zod.z.enum(["ChatGPT", "Claude"]).describe("Which AI-search surface the fan-out was captured from."),
   captured_at: import_zod.z.string(),
   prompt: import_zod.z.string().describe("The user prompt that triggered the captured fan-out, when recoverable."),
@@ -1703,7 +1706,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     }
   );
   server2.registerTool(
-    "browser_capture_fanout",
+    "query_fanout_workflow",
     {
       title: "Capture AI Search Fan-Out",
       description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. Set export=true for durable JSON, CSV, TSV, and HTML artifacts; export paths are returned relative to MCP_SCRAPER_OUTPUT_DIR. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them complete login, then call browser_profile_status until status is AUTHENTICATED. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
@@ -1714,7 +1717,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     async (input) => {
       const emit = (result, exports3) => structuredResult({
         ok: true,
-        tool: "browser_capture_fanout",
+        tool: "query_fanout_workflow",
         session_id: input.session_id,
         platform: result.platform,
         captured_at: result.capturedAt,
@@ -1738,7 +1741,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
         reset: input.reset,
         export: input.export
       }, Math.max(timeoutMs, (input.wait_ms ?? (input.prompt ? 9e4 : 8e3)) + 3e4));
-      if (!res.ok) return errorResult("browser_capture_fanout", res.data, input.session_id);
+      if (!res.ok) return errorResult("query_fanout_workflow", res.data, input.session_id);
       const hosted = res.data?.result ?? res.data;
       let exports2 = res.data?.exports ?? null;
       if (input.export && !exports2 && isEnrichedFanout(hosted)) {
@@ -1746,7 +1749,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
           exports2 = exportFanout(hosted);
         } catch (err) {
           return errorResult(
-            "browser_capture_fanout",
+            "query_fanout_workflow",
             {
               error: `Fan-out captured but local export failed: ${err instanceof Error ? err.message : String(err)}`,
               result: hosted
@@ -1890,6 +1893,322 @@ function suggestWorkflowRecipes(goal, limit = 3) {
   return scored.sort((a, b) => b.score - a.score || a.recipe.title.localeCompare(b.recipe.title)).slice(0, Math.max(1, limit)).map((item) => item.recipe);
 }
 
+// src/api/seo-link-graph.ts
+function normalize2(u) {
+  return u.split("#")[0].replace(/\/$/, "");
+}
+function buildLinkGraph(pages, startUrl) {
+  const statusByUrl = /* @__PURE__ */ new Map();
+  for (const p of pages) statusByUrl.set(normalize2(p.url), p.status);
+  const edges = [];
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
+      edges.push({
+        from: p.url,
+        to: l.href,
+        anchor: l.anchor,
+        rel: l.rel,
+        internal: l.internal,
+        nofollow,
+        targetStatus: statusByUrl.get(normalize2(l.href)) ?? null
+      });
+    }
+  }
+  const inboundFrom = /* @__PURE__ */ new Map();
+  const inboundAnchors = /* @__PURE__ */ new Map();
+  const adjacency = /* @__PURE__ */ new Map();
+  for (const e of edges) {
+    if (!e.internal) continue;
+    const to = normalize2(e.to);
+    const from = normalize2(e.from);
+    if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
+    inboundFrom.get(to).add(from);
+    if (e.anchor) {
+      if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
+      inboundAnchors.get(to).push(e.anchor);
+    }
+    if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
+      if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
+      adjacency.get(from).add(to);
+    }
+  }
+  const depth = /* @__PURE__ */ new Map();
+  const start = normalize2(startUrl);
+  const queue = [start];
+  depth.set(start, 0);
+  while (queue.length > 0) {
+    const cur = queue.shift();
+    const d = depth.get(cur);
+    for (const next of adjacency.get(cur) ?? []) {
+      if (!depth.has(next)) {
+        depth.set(next, d + 1);
+        queue.push(next);
+      }
+    }
+  }
+  const topAnchorsFor = (url) => {
+    const counts = /* @__PURE__ */ new Map();
+    for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
+    return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
+  };
+  const metrics = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const key = normalize2(p.url);
+    const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
+    metrics.set(p.url, {
+      url: p.url,
+      inlinks: inbound.size,
+      uniqueInlinks: inbound.size,
+      outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
+      outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
+      crawlDepth: depth.has(key) ? depth.get(key) : null,
+      orphan: inbound.size === 0 && key !== start,
+      topAnchors: topAnchorsFor(key)
+    });
+  }
+  return { edges, metrics };
+}
+
+// src/api/seo-issues.ts
+var THIN_WORDS = 200;
+var TITLE_MAX = 60;
+var TITLE_PX_MAX = 561;
+var TITLE_MIN = 30;
+var META_MAX = 155;
+var META_MIN = 70;
+var H1_MAX = 70;
+var URL_MAX = 115;
+function dupes(pages, key) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const k = key(p);
+    if (k === null || k === void 0 || k === "") continue;
+    const ks = String(k);
+    if (!groups.has(ks)) groups.set(ks, []);
+    groups.get(ks).push(p.url);
+  }
+  return [...groups.values()].filter((g) => g.length > 1).flat();
+}
+function computeIssues(pages, metrics) {
+  const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
+  const where = (fn) => pages.filter(fn).map((p) => p.url);
+  const ok = (p) => p.status === 200;
+  const pathname = (u) => {
+    try {
+      return new URL(u).pathname;
+    } catch {
+      return "";
+    }
+  };
+  const report = {
+    "title.missing": group(where((p) => ok(p) && !p.title)),
+    "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
+    "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
+    "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
+    "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
+    "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
+    "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
+    "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
+    "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
+    "h1.missing": group(where((p) => ok(p) && !p.h1)),
+    "h1.multiple": group(where((p) => !!p.h1_2)),
+    "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
+    "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
+    "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
+    "indexability.nonIndexable": group(where((p) => !p.indexable)),
+    "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
+    "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
+    "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
+    "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
+    "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
+    "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
+    "response.noResponse": group(where((p) => p.status === null)),
+    "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
+    "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
+    "images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
+    "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
+    "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
+    "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
+    "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
+    "links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
+  };
+  const normUrl = (u) => {
+    try {
+      const x = new URL(u);
+      x.hash = "";
+      return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
+    } catch {
+      return u.replace(/#.*$/, "").replace(/\/+$/, "");
+    }
+  };
+  const statusByUrl = new Map(pages.map((p) => [normUrl(p.url), p.status]));
+  const brokenLinkPages = /* @__PURE__ */ new Set();
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      if (!l.internal) continue;
+      const st = statusByUrl.get(normUrl(l.href));
+      if (st != null && st >= 400) brokenLinkPages.add(p.url);
+    }
+  }
+  report["links.brokenInternal"] = group([...brokenLinkPages]);
+  return report;
+}
+function renderIssueReport(siteUrl, pages, report, metrics) {
+  const total = pages.length;
+  const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
+  const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
+  const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
+  const maxDepth = depths.length ? Math.max(...depths) : 0;
+  const orphans = [...metrics.values()].filter((m) => m.orphan).length;
+  const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
+  return [
+    `# SEO Crawl Report: ${siteUrl}`,
+    `**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
+    `
+## Issues
+| | Issue | Count | Examples |
+|---|-------|-------|----------|
+${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
+    `
+## Most-linked pages
+${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
+    `
+_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
+  ].join("\n");
+}
+
+// src/api/image-audit.ts
+var OVER_BYTES = 100 * 1024;
+var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
+function formatBytes(n) {
+  if (n == null) return null;
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
+}
+var decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
+var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
+var formatOf = (ct, url) => {
+  if (ct) {
+    const m = ct.split(";")[0].trim().toLowerCase();
+    if (m.startsWith("image/")) return m.replace("image/", "");
+  }
+  return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
+};
+function collectUrls(pages) {
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of pages) for (const raw of p.imageLinks || []) {
+    const u = decodeEntities(raw);
+    if (!/^https?:\/\//i.test(u)) continue;
+    if (/[{}]|%7[bd]/i.test(u)) continue;
+    const k = dedupKey(u);
+    const prev = byKey.get(k);
+    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
+  }
+  return [...byKey.values()];
+}
+async function sizeAndType(url, timeoutMs) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  const read = (res) => ({
+    len: res.headers.get("content-length"),
+    cr: res.headers.get("content-range"),
+    ct: res.headers.get("content-type"),
+    status: res.status
+  });
+  try {
+    let bytes = null;
+    let ct = null;
+    let status = null;
+    try {
+      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
+      status = h.status;
+      ct = h.ct;
+      if (h.len) bytes = Number(h.len);
+    } catch {
+    }
+    if (bytes == null) {
+      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
+      const r = read(g);
+      status = status ?? r.status;
+      ct = ct ?? r.ct;
+      if (r.cr && r.cr.includes("/")) {
+        const tail = r.cr.split("/")[1];
+        if (tail && tail !== "*") bytes = Number(tail);
+      } else if (r.len) bytes = Number(r.len);
+      try {
+        await g.body?.cancel();
+      } catch {
+      }
+    }
+    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
+  } catch (e) {
+    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+async function pool(items, n, fn) {
+  const out = new Array(items.length);
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) {
+      const idx = i++;
+      out[idx] = await fn(items[idx]);
+    }
+  }));
+  return out;
+}
+async function auditImages(pages, opts = {}) {
+  const concurrency = opts.concurrency ?? 12;
+  const timeoutMs = opts.timeoutMs ?? 12e3;
+  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
+  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
+  const rows = heads.map((r) => {
+    const format = formatOf(r.contentType, r.url);
+    return {
+      ...r,
+      size: formatBytes(r.bytes),
+      format,
+      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
+      legacyFormat: format !== "unknown" && !MODERN.has(format)
+    };
+  });
+  const sized = rows.filter((r) => r.bytes != null);
+  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
+  const formatCounts = {};
+  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
+  return {
+    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
+    summary: {
+      unique: rows.length,
+      sized: sized.length,
+      totalBytes,
+      totalSize: formatBytes(totalBytes) ?? "0 B",
+      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
+      over100kb: rows.filter((r) => r.over100kb).length,
+      legacyFormat: rows.filter((r) => r.legacyFormat).length,
+      formatCounts
+    }
+  };
+}
+function renderImageSection(audit) {
+  const s = audit.summary;
+  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
+  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
+  const lines = [
+    `## Images`,
+    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
+    `Formats: ${fmt}`
+  ];
+  if (heaviest.length) {
+    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
+    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
+  }
+  return lines.join("\n");
+}
+
 // src/mcp/mcp-response-formatter.ts
 var reportSavingEnabled = true;
 function sanitizeVendorText(text) {
@@ -1920,6 +2239,111 @@ function saveFullReport(full) {
     return null;
   }
 }
+var BULK_PAGE_THRESHOLD = 25;
+var BULK_URL_THRESHOLD = 500;
+function reportSavingActive() {
+  return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
+}
+function saveBulkSite(siteUrl, pages, seo, imageAudit) {
+  if (!reportSavingActive()) return null;
+  try {
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const dir = (0, import_node_path4.join)(outputBaseDir3(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
+    const pagesDir = (0, import_node_path4.join)(dir, "pages");
+    (0, import_node_fs3.mkdirSync)(pagesDir, { recursive: true });
+    const indexRows = pages.map((p, i) => {
+      const num = String(i + 1).padStart(4, "0");
+      const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
+      const fname = `${num}-${slug}.md`;
+      const body = (p.bodyMarkdown ?? "").trim();
+      const content = [
+        `# ${p.title ?? "Untitled"}`,
+        `- **URL:** ${p.url}`,
+        p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
+        p.schemaTypes?.length ? `- **Schema:** ${p.schemaTypes.join(", ")}` : "",
+        "",
+        body || "_(no content extracted)_"
+      ].filter(Boolean).join("\n");
+      (0, import_node_fs3.writeFileSync)((0, import_node_path4.join)(pagesDir, fname), content, "utf8");
+      return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
+    });
+    const dataFilesSection = seo ? [
+      `
+## Data files (in this folder)`,
+      `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
+      `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
+      `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
+      `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
+      `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      ...imageAudit ? [
+        `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
+        `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
+      ] : [],
+      ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
+    ].join("\n") : "";
+    const index = [
+      `# Site Extract: ${siteUrl}`,
+      `**${pages.length} pages** crawled. Everything is in this folder.`,
+      `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
+      seo ? `- SEO crawl data: the data files listed below.` : "",
+      dataFilesSection,
+      `
+## Pages
+| # | Title | URL | File |
+|---|-------|-----|------|
+${indexRows.join("\n")}`
+    ].filter(Boolean).join("\n");
+    const indexFile = (0, import_node_path4.join)(dir, "index.md");
+    (0, import_node_fs3.writeFileSync)(indexFile, index, "utf8");
+    let seoFiles;
+    if (seo) {
+      const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
+      const pagesJsonl = (0, import_node_path4.join)(dir, "pages.jsonl");
+      const linksJsonl = (0, import_node_path4.join)(dir, "links.jsonl");
+      const metricsJsonl = (0, import_node_path4.join)(dir, "link-metrics.jsonl");
+      const issuesFile = (0, import_node_path4.join)(dir, "issues.json");
+      const reportFile = (0, import_node_path4.join)(dir, "report.md");
+      (0, import_node_fs3.writeFileSync)(pagesJsonl, toJsonl(seo.pageRows), "utf8");
+      (0, import_node_fs3.writeFileSync)(linksJsonl, toJsonl(seo.edges), "utf8");
+      (0, import_node_fs3.writeFileSync)(metricsJsonl, toJsonl(seo.metrics), "utf8");
+      (0, import_node_fs3.writeFileSync)(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
+      (0, import_node_fs3.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
+
+${renderImageSection(imageAudit)}` : ""), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      if (imageAudit) {
+        const imagesJsonl = (0, import_node_path4.join)(dir, "images.jsonl");
+        const imagesSummary = (0, import_node_path4.join)(dir, "images-summary.json");
+        (0, import_node_fs3.writeFileSync)(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
+        (0, import_node_fs3.writeFileSync)(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
+        seoFiles.push(imagesJsonl, imagesSummary);
+      }
+      if (seo.branding) {
+        const brandingFile = (0, import_node_path4.join)(dir, "branding.json");
+        (0, import_node_fs3.writeFileSync)(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
+        seoFiles.push(brandingFile);
+      }
+    }
+    return { dir, indexFile, fileCount: pages.length, seoFiles };
+  } catch {
+    return null;
+  }
+}
+function saveUrlInventory(siteUrl, urls) {
+  if (!reportSavingActive()) return null;
+  try {
+    const outDir = outputBaseDir3();
+    (0, import_node_fs3.mkdirSync)(outDir, { recursive: true });
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const file = (0, import_node_path4.join)(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
+    const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
+    const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
+    (0, import_node_fs3.writeFileSync)(file, rows.join("\n"), "utf8");
+    return file;
+  } catch {
+    return null;
+  }
+}
 function persistScreenshotLocally(base64, url) {
   if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
   try {
@@ -1934,8 +2358,8 @@ function persistScreenshotLocally(base64, url) {
     return null;
   }
 }
-function oneBlock(content) {
-  const filePath = saveFullReport(content);
+function oneBlock(content, diskContent) {
+  const filePath = saveFullReport(diskContent ?? content);
   const text = filePath ? `${content}
 
 \u{1F4C4} Saved: \`${filePath}\`` : content;
@@ -2204,7 +2628,12 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
   ].filter(Boolean).join("\n") : "";
   const bodySection = bodyMd ? `
 ## Page Content
-${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
+${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? `
+
+*(truncated to 3,000 of ${bodyMd.length.toLocaleString()} chars \u2014 full content in the saved report)*` : ""}` : "";
+  const bodySectionFull = bodyMd ? `
+## Page Content
+${bodyMd}` : "";
   const screenshotSection = screenshotMeta ? `
 ## Screenshot
 - **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
@@ -2235,7 +2664,10 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
   const full = `# URL Extract: ${url}
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
-  const textResult = oneBlock(full);
+  const diskReport = `# URL Extract: ${url}
+**${title}**
+${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${screenshotSection}${mediaSection}${tips}`;
+  const textResult = oneBlock(full, diskReport);
   const structuredContent = {
     url,
     title: d.title ?? null,
@@ -2268,7 +2700,16 @@ function formatMapSiteUrls(raw, input) {
   const ok = urls.filter((u) => (u.status ?? 0) >= 200 && (u.status ?? 0) < 300);
   const broken = urls.filter((u) => u.status !== null && u.status >= 400);
   const redirects = urls.filter((u) => u.status !== null && u.status >= 300 && u.status < 400);
-  const urlRows = urls.slice(0, 200).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
+  const isBulk = urls.length > BULK_URL_THRESHOLD;
+  const inventoryFile = isBulk ? saveUrlInventory(input.url, urls.map((u) => ({ url: u.url, status: u.status ?? null }))) : null;
+  const inlineCount = isBulk ? BULK_URL_THRESHOLD : 200;
+  const urlRows = urls.slice(0, inlineCount).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
+  const inventoryNote = isBulk ? inventoryFile ? `
+## \u{1F4C1} Full inventory saved
+- **File:** \`${inventoryFile}\` (CSV: url, status \u2014 all ${urls.length} URLs)
+Showing the first ${inlineCount} below; read the file for the complete list.` : `
+## \u26A0\uFE0F Full inventory not saved
+Report saving is disabled, so only the first ${inlineCount} of ${urls.length} URLs are shown. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the full inventory.` : "";
   const full = [
     `# URL Map: ${input.url}`,
     `**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s${d.truncated ? " \xB7 *truncated*" : ""}`,
@@ -2277,12 +2718,13 @@ function formatMapSiteUrls(raw, input) {
 - \u2705 2xx: ${ok.length}
 - \u{1F500} 3xx: ${redirects.length}
 - \u274C 4xx+: ${broken.length}`,
+    inventoryNote,
     `
-## URL Inventory
+## URL Inventory${isBulk ? ` (first ${inlineCount} of ${urls.length})` : ""}
 | # | URL | Status |
 |---|-----|--------|
 ${urlRows}`,
-    broken.length ? `
+    !isBulk && broken.length ? `
 ## Broken URLs
 ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
     `
@@ -2300,47 +2742,164 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
       okCount: ok.length,
       redirectCount: redirects.length,
       brokenCount: broken.length,
+      inventoryFile: inventoryFile ?? null,
       urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
       durationMs: d.durationMs ?? 0
     }
   };
 }
+function buildSeoExport(siteUrl, pages, branding) {
+  const { edges, metrics } = buildLinkGraph(pages, siteUrl);
+  const pageRows = pages.map((p) => {
+    const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
+    const m = metrics.get(p.url);
+    return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
+  });
+  const issues = computeIssues(pages, metrics);
+  return {
+    pageRows,
+    edges,
+    metrics: [...metrics.values()],
+    issues,
+    reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    branding: branding ?? void 0
+  };
+}
 function formatExtractSite(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
   const d = parsed.data;
   const pages = d.pages ?? [];
-  const pageRows = pages.map((p, i) => {
-    const schemaInfo = p.kpo?.type?.join(", ") ?? (Array.isArray(p.schema) && p.schema.length ? `${p.schema.length} block(s)` : "\u2014");
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const pageRow = (p, i) => {
+    const schemaInfo = schemaTypesOf(p).join(", ") || "\u2014";
     return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | ${schemaInfo} |`;
-  }).join("\n");
-  const full = [
+  };
+  const tips = `
+---
+\u{1F4A1} **Tips**
+- Map URLs first: use \`map_site_urls\`
+- Inspect a single page: use \`extract_url\``;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    pages: pages.map((p) => ({
+      url: String(p.url ?? ""),
+      title: p.title ?? null,
+      schemaTypes: schemaTypesOf(p)
+    })),
+    durationMs: d.durationMs ?? 0
+  };
+  if (pages.length > BULK_PAGE_THRESHOLD) {
+    const bulk = saveBulkSite(input.url, pages.map((p) => ({
+      url: String(p.url ?? ""),
+      title: p.title ?? null,
+      bodyMarkdown: p.bodyMarkdown ?? null,
+      metaDescription: p.metaDescription ?? null,
+      schemaTypes: schemaTypesOf(p)
+    })));
+    const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
+    const location = bulk ? `
+## \u{1F4C1} Bulk scrape saved
+- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
+- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
+- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
+
+The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
+
+\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
+## \u26A0\uFE0F Bulk scrape not saved
+Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
+    const full2 = [
+      `# Site Extract: ${input.url}`,
+      durationLine,
+      location,
+      `
+## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
+| # | Title | URL | Schema |
+|---|-------|-----|--------|
+${preview}`,
+      tips
+    ].join("\n");
+    return { content: [{ type: "text", text: full2 }], structuredContent: { ...structuredContent, bulkFolder: bulk?.dir ?? null } };
+  }
+  const header = [
     `# Site Extract: ${input.url}`,
-    `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`,
+    durationLine,
     `
 ## Pages
 | # | Title | URL | Schema |
 |---|-------|-----|--------|
-${pageRows}`,
-    `
----
-\u{1F4A1} **Tips**
-- Map URLs first: use \`map_site_urls\`
-- Inspect a single page: use \`extract_url\``
+${pages.map(pageRow).join("\n")}`
   ].join("\n");
-  return {
-    ...oneBlock(full),
-    structuredContent: {
-      url: input.url,
-      pageCount: pages.length,
-      pages: pages.map((p) => ({
-        url: String(p.url ?? ""),
-        title: p.title ?? null,
-        schemaTypes: p.kpo?.type ?? []
-      })),
-      durationMs: d.durationMs ?? 0
-    }
+  const full = `${header}${tips}`;
+  const pageDetails = pages.map((p, i) => {
+    const body = (p.bodyMarkdown ?? "").trim();
+    return [
+      `
+## ${i + 1}. ${p.title ?? "Untitled"}`,
+      `- **URL:** ${p.url}`,
+      p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
+      body ? `
+${body}` : "_(no content extracted)_"
+    ].filter(Boolean).join("\n");
+  }).join("\n");
+  const diskReport = `${header}
+
+---
+# Full Page Content
+${pageDetails}${tips}`;
+  return { ...oneBlock(full, diskReport), structuredContent };
+}
+async function formatAuditSite(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const pages = d.pages ?? [];
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const seo = buildSeoExport(input.url, pages, d.branding);
+  const imageAudit = await auditImages(pages, { concurrency: 12 });
+  const bulk = saveBulkSite(input.url, pages.map((p) => ({
+    url: String(p.url ?? ""),
+    title: p.title ?? null,
+    bodyMarkdown: p.bodyMarkdown ?? null,
+    metaDescription: p.metaDescription ?? null,
+    schemaTypes: schemaTypesOf(p)
+  })), seo, imageAudit);
+  const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
+  const img = imageAudit.summary;
+  const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    durationMs: d.durationMs ?? 0,
+    bulkFolder: bulk?.dir ?? null,
+    issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
   };
+  const location = bulk ? `
+## \u{1F4C1} Technical audit saved
+- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
+- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
+- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
+
+Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
+## \u26A0\uFE0F Audit not saved
+Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
+  const full = [
+    `# Technical SEO Audit: ${input.url}`,
+    durationLine,
+    location,
+    `
+## Top issues
+${topIssues || "_none found_"}`,
+    `
+## Images
+${imgLine}`
+  ].join("\n");
+  return { content: [{ type: "text", text: full }], structuredContent };
 }
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
@@ -3427,6 +3986,33 @@ ${limitations.map((l) => `- ${l}`).join("\n")}` : "",
   };
 }
 
+// src/mcp/server-instructions.ts
+var SERVER_INSTRUCTIONS = `
+This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
+this routing map, then load its schema before calling.
+
+ROUTING
+- One web page -> extract_url. Whole site (crawl + SEO report) -> extract_site. Just the URL list -> map_site_urls.
+- Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail -> harvest_paa.
+- Google Maps: find places -> maps_search; one place deep-dive + reviews -> maps_place_intel.
+- YouTube: find or list videos -> youtube_harvest; transcribe one video -> youtube_transcribe.
+- Facebook: find ads -> facebook_ad_search; transcribe an ad -> facebook_ad_transcribe; transcribe a
+  video -> facebook_video_transcribe; page profile/intel -> facebook_page_intel.
+- Instagram: profile inventory -> instagram_profile_content; one post or reel -> instagram_media_download.
+- Workflows (multi-step): local directory build -> directory_workflow; rank blueprint -> rank_tracker_workflow;
+  AI-answer citation fan-out (AEO) -> query_fanout_workflow; deep research -> deep_research_workflow.
+- Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) -> the browser_* agent
+  (browser_open, then navigate/read); save/reuse logins via browser_profile_*.
+
+NOTES
+- Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
+  It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
+- Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
+  cheapest tool that answers the question (plain search/extract before browser agents).
+- Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
+  for full detail rather than expecting the whole payload inline.
+`.trim();
+
 // src/mcp/mcp-tool-schemas.ts
 var import_zod3 = require("zod");
 
@@ -3551,11 +4137,20 @@ var ExtractUrlInputSchema = {
 };
 var MapSiteUrlsInputSchema = {
   url: import_zod3.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
-  maxUrls: import_zod3.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
+  maxUrls: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
 };
 var ExtractSiteInputSchema = {
-  url: import_zod3.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
-  maxPages: import_zod3.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
+  url: import_zod3.z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
+  maxPages: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
+  rotateProxies: import_zod3.z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+  rotateProxyEvery: import_zod3.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
+  formats: import_zod3.z.array(import_zod3.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
+};
+var AuditSiteInputSchema = {
+  url: import_zod3.z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
+  maxPages: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
+  rotateProxies: import_zod3.z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+  rotateProxyEvery: import_zod3.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
 };
 var YoutubeHarvestInputSchema = {
   mode: import_zod3.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
@@ -3857,6 +4452,19 @@ var ExtractSiteOutputSchema = {
   })),
   durationMs: import_zod3.z.number().min(0)
 };
+var AuditSiteOutputSchema = {
+  url: import_zod3.z.string(),
+  pageCount: import_zod3.z.number().int().min(0),
+  durationMs: import_zod3.z.number().min(0),
+  bulkFolder: import_zod3.z.string().nullable(),
+  issues: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.number()),
+  images: import_zod3.z.object({
+    unique: import_zod3.z.number().int().min(0),
+    totalBytes: import_zod3.z.number().min(0),
+    over100kb: import_zod3.z.number().int().min(0),
+    legacyFormat: import_zod3.z.number().int().min(0)
+  })
+};
 var MapsPlaceIntelOutputSchema = {
   name: import_zod3.z.string(),
   rating: NullableString2,
@@ -4687,18 +5295,25 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
   }, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
   server2.registerTool("map_site_urls", {
     title: "Site URL Map",
-    description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
+    description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Supports up to maxUrls=10000; maps over 500 URLs write the complete inventory to a local CSV file and return a summary plus the file path instead of the full list inline. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
     inputSchema: MapSiteUrlsInputSchema,
     outputSchema: MapSiteUrlsOutputSchema,
     annotations: liveWebToolAnnotations("Site URL Map")
   }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
   server2.registerTool("extract_site", {
-    title: "Multi-Page Site Extract",
-    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
+    title: "Multi-Page Site Content Crawl",
+    description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
     inputSchema: ExtractSiteInputSchema,
     outputSchema: ExtractSiteOutputSchema,
-    annotations: liveWebToolAnnotations("Multi-Page Site Extract")
+    annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
+  server2.registerTool("audit_site", {
+    title: "Technical SEO Audit",
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    inputSchema: AuditSiteInputSchema,
+    outputSchema: AuditSiteOutputSchema,
+    annotations: liveWebToolAnnotations("Technical SEO Audit")
+  }, async (input) => formatAuditSite(await executor.auditSite(input), input));
   server2.registerTool("youtube_harvest", {
     title: "YouTube Video Harvest",
     description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -4818,7 +5433,7 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
     outputSchema: WorkflowArtifactReadOutputSchema,
     annotations: liveWebToolAnnotations("Read Workflow Artifact")
   }, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
-  server2.registerTool("rank_tracker_blueprint", {
+  server2.registerTool("rank_tracker_workflow", {
     title: "Rank Tracker Blueprint Builder",
     description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
     inputSchema: RankTrackerBlueprintInputSchema,
@@ -4892,10 +5507,10 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(44 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(45 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
-    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
-    toolRow("build", ["rank_tracker_blueprint", "cron plan", "database prompt"], color),
+    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
+    toolRow("build", ["rank_tracker_workflow", "cron plan", "database prompt"], color),
     toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
     toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_profile_status", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
     toolRow("account", ["credits_info", "reports", "MCP resources"], color),
@@ -4971,7 +5586,7 @@ if (!apiKey) {
 }
 var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
 var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-var server = new import_mcp3.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION });
+var server = new import_mcp3.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
 registerPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey));
 registerBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl });
 var transport = new import_stdio.StdioServerTransport();