mcp-scraper 0.3.13 → 0.3.15

package/dist/bin/mcp-scraper-combined-stdio-server.cjs

@@ -453,7 +453,7 @@ render();
 }
 
 // src/version.ts
-var PACKAGE_VERSION = "0.3.13";
+var PACKAGE_VERSION = "0.3.15";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -604,7 +604,7 @@ var FanoutSourceOutput = import_zod.z.object({
 });
 var BrowserCaptureFanoutOutputSchema = {
   ...BrowserBaseOutput,
-  tool: import_zod.z.literal("browser_capture_fanout"),
+  tool: import_zod.z.literal("query_fanout_workflow"),
   platform: import_zod.z.enum(["ChatGPT", "Claude"]).describe("Which AI-search surface the fan-out was captured from."),
   captured_at: import_zod.z.string(),
   prompt: import_zod.z.string().describe("The user prompt that triggered the captured fan-out, when recoverable."),
@@ -1145,13 +1145,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
   const baseUrl2 = opts.baseUrl.replace(/\/$/, "");
   const consoleBase = (opts.consoleBaseUrl ?? opts.baseUrl).replace(/\/$/, "");
   const timeoutMs = opts.timeoutMs ?? 9e4;
-  async function req(method, path, body) {
+  async function req(method, path, body, requestTimeoutMs = timeoutMs) {
     try {
       const res = await fetch(`${baseUrl2}${path}`, {
         method,
         headers: { "Content-Type": "application/json", "x-api-key": opts.apiKey },
         body: body ? JSON.stringify(body) : void 0,
-        signal: AbortSignal.timeout(timeoutMs)
+        signal: AbortSignal.timeout(requestTimeoutMs)
       });
       const data = await res.json().catch(() => ({}));
       return { ok: res.ok, data };
@@ -1703,7 +1703,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     }
   );
   server2.registerTool(
-    "browser_capture_fanout",
+    "query_fanout_workflow",
     {
       title: "Capture AI Search Fan-Out",
       description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. Set export=true for durable JSON, CSV, TSV, and HTML artifacts; export paths are returned relative to MCP_SCRAPER_OUTPUT_DIR. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them complete login, then call browser_profile_status until status is AUTHENTICATED. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
@@ -1714,7 +1714,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
     async (input) => {
       const emit = (result, exports3) => structuredResult({
         ok: true,
-        tool: "browser_capture_fanout",
+        tool: "query_fanout_workflow",
         session_id: input.session_id,
         platform: result.platform,
         captured_at: result.capturedAt,
@@ -1737,8 +1737,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
         first_party_domain: input.first_party_domain,
         reset: input.reset,
         export: input.export
-      });
-      if (!res.ok) return errorResult("browser_capture_fanout", res.data, input.session_id);
+      }, Math.max(timeoutMs, (input.wait_ms ?? (input.prompt ? 9e4 : 8e3)) + 3e4));
+      if (!res.ok) return errorResult("query_fanout_workflow", res.data, input.session_id);
       const hosted = res.data?.result ?? res.data;
       let exports2 = res.data?.exports ?? null;
       if (input.export && !exports2 && isEnrichedFanout(hosted)) {
@@ -1746,7 +1746,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
           exports2 = exportFanout(hosted);
         } catch (err) {
           return errorResult(
-            "browser_capture_fanout",
+            "query_fanout_workflow",
             {
               error: `Fan-out captured but local export failed: ${err instanceof Error ? err.message : String(err)}`,
               result: hosted
@@ -1890,6 +1890,191 @@ function suggestWorkflowRecipes(goal, limit = 3) {
   return scored.sort((a, b) => b.score - a.score || a.recipe.title.localeCompare(b.recipe.title)).slice(0, Math.max(1, limit)).map((item) => item.recipe);
 }
 
+// src/api/seo-link-graph.ts
+function normalize2(u) {
+  return u.split("#")[0].replace(/\/$/, "");
+}
+function buildLinkGraph(pages, startUrl) {
+  const statusByUrl = /* @__PURE__ */ new Map();
+  for (const p of pages) statusByUrl.set(normalize2(p.url), p.status);
+  const edges = [];
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
+      edges.push({
+        from: p.url,
+        to: l.href,
+        anchor: l.anchor,
+        rel: l.rel,
+        internal: l.internal,
+        nofollow,
+        targetStatus: statusByUrl.get(normalize2(l.href)) ?? null
+      });
+    }
+  }
+  const inboundFrom = /* @__PURE__ */ new Map();
+  const inboundAnchors = /* @__PURE__ */ new Map();
+  const adjacency = /* @__PURE__ */ new Map();
+  for (const e of edges) {
+    if (!e.internal) continue;
+    const to = normalize2(e.to);
+    const from = normalize2(e.from);
+    if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
+    inboundFrom.get(to).add(from);
+    if (e.anchor) {
+      if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
+      inboundAnchors.get(to).push(e.anchor);
+    }
+    if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
+      if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
+      adjacency.get(from).add(to);
+    }
+  }
+  const depth = /* @__PURE__ */ new Map();
+  const start = normalize2(startUrl);
+  const queue = [start];
+  depth.set(start, 0);
+  while (queue.length > 0) {
+    const cur = queue.shift();
+    const d = depth.get(cur);
+    for (const next of adjacency.get(cur) ?? []) {
+      if (!depth.has(next)) {
+        depth.set(next, d + 1);
+        queue.push(next);
+      }
+    }
+  }
+  const topAnchorsFor = (url) => {
+    const counts = /* @__PURE__ */ new Map();
+    for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
+    return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
+  };
+  const metrics = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const key = normalize2(p.url);
+    const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
+    metrics.set(p.url, {
+      url: p.url,
+      inlinks: inbound.size,
+      uniqueInlinks: inbound.size,
+      outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
+      outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
+      crawlDepth: depth.has(key) ? depth.get(key) : null,
+      orphan: inbound.size === 0 && key !== start,
+      topAnchors: topAnchorsFor(key)
+    });
+  }
+  return { edges, metrics };
+}
+
+// src/api/seo-issues.ts
+var THIN_WORDS = 200;
+var TITLE_MAX = 60;
+var TITLE_PX_MAX = 561;
+var TITLE_MIN = 30;
+var META_MAX = 155;
+var META_MIN = 70;
+var H1_MAX = 70;
+var URL_MAX = 115;
+function dupes(pages, key) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const k = key(p);
+    if (k === null || k === void 0 || k === "") continue;
+    const ks = String(k);
+    if (!groups.has(ks)) groups.set(ks, []);
+    groups.get(ks).push(p.url);
+  }
+  return [...groups.values()].filter((g) => g.length > 1).flat();
+}
+function computeIssues(pages, metrics) {
+  const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
+  const where = (fn) => pages.filter(fn).map((p) => p.url);
+  const ok = (p) => p.status === 200;
+  const pathname = (u) => {
+    try {
+      return new URL(u).pathname;
+    } catch {
+      return "";
+    }
+  };
+  const report = {
+    "title.missing": group(where((p) => ok(p) && !p.title)),
+    "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
+    "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
+    "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
+    "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
+    "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
+    "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
+    "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
+    "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
+    "h1.missing": group(where((p) => ok(p) && !p.h1)),
+    "h1.multiple": group(where((p) => !!p.h1_2)),
+    "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
+    "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
+    "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
+    "indexability.nonIndexable": group(where((p) => !p.indexable)),
+    "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
+    "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
+    "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
+    "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
+    "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
+    "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
+    "response.noResponse": group(where((p) => p.status === null)),
+    "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
+    "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
+    "images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
+    "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
+    "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
+    "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
+    "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
+    "links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
+  };
+  const normUrl = (u) => {
+    try {
+      const x = new URL(u);
+      x.hash = "";
+      return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
+    } catch {
+      return u.replace(/#.*$/, "").replace(/\/+$/, "");
+    }
+  };
+  const statusByUrl = new Map(pages.map((p) => [normUrl(p.url), p.status]));
+  const brokenLinkPages = /* @__PURE__ */ new Set();
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      if (!l.internal) continue;
+      const st = statusByUrl.get(normUrl(l.href));
+      if (st != null && st >= 400) brokenLinkPages.add(p.url);
+    }
+  }
+  report["links.brokenInternal"] = group([...brokenLinkPages]);
+  return report;
+}
+function renderIssueReport(siteUrl, pages, report, metrics) {
+  const total = pages.length;
+  const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
+  const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
+  const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
+  const maxDepth = depths.length ? Math.max(...depths) : 0;
+  const orphans = [...metrics.values()].filter((m) => m.orphan).length;
+  const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
+  return [
+    `# SEO Crawl Report: ${siteUrl}`,
+    `**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
+    `
+## Issues
+| | Issue | Count | Examples |
+|---|-------|-------|----------|
+${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
+    `
+## Most-linked pages
+${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
+    `
+_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
+  ].join("\n");
+}
+
 // src/mcp/mcp-response-formatter.ts
 var reportSavingEnabled = true;
 function sanitizeVendorText(text) {
@@ -1920,6 +2105,85 @@ function saveFullReport(full) {
     return null;
   }
 }
+var BULK_PAGE_THRESHOLD = 25;
+var BULK_URL_THRESHOLD = 500;
+function reportSavingActive() {
+  return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
+}
+function saveBulkSite(siteUrl, pages, seo) {
+  if (!reportSavingActive()) return null;
+  try {
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const dir = (0, import_node_path4.join)(outputBaseDir3(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
+    const pagesDir = (0, import_node_path4.join)(dir, "pages");
+    (0, import_node_fs3.mkdirSync)(pagesDir, { recursive: true });
+    const indexRows = pages.map((p, i) => {
+      const num = String(i + 1).padStart(4, "0");
+      const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
+      const fname = `${num}-${slug}.md`;
+      const body = (p.bodyMarkdown ?? "").trim();
+      const content = [
+        `# ${p.title ?? "Untitled"}`,
+        `- **URL:** ${p.url}`,
+        p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
+        p.schemaTypes?.length ? `- **Schema:** ${p.schemaTypes.join(", ")}` : "",
+        "",
+        body || "_(no content extracted)_"
+      ].filter(Boolean).join("\n");
+      (0, import_node_fs3.writeFileSync)((0, import_node_path4.join)(pagesDir, fname), content, "utf8");
+      return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
+    });
+    const index = [
+      `# Site Extract: ${siteUrl}`,
+      `**${pages.length} pages** \u2014 each saved as an individual Markdown file under \`pages/\`.`,
+      `
+## Pages
+| # | Title | URL | File |
+|---|-------|-----|------|
+${indexRows.join("\n")}`
+    ].join("\n");
+    const indexFile = (0, import_node_path4.join)(dir, "index.md");
+    (0, import_node_fs3.writeFileSync)(indexFile, index, "utf8");
+    let seoFiles;
+    if (seo) {
+      const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
+      const pagesJsonl = (0, import_node_path4.join)(dir, "pages.jsonl");
+      const linksJsonl = (0, import_node_path4.join)(dir, "links.jsonl");
+      const metricsJsonl = (0, import_node_path4.join)(dir, "link-metrics.jsonl");
+      const issuesFile = (0, import_node_path4.join)(dir, "issues.json");
+      const reportFile = (0, import_node_path4.join)(dir, "report.md");
+      (0, import_node_fs3.writeFileSync)(pagesJsonl, toJsonl(seo.pageRows), "utf8");
+      (0, import_node_fs3.writeFileSync)(linksJsonl, toJsonl(seo.edges), "utf8");
+      (0, import_node_fs3.writeFileSync)(metricsJsonl, toJsonl(seo.metrics), "utf8");
+      (0, import_node_fs3.writeFileSync)(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
+      (0, import_node_fs3.writeFileSync)(reportFile, seo.reportMd, "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      if (seo.branding) {
+        const brandingFile = (0, import_node_path4.join)(dir, "branding.json");
+        (0, import_node_fs3.writeFileSync)(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
+        seoFiles.push(brandingFile);
+      }
+    }
+    return { dir, indexFile, fileCount: pages.length, seoFiles };
+  } catch {
+    return null;
+  }
+}
+function saveUrlInventory(siteUrl, urls) {
+  if (!reportSavingActive()) return null;
+  try {
+    const outDir = outputBaseDir3();
+    (0, import_node_fs3.mkdirSync)(outDir, { recursive: true });
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const file = (0, import_node_path4.join)(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
+    const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
+    const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
+    (0, import_node_fs3.writeFileSync)(file, rows.join("\n"), "utf8");
+    return file;
+  } catch {
+    return null;
+  }
+}
 function persistScreenshotLocally(base64, url) {
   if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
   try {
@@ -1934,8 +2198,8 @@ function persistScreenshotLocally(base64, url) {
     return null;
   }
 }
-function oneBlock(content) {
-  const filePath = saveFullReport(content);
+function oneBlock(content, diskContent) {
+  const filePath = saveFullReport(diskContent ?? content);
   const text = filePath ? `${content}
 
 \u{1F4C4} Saved: \`${filePath}\`` : content;
@@ -2204,7 +2468,12 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
   ].filter(Boolean).join("\n") : "";
   const bodySection = bodyMd ? `
 ## Page Content
-${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
+${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? `
+
+*(truncated to 3,000 of ${bodyMd.length.toLocaleString()} chars \u2014 full content in the saved report)*` : ""}` : "";
+  const bodySectionFull = bodyMd ? `
+## Page Content
+${bodyMd}` : "";
   const screenshotSection = screenshotMeta ? `
 ## Screenshot
 - **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
@@ -2235,7 +2504,10 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
   const full = `# URL Extract: ${url}
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
-  const textResult = oneBlock(full);
+  const diskReport = `# URL Extract: ${url}
+**${title}**
+${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${screenshotSection}${mediaSection}${tips}`;
+  const textResult = oneBlock(full, diskReport);
   const structuredContent = {
     url,
     title: d.title ?? null,
@@ -2268,7 +2540,16 @@ function formatMapSiteUrls(raw, input) {
   const ok = urls.filter((u) => (u.status ?? 0) >= 200 && (u.status ?? 0) < 300);
   const broken = urls.filter((u) => u.status !== null && u.status >= 400);
   const redirects = urls.filter((u) => u.status !== null && u.status >= 300 && u.status < 400);
-  const urlRows = urls.slice(0, 200).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
+  const isBulk = urls.length > BULK_URL_THRESHOLD;
+  const inventoryFile = isBulk ? saveUrlInventory(input.url, urls.map((u) => ({ url: u.url, status: u.status ?? null }))) : null;
+  const inlineCount = isBulk ? BULK_URL_THRESHOLD : 200;
+  const urlRows = urls.slice(0, inlineCount).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
+  const inventoryNote = isBulk ? inventoryFile ? `
+## \u{1F4C1} Full inventory saved
+- **File:** \`${inventoryFile}\` (CSV: url, status \u2014 all ${urls.length} URLs)
+Showing the first ${inlineCount} below; read the file for the complete list.` : `
+## \u26A0\uFE0F Full inventory not saved
+Report saving is disabled, so only the first ${inlineCount} of ${urls.length} URLs are shown. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the full inventory.` : "";
   const full = [
     `# URL Map: ${input.url}`,
     `**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s${d.truncated ? " \xB7 *truncated*" : ""}`,
@@ -2277,12 +2558,13 @@ function formatMapSiteUrls(raw, input) {
 - \u2705 2xx: ${ok.length}
 - \u{1F500} 3xx: ${redirects.length}
 - \u274C 4xx+: ${broken.length}`,
+    inventoryNote,
     `
-## URL Inventory
+## URL Inventory${isBulk ? ` (first ${inlineCount} of ${urls.length})` : ""}
 | # | URL | Status |
 |---|-----|--------|
 ${urlRows}`,
-    broken.length ? `
+    !isBulk && broken.length ? `
 ## Broken URLs
 ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
     `
@@ -2300,47 +2582,116 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
       okCount: ok.length,
       redirectCount: redirects.length,
       brokenCount: broken.length,
+      inventoryFile: inventoryFile ?? null,
       urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
       durationMs: d.durationMs ?? 0
     }
   };
 }
+function buildSeoExport(siteUrl, pages, branding) {
+  const { edges, metrics } = buildLinkGraph(pages, siteUrl);
+  const pageRows = pages.map((p) => {
+    const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
+    const m = metrics.get(p.url);
+    return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
+  });
+  const issues = computeIssues(pages, metrics);
+  return {
+    pageRows,
+    edges,
+    metrics: [...metrics.values()],
+    issues,
+    reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    branding: branding ?? void 0
+  };
+}
 function formatExtractSite(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
   const d = parsed.data;
   const pages = d.pages ?? [];
-  const pageRows = pages.map((p, i) => {
-    const schemaInfo = p.kpo?.type?.join(", ") ?? (Array.isArray(p.schema) && p.schema.length ? `${p.schema.length} block(s)` : "\u2014");
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const pageRow = (p, i) => {
+    const schemaInfo = schemaTypesOf(p).join(", ") || "\u2014";
     return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | ${schemaInfo} |`;
-  }).join("\n");
-  const full = [
+  };
+  const tips = `
+---
+\u{1F4A1} **Tips**
+- Map URLs first: use \`map_site_urls\`
+- Inspect a single page: use \`extract_url\``;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    pages: pages.map((p) => ({
+      url: String(p.url ?? ""),
+      title: p.title ?? null,
+      schemaTypes: schemaTypesOf(p)
+    })),
+    durationMs: d.durationMs ?? 0
+  };
+  if (pages.length > BULK_PAGE_THRESHOLD) {
+    const seo = buildSeoExport(input.url, pages, d.branding);
+    const bulk = saveBulkSite(input.url, pages.map((p) => ({
+      url: String(p.url ?? ""),
+      title: p.title ?? null,
+      bodyMarkdown: p.bodyMarkdown ?? null,
+      metaDescription: p.metaDescription ?? null,
+      schemaTypes: schemaTypesOf(p)
+    })), seo);
+    const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
+    const seoLine = bulk?.seoFiles?.length ? `
+- **SEO data:** \`report.md\` (issues summary), \`issues.json\` (findings + offender URLs), \`pages.jsonl\` (per-page fields), \`links.jsonl\` (link edges), \`link-metrics.jsonl\` (inlinks / crawl depth / orphans)` : "";
+    const location = bulk ? `
+## \u{1F4C1} Bulk scrape saved
+- **Folder:** \`${bulk.dir}\`
+- **Index:** \`${bulk.indexFile}\`
+- **Files:** ${bulk.fileCount} page files under \`pages/\`${seoLine}
+
+Read \`index.md\` for the full page list, open files in \`pages/\` for full content, or read \`pages.jsonl\` / \`link-metrics.jsonl\` for the SEO crawl data. Page bodies are not inlined here to keep the context window clear.` : `
+## \u26A0\uFE0F Bulk scrape not saved
+Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
+    const full2 = [
+      `# Site Extract: ${input.url}`,
+      durationLine,
+      location,
+      `
+## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
+| # | Title | URL | Schema |
+|---|-------|-----|--------|
+${preview}`,
+      tips
+    ].join("\n");
+    return { content: [{ type: "text", text: full2 }], structuredContent: { ...structuredContent, bulkFolder: bulk?.dir ?? null } };
+  }
+  const header = [
     `# Site Extract: ${input.url}`,
-    `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`,
+    durationLine,
     `
 ## Pages
 | # | Title | URL | Schema |
 |---|-------|-----|--------|
-${pageRows}`,
-    `
----
-\u{1F4A1} **Tips**
-- Map URLs first: use \`map_site_urls\`
-- Inspect a single page: use \`extract_url\``
+${pages.map(pageRow).join("\n")}`
   ].join("\n");
-  return {
-    ...oneBlock(full),
-    structuredContent: {
-      url: input.url,
-      pageCount: pages.length,
-      pages: pages.map((p) => ({
-        url: String(p.url ?? ""),
-        title: p.title ?? null,
-        schemaTypes: p.kpo?.type ?? []
-      })),
-      durationMs: d.durationMs ?? 0
-    }
-  };
+  const full = `${header}${tips}`;
+  const pageDetails = pages.map((p, i) => {
+    const body = (p.bodyMarkdown ?? "").trim();
+    return [
+      `
+## ${i + 1}. ${p.title ?? "Untitled"}`,
+      `- **URL:** ${p.url}`,
+      p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
+      body ? `
+${body}` : "_(no content extracted)_"
+    ].filter(Boolean).join("\n");
+  }).join("\n");
+  const diskReport = `${header}
+
+---
+# Full Page Content
+${pageDetails}${tips}`;
+  return { ...oneBlock(full, diskReport), structuredContent };
 }
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
@@ -3427,6 +3778,33 @@ ${limitations.map((l) => `- ${l}`).join("\n")}` : "",
   };
 }
 
+// src/mcp/server-instructions.ts
+var SERVER_INSTRUCTIONS = `
+This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
+this routing map, then load its schema before calling.
+
+ROUTING
+- One web page -> extract_url. Whole site (crawl + SEO report) -> extract_site. Just the URL list -> map_site_urls.
+- Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail -> harvest_paa.
+- Google Maps: find places -> maps_search; one place deep-dive + reviews -> maps_place_intel.
+- YouTube: find or list videos -> youtube_harvest; transcribe one video -> youtube_transcribe.
+- Facebook: find ads -> facebook_ad_search; transcribe an ad -> facebook_ad_transcribe; transcribe a
+  video -> facebook_video_transcribe; page profile/intel -> facebook_page_intel.
+- Instagram: profile inventory -> instagram_profile_content; one post or reel -> instagram_media_download.
+- Workflows (multi-step): local directory build -> directory_workflow; rank blueprint -> rank_tracker_workflow;
+  AI-answer citation fan-out (AEO) -> query_fanout_workflow; deep research -> deep_research_workflow.
+- Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) -> the browser_* agent
+  (browser_open, then navigate/read); save/reuse logins via browser_profile_*.
+
+NOTES
+- Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
+  It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
+- Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
+  cheapest tool that answers the question (plain search/extract before browser agents).
+- Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
+  for full detail rather than expecting the whole payload inline.
+`.trim();
+
 // src/mcp/mcp-tool-schemas.ts
 var import_zod3 = require("zod");
 
@@ -3551,11 +3929,14 @@ var ExtractUrlInputSchema = {
 };
 var MapSiteUrlsInputSchema = {
   url: import_zod3.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
-  maxUrls: import_zod3.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
+  maxUrls: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
 };
 var ExtractSiteInputSchema = {
   url: import_zod3.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
-  maxPages: import_zod3.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
+  maxPages: import_zod3.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
+  rotateProxies: import_zod3.z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+  rotateProxyEvery: import_zod3.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
+  formats: import_zod3.z.array(import_zod3.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
 };
 var YoutubeHarvestInputSchema = {
   mode: import_zod3.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
@@ -4687,14 +5068,14 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
   }, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
   server2.registerTool("map_site_urls", {
     title: "Site URL Map",
-    description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
+    description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Supports up to maxUrls=10000; maps over 500 URLs write the complete inventory to a local CSV file and return a summary plus the file path instead of the full list inline. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
     inputSchema: MapSiteUrlsInputSchema,
     outputSchema: MapSiteUrlsOutputSchema,
     annotations: liveWebToolAnnotations("Site URL Map")
   }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
   server2.registerTool("extract_site", {
     title: "Multi-Page Site Extract",
-    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
+    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all page content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
     inputSchema: ExtractSiteInputSchema,
     outputSchema: ExtractSiteOutputSchema,
     annotations: liveWebToolAnnotations("Multi-Page Site Extract")
@@ -4818,7 +5199,7 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
     outputSchema: WorkflowArtifactReadOutputSchema,
     annotations: liveWebToolAnnotations("Read Workflow Artifact")
   }, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
-  server2.registerTool("rank_tracker_blueprint", {
+  server2.registerTool("rank_tracker_workflow", {
     title: "Rank Tracker Blueprint Builder",
     description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
     inputSchema: RankTrackerBlueprintInputSchema,
@@ -4895,7 +5276,7 @@ function renderInstallTerminal(options) {
     `${colorize("Tools", "cyan", color)}  ${colorize("(44 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
     toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
-    toolRow("build", ["rank_tracker_blueprint", "cron plan", "database prompt"], color),
+    toolRow("build", ["rank_tracker_workflow", "cron plan", "database prompt"], color),
     toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
     toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_profile_status", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
     toolRow("account", ["credits_info", "reports", "MCP resources"], color),
@@ -4971,7 +5352,7 @@ if (!apiKey) {
 }
 var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
 var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-var server = new import_mcp3.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION });
+var server = new import_mcp3.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
 registerPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey));
 registerBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl });
 var transport = new import_stdio.StdioServerTransport();