mcp-scraper 0.3.14 → 0.3.16

package/dist/bin/api-server.cjs

@@ -4214,6 +4214,10 @@ async function captureScreenshot(url, kernelApiKey, device = "desktop") {
   const result = await capturePageData(url, { kernelApiKey, device, screenshot: true, branding: false });
   return result.screenshot;
 }
+async function extractBranding(url, kernelApiKey, device = "desktop") {
+  const result = await capturePageData(url, { kernelApiKey, device, screenshot: false, branding: true });
+  return result.branding;
+}
 async function capturePageData(url, opts = {}) {
   const device = opts.device ?? "desktop";
   const wantShot = opts.screenshot ?? false;
@@ -4646,9 +4650,10 @@ function baseDomain(url) {
 }
 async function spiderSite(opts) {
   const startMs = Date.now();
-  const maxUrls = Math.min(opts.maxUrls ?? 500, 2e3);
+  const maxUrls = Math.min(opts.maxUrls ?? 500, 1e4);
   const concurrency = Math.min(opts.concurrency ?? 12, 20);
   const timeoutMs = opts.timeoutMs ?? 8e3;
+  const budgetMs = opts.budgetMs ?? 24e4;
   const startNorm = normalizeUrl(opts.startUrl, opts.startUrl) ?? opts.startUrl;
   const startDomain = baseDomain(opts.startUrl);
   const visited = /* @__PURE__ */ new Set();
@@ -4662,7 +4667,7 @@ async function spiderSite(opts) {
       queue.push({ url: loc, from: "sitemap" });
     }
   }
-  while (queue.length > 0 && results.length < maxUrls) {
+  while (queue.length > 0 && results.length < maxUrls && Date.now() - startMs < budgetMs) {
     const batch = queue.splice(0, concurrency);
     const settled = await Promise.all(
       batch.map(async ({ url, from }) => {
@@ -4819,9 +4824,193 @@ var init_site_mapper = __esm({
   }
 });
 
+// src/api/rotating-proxy-crawl.ts
+async function discoverSitemapUrls2(startUrl, maxUrls) {
+  const domain = new URL(startUrl).hostname.replace(/^www\./, "");
+  const urls = await discoverSitemapUrls(startUrl, domain);
+  return urls.slice(0, maxUrls);
+}
+function withTimeout(p, ms) {
+  return Promise.race([p, new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), ms))]);
+}
+async function createBrowserWithRetry(client2, headless) {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      return await client2.browsers.create({ stealth: true, headless, timeout_seconds: 120 });
+    } catch (err) {
+      if (attempt === 2) throw err;
+      await new Promise((r) => setTimeout(r, 5e3));
+    }
+  }
+  throw new Error("unreachable");
+}
+async function crawlPage(context, url) {
+  const startedAt = Date.now();
+  let page = null;
+  try {
+    page = await context.newPage();
+    const work = (async (p) => {
+      const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
+      const html = await p.content();
+      const headers = resp ? Object.fromEntries(Object.entries(await resp.headers()).map(([k, v]) => [k.toLowerCase(), v])) : void 0;
+      return { url, html, status: resp?.status() ?? 200, via: "browser", headers, responseTimeMs: Date.now() - startedAt, redirectUrl: headers?.["location"] ?? null };
+    })(page);
+    const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
+    return await Promise.race([work, timeout]);
+  } catch {
+    return { url, html: null, status: 0, via: "browser", responseTimeMs: Date.now() - startedAt };
+  } finally {
+    if (page) void page.close().catch(() => {
+    });
+  }
+}
+async function crawlWithRotation(urls, opts) {
+  const client2 = new import_sdk3.default({ apiKey: opts.apiKey });
+  const concurrency = Math.max(1, opts.concurrency ?? DEFAULT_CONCURRENCY);
+  const urlsPerBrowser = Math.max(1, opts.urlsPerBrowser ?? DEFAULT_URLS_PER_BROWSER);
+  const headless = opts.headless ?? DEFAULT_HEADLESS;
+  const uniqueUrls = [...new Set(urls)];
+  const byUrl = /* @__PURE__ */ new Map();
+  const runPass = async (targets, phase) => {
+    if (targets.length === 0) return;
+    let next = 0;
+    let done = 0;
+    const workers = Math.min(concurrency, targets.length);
+    const worker = async () => {
+      let kb = null;
+      let browser = null;
+      let inBrowser = 0;
+      const retire = () => {
+        const b = browser, k = kb;
+        browser = null;
+        kb = null;
+        if (b) void b.close().catch(() => {
+        });
+        if (k) void client2.browsers.deleteByID(k.session_id).catch(() => {
+        });
+      };
+      try {
+        while (true) {
+          const i = next++;
+          if (i >= targets.length) break;
+          const url = targets[i];
+          let r;
+          try {
+            if (!browser || inBrowser >= urlsPerBrowser) {
+              retire();
+              kb = await createBrowserWithRetry(client2, headless);
+              browser = await withTimeout(import_playwright3.chromium.connectOverCDP(kb.cdp_ws_url), 3e4);
+              inBrowser = 0;
+            }
+            const context = browser.contexts()[0] ?? await browser.newContext();
+            r = await crawlPage(context, url);
+            inBrowser++;
+          } catch {
+            r = { url, html: null, status: 0, via: "browser" };
+            retire();
+          }
+          byUrl.set(url, r);
+          done++;
+          opts.onPage?.({ url, status: r.status, ok: !!(r.html && r.status >= 200 && r.status < 300), ms: r.responseTimeMs ?? 0, done, total: targets.length, phase });
+        }
+      } finally {
+        retire();
+      }
+    };
+    await Promise.all(Array.from({ length: workers }, () => worker()));
+  };
+  await runPass(uniqueUrls, "main");
+  const failed = uniqueUrls.filter((u) => {
+    const r = byUrl.get(u);
+    return !r || !r.html || r.status < 200 || r.status >= 300;
+  });
+  if (failed.length > 0) await runPass(failed, "retry");
+  return urls.map((u) => byUrl.get(u) ?? { url: u, html: null, status: 0, via: "browser" });
+}
+var import_sdk3, import_playwright3, DEFAULT_CONCURRENCY, DEFAULT_URLS_PER_BROWSER, DEFAULT_HEADLESS, PAGE_HARD_TIMEOUT_MS;
+var init_rotating_proxy_crawl = __esm({
+  "src/api/rotating-proxy-crawl.ts"() {
+    "use strict";
+    import_sdk3 = __toESM(require("@onkernel/sdk"), 1);
+    import_playwright3 = require("playwright");
+    init_site_mapper();
+    DEFAULT_CONCURRENCY = 1;
+    DEFAULT_URLS_PER_BROWSER = 10;
+    DEFAULT_HEADLESS = true;
+    PAGE_HARD_TIMEOUT_MS = 35e3;
+  }
+});
+
 // src/api/site-extractor.ts
-function parsePageData(url, html, status, via) {
+function estimatePixels(s) {
+  let px = 0;
+  for (const ch of s) px += PIXEL_WIDTHS[ch] ?? (ch === ch.toUpperCase() && ch !== ch.toLowerCase() ? 10 : 8);
+  return px;
+}
+function emptyPageData(url, status, via) {
+  return {
+    url,
+    status,
+    via,
+    contentType: null,
+    responseTimeMs: null,
+    sizeBytes: null,
+    redirectUrl: null,
+    lastModified: null,
+    xRobotsTag: null,
+    title: null,
+    titleLength: null,
+    titlePixels: null,
+    metaDescription: null,
+    metaDescLength: null,
+    metaKeywords: null,
+    h1: null,
+    h1_2: null,
+    h2Count: 0,
+    headings: [],
+    wordCount: 0,
+    textRatio: 0,
+    contentHash: "",
+    metaRobots: null,
+    relNext: null,
+    relPrev: null,
+    hreflang: [],
+    og: null,
+    twitter: null,
+    ampHref: null,
+    imageCount: 0,
+    imagesMissingAlt: 0,
+    imageLinks: [],
+    indexable: status === 200,
+    indexabilityReason: status === 200 ? null : "non-200",
+    schemaTypes: [],
+    canonicalUrl: null,
+    internalLinks: 0,
+    externalLinks: 0,
+    outlinks: [],
+    bodyMarkdown: "",
+    schema: []
+  };
+}
+function makeSeedSpider(startUrl, urls) {
+  return {
+    startUrl,
+    urls: urls.map((url) => ({ url, status: 200, linkedFrom: "seed", via: "fetch" })),
+    totalFound: urls.length,
+    durationMs: 0,
+    truncated: false,
+    browserRetries: 0
+  };
+}
+function parsePageData(url, html, status, via, resp) {
   const origin = new URL(url).origin;
+  const headers = resp?.headers ?? {};
+  const contentType = headers["content-type"] ?? null;
+  const lastModified = headers["last-modified"] ?? null;
+  const xRobotsTag = headers["x-robots-tag"] ?? null;
+  const responseTimeMs = resp?.responseTimeMs ?? null;
+  const redirectUrl = resp?.redirectUrl ?? headers["location"] ?? null;
+  const sizeBytes = headers["content-length"] ? Number(headers["content-length"]) : Buffer.byteLength(html);
   const title = html.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.trim() ?? null;
   const metaDescription = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i)?.[1]?.trim() ?? html.match(/<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i)?.[1]?.trim() ?? null;
   const canonicalUrl = html.match(/<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? html.match(/<link[^>]+href=["']([^"']*)["'][^>]+rel=["']canonical["']/i)?.[1]?.trim() ?? null;
@@ -4858,38 +5047,170 @@ function parsePageData(url, html, status, via) {
   } catch {
     bodyMarkdown = bodyText.slice(0, MAX_PAGE_MARKDOWN);
   }
-  let internalLinks = 0;
-  let externalLinks = 0;
-  for (const m of html.matchAll(/href\s*=\s*["']([^"'\s>]+)/gi)) {
+  const outlinks = [];
+  for (const m of html.matchAll(/<a\b([^>]*?)href\s*=\s*["']([^"'#][^"']*)["']([^>]*)>([\s\S]*?)<\/a>/gi)) {
+    try {
+      const abs = new URL(m[2], url);
+      if (abs.protocol !== "http:" && abs.protocol !== "https:") continue;
+      const attrs = `${m[1]} ${m[3]}`;
+      const rel = attrs.match(/\brel\s*=\s*["']([^"']*)["']/i)?.[1]?.toLowerCase() ?? null;
+      const anchor = m[4].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 160);
+      outlinks.push({ href: abs.href, anchor, rel, internal: abs.origin === origin });
+    } catch {
+    }
+  }
+  const internalLinks = outlinks.filter((l) => l.internal).length;
+  const externalLinks = outlinks.length - internalLinks;
+  const titleLength = title?.length ?? null;
+  const titlePixels = title ? estimatePixels(title) : null;
+  const metaDescLength = metaDescription?.length ?? null;
+  const metaKeywords = html.match(/<meta[^>]+name=["']keywords["'][^>]+content=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  const h1List = headings.filter((h) => h.level === 1);
+  const h1_2 = h1List[1]?.text ?? null;
+  const h2Count = headings.filter((h) => h.level === 2).length;
+  const textRatio = Number((bodyText.length / Math.max(1, html.length)).toFixed(3));
+  const contentHash = (0, import_node_crypto.createHash)("sha1").update(bodyText).digest("hex");
+  const metaRobots = html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']*)["']/i)?.[1]?.trim().toLowerCase() ?? null;
+  const relNext = html.match(/<link[^>]+rel=["']next["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  const relPrev = html.match(/<link[^>]+rel=["']prev["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  const hreflang = [];
+  for (const m of html.matchAll(/<link[^>]+rel=["']alternate["'][^>]*>/gi)) {
+    const lang = m[0].match(/hreflang=["']([^"']*)["']/i)?.[1];
+    const href = m[0].match(/href=["']([^"']*)["']/i)?.[1];
+    if (lang && href) hreflang.push({ lang, href });
+  }
+  const ogTag = (p) => html.match(new RegExp(`<meta[^>]+property=["']og:${p}["'][^>]+content=["']([^"']*)["']`, "i"))?.[1]?.trim();
+  const og = ogTag("title") || ogTag("description") || ogTag("image") || ogTag("type") ? { title: ogTag("title"), description: ogTag("description"), image: ogTag("image"), type: ogTag("type") } : null;
+  const twTag = (p) => html.match(new RegExp(`<meta[^>]+name=["']twitter:${p}["'][^>]+content=["']([^"']*)["']`, "i"))?.[1]?.trim();
+  const twitter = twTag("card") || twTag("title") || twTag("description") ? { card: twTag("card"), title: twTag("title"), description: twTag("description") } : null;
+  const ampHref = html.match(/<link[^>]+rel=["']amphtml["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  let imageCount = 0;
+  let imagesMissingAlt = 0;
+  const imageLinkSet = /* @__PURE__ */ new Set();
+  for (const m of html.matchAll(/<img\b([^>]*)>/gi)) {
+    imageCount++;
+    const alt = m[1].match(/\balt\s*=\s*["']([^"']*)["']/i)?.[1];
+    if (!alt || !alt.trim()) imagesMissingAlt++;
+    const src = m[1].match(/\bsrc\s*=\s*["']([^"']+)["']/i)?.[1];
+    if (src) {
+      try {
+        const abs = new URL(src, url);
+        if (abs.protocol === "http:" || abs.protocol === "https:") imageLinkSet.add(abs.href);
+      } catch {
+      }
+    }
+  }
+  if (og?.image) {
+    try {
+      imageLinkSet.add(new URL(og.image, url).href);
+    } catch {
+    }
+  }
+  const imageLinks = [...imageLinkSet].slice(0, 300);
+  const canonNorm = (u) => {
     try {
-      const abs = new URL(m[1], url);
-      if (abs.origin === origin) internalLinks++;
-      else externalLinks++;
+      const x = new URL(u, url);
+      return (x.origin + x.pathname).replace(/\/+$/, "");
     } catch {
+      return u.replace(/\/+$/, "");
     }
+  };
+  const canonicalSelf = canonicalUrl ? canonNorm(canonicalUrl) === canonNorm(url) : true;
+  let indexable = true;
+  let indexabilityReason = null;
+  if (status !== 200) {
+    indexable = false;
+    indexabilityReason = "non-200";
+  } else if (metaRobots?.includes("noindex")) {
+    indexable = false;
+    indexabilityReason = "noindex";
+  } else if (xRobotsTag?.toLowerCase().includes("noindex")) {
+    indexable = false;
+    indexabilityReason = "x-robots-noindex";
+  } else if (canonicalUrl && !canonicalSelf) {
+    indexable = false;
+    indexabilityReason = "canonicalised";
   }
-  return { url, status, via, title, metaDescription, h1, headings, wordCount: wordCount2, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
+  return {
+    url,
+    status,
+    via,
+    contentType,
+    responseTimeMs,
+    sizeBytes,
+    redirectUrl,
+    lastModified,
+    xRobotsTag,
+    title,
+    titleLength,
+    titlePixels,
+    metaDescription,
+    metaDescLength,
+    metaKeywords,
+    h1,
+    h1_2,
+    h2Count,
+    headings,
+    wordCount: wordCount2,
+    textRatio,
+    contentHash,
+    metaRobots,
+    relNext,
+    relPrev,
+    hreflang,
+    og,
+    twitter,
+    ampHref,
+    imageCount,
+    imagesMissingAlt,
+    imageLinks,
+    indexable,
+    indexabilityReason,
+    schemaTypes,
+    canonicalUrl,
+    internalLinks,
+    externalLinks,
+    outlinks,
+    bodyMarkdown,
+    schema
+  };
+}
+async function extractPagesRotating(urls, opts) {
+  const fetched = await crawlWithRotation(urls, { apiKey: opts.kernelApiKey, concurrency: opts.concurrency, urlsPerBrowser: opts.urlsPerBrowser });
+  return fetched.map((r) => r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser"));
+}
+async function discoverUrls(startUrl, maxPages, kernelApiKey) {
+  const sitemapUrls = await discoverSitemapUrls2(startUrl, maxPages);
+  if (sitemapUrls.length > 0) return sitemapUrls;
+  const spider = await spiderSite({ startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey });
+  return spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
 }
 async function fetchAndParse(url, kernelApiKey) {
+  const startedAt = Date.now();
   try {
     const res = await fetch(url, {
       headers: { "User-Agent": UA2, "Accept": "text/html,application/xhtml+xml" },
       signal: AbortSignal.timeout(1e4),
       redirect: "manual"
     });
+    const meta = {
+      headers: Object.fromEntries([...res.headers.entries()].map(([k, v]) => [k.toLowerCase(), v])),
+      responseTimeMs: Date.now() - startedAt,
+      redirectUrl: res.headers.get("location")
+    };
     if (res.ok) {
       const ct = res.headers.get("content-type") ?? "";
       if (ct.includes("html")) {
         const html = await res.text();
-        return parsePageData(url, html, res.status, "fetch");
+        return parsePageData(url, html, res.status, "fetch", meta);
       }
-      return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
+      return { ...emptyPageData(url, res.status, "fetch"), contentType: ct || null, responseTimeMs: meta.responseTimeMs ?? null };
     }
     if ((res.status === 403 || res.status === 429) && kernelApiKey) {
       const html = await fetchWithKernel(url);
       return parsePageData(url, html, 200, "browser");
     }
-    return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
+    return { ...emptyPageData(url, res.status, "fetch"), responseTimeMs: meta.responseTimeMs ?? null, redirectUrl: meta.redirectUrl ?? null };
   } catch {
     if (kernelApiKey) {
       try {
@@ -4898,7 +5219,7 @@ async function fetchAndParse(url, kernelApiKey) {
       } catch {
       }
     }
-    return { url, status: null, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
+    return emptyPageData(url, null, "fetch");
   }
 }
 async function runWithConcurrency(items, concurrency, fn) {
@@ -4913,39 +5234,95 @@ async function runWithConcurrency(items, concurrency, fn) {
 }
 async function extractSite(opts) {
   const startMs = Date.now();
-  const maxPages = Math.min(opts.maxPages ?? 100, 200);
+  const maxPages = Math.min(opts.maxPages ?? 100, 1e4);
   const concurrency = Math.min(opts.concurrency ?? EXTRACT_CONCURRENCY, 10);
-  const spider = await spiderSite({
-    startUrl: opts.startUrl,
-    maxUrls: maxPages,
-    concurrency: 12,
-    kernelApiKey: opts.kernelApiKey
-  });
-  const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+  const rotating = Boolean(opts.rotateProxyEvery && opts.kernelApiKey);
+  const seeded = opts.seedUrls && opts.seedUrls.length > 0;
   const pages = [];
-  let browserRetries = spider.browserRetries;
-  await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
-    const data = await fetchAndParse(url, opts.kernelApiKey);
-    if (data.via === "browser") browserRetries++;
-    pages.push(data);
-  });
+  let browserRetries = 0;
+  let truncated = false;
+  let spider;
+  if (rotating) {
+    const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
+    const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
+    const followLinks = !seeded;
+    const seen = /* @__PURE__ */ new Set();
+    const frontier = [];
+    const enqueue = (u) => {
+      const n = normalizeUrl(u, opts.startUrl);
+      if (n && !seen.has(n)) {
+        seen.add(n);
+        frontier.push(n);
+      }
+    };
+    if (seeded) {
+      for (const u of opts.seedUrls) enqueue(u);
+    } else {
+      enqueue(opts.startUrl);
+      for (const u of await discoverSitemapUrls2(opts.startUrl, maxPages)) enqueue(u);
+    }
+    const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
+    let done = 0;
+    while (frontier.length > 0 && pages.length < maxPages) {
+      const capacity = maxPages - pages.length;
+      const wave = frontier.splice(0, Math.min(capacity, waveCap));
+      const fetched = await crawlWithRotation(wave, {
+        apiKey: opts.kernelApiKey,
+        concurrency: browserConcurrency,
+        urlsPerBrowser,
+        onPage: (p) => {
+          done++;
+          opts.onProgress?.({ done, discovered: seen.size, frontier: frontier.length, lastOk: p.ok ? 1 : 0, lastFail: p.ok ? 0 : 1, phase: p.phase });
+        }
+      });
+      for (const r of fetched) {
+        const page = r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser");
+        pages.push(page);
+        browserRetries++;
+        if (followLinks && r.html) {
+          for (const l of page.outlinks) if (l.internal) enqueue(l.href);
+        }
+      }
+    }
+    truncated = frontier.length > 0;
+    spider = makeSeedSpider(opts.startUrl, [...seen]);
+  } else {
+    spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
+    const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+    browserRetries = spider.browserRetries;
+    await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
+      const data = await fetchAndParse(url, opts.kernelApiKey);
+      if (data.via === "browser") browserRetries++;
+      pages.push(data);
+    });
+    truncated = spider.truncated || pages.length >= maxPages;
+  }
+  let branding = null;
+  if (opts.formats?.includes("branding") && opts.kernelApiKey) {
+    branding = await extractBranding(opts.startUrl, opts.kernelApiKey).catch(() => null);
+  }
   return {
     startUrl: opts.startUrl,
     pages,
     totalPages: pages.length,
     spider,
     durationMs: Date.now() - startMs,
-    truncated: spider.truncated || pages.length >= maxPages,
-    browserRetries
+    truncated,
+    browserRetries,
+    branding
   };
 }
-var import_turndown2, UA2, EXTRACT_CONCURRENCY, MAX_PAGE_MARKDOWN, turndown;
+var import_turndown2, import_node_crypto, PIXEL_WIDTHS, UA2, EXTRACT_CONCURRENCY, MAX_PAGE_MARKDOWN, turndown;
 var init_site_extractor = __esm({
   "src/api/site-extractor.ts"() {
     "use strict";
     import_turndown2 = __toESM(require("turndown"), 1);
+    import_node_crypto = require("crypto");
     init_site_mapper();
     init_kernel_fetch();
+    init_rotating_proxy_crawl();
+    init_screenshot();
+    PIXEL_WIDTHS = { i: 4, l: 4, j: 4, ".": 4, ",": 4, "'": 4, t: 6, f: 6, r: 6, " ": 4, m: 14, w: 13, W: 16, M: 16 };
     UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
     EXTRACT_CONCURRENCY = 6;
     MAX_PAGE_MARKDOWN = 4e4;
@@ -5033,11 +5410,11 @@ async function llmParseWithRetry(schema, promptFn, llm) {
   const detail = lastError?.message?.slice(0, 200) ?? "unknown error";
   throw new Error(`LLM failed to produce valid output after 3 attempts: ${detail}`);
 }
-var import_sdk3, DeepInfraLlmClient, OpenRouterLlmClient, FallbackLlmClient;
+var import_sdk4, DeepInfraLlmClient, OpenRouterLlmClient, FallbackLlmClient;
 var init_llm_parse_with_retry = __esm({
   "src/lib/llm-parse-with-retry.ts"() {
     "use strict";
-    import_sdk3 = __toESM(require("@anthropic-ai/sdk"), 1);
+    import_sdk4 = __toESM(require("@anthropic-ai/sdk"), 1);
     DeepInfraLlmClient = class {
       apiKey;
       model;
@@ -7185,6 +7562,33 @@ async function migrate() {
   await db.execute(`CREATE INDEX IF NOT EXISTS site_audit_jobs_user_id ON site_audit_jobs(user_id)`);
   await db.execute(`CREATE INDEX IF NOT EXISTS site_audit_jobs_status ON site_audit_jobs(status)`);
   await db.execute(`CREATE INDEX IF NOT EXISTS site_audit_phase_log_job_id ON site_audit_phase_log(job_id)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS site_extract_jobs (
+      id            TEXT    PRIMARY KEY,
+      user_id       INTEGER,
+      status        TEXT    DEFAULT 'pending',
+      start_url     TEXT,
+      options       TEXT,
+      total_urls    INTEGER DEFAULT 0,
+      done_urls     INTEGER DEFAULT 0,
+      artifacts     TEXT,
+      error         TEXT,
+      billed_mc     INTEGER,
+      created_at    TEXT    DEFAULT (datetime('now')),
+      updated_at    TEXT    DEFAULT (datetime('now'))
+    )
+  `);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS site_extract_pages (
+      job_id     TEXT,
+      url        TEXT,
+      page       TEXT,
+      PRIMARY KEY (job_id, url)
+    )
+  `);
+  await db.execute(`CREATE INDEX IF NOT EXISTS site_extract_jobs_user_id ON site_extract_jobs(user_id)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS site_extract_jobs_status ON site_extract_jobs(status)`);
+  await db.execute(`CREATE INDEX IF NOT EXISTS site_extract_pages_job_id ON site_extract_pages(job_id)`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS password_reset_tokens (
       token      TEXT    PRIMARY KEY,
@@ -7341,18 +7745,18 @@ async function stripeEventAlreadyProcessed(eventId) {
   return res.rows.length > 0;
 }
 function generateApiKey() {
-  return "sk_" + (0, import_node_crypto.randomBytes)(24).toString("hex");
+  return "sk_" + (0, import_node_crypto2.randomBytes)(24).toString("hex");
 }
 function hashPassword(password) {
-  const salt = (0, import_node_crypto.randomBytes)(16).toString("hex");
-  const hash = (0, import_node_crypto.scryptSync)(password, salt, 64);
+  const salt = (0, import_node_crypto2.randomBytes)(16).toString("hex");
+  const hash = (0, import_node_crypto2.scryptSync)(password, salt, 64);
   return `${salt}:${hash.toString("hex")}`;
 }
 function verifyPassword(password, stored) {
   const [salt, hash] = stored.split(":");
   const hashBuf = Buffer.from(hash, "hex");
-  const derived = (0, import_node_crypto.scryptSync)(password, salt, 64);
-  return (0, import_node_crypto.timingSafeEqual)(hashBuf, derived);
+  const derived = (0, import_node_crypto2.scryptSync)(password, salt, 64);
+  return (0, import_node_crypto2.timingSafeEqual)(hashBuf, derived);
 }
 function rowToUser(row) {
   return {
@@ -7405,7 +7809,7 @@ async function getUserStats(userId) {
 async function createUser(email, name, password, stripeCustomerId) {
   const db = getDb();
   const api_key = generateApiKey();
-  const plainPassword = password ?? (0, import_node_crypto.randomBytes)(6).toString("hex");
+  const plainPassword = password ?? (0, import_node_crypto2.randomBytes)(6).toString("hex");
   const password_hash = hashPassword(plainPassword);
   const result = await db.execute({
     sql: "INSERT INTO users (email, name, api_key, password_hash, stripe_customer_id) VALUES (?, ?, ?, ?, ?)",
@@ -7416,7 +7820,7 @@ async function createUser(email, name, password, stripeCustomerId) {
 async function checkRateLimit(scope, key, limit, windowSeconds) {
   const nowSeconds = Math.floor(Date.now() / 1e3);
   const windowStart = Math.floor(nowSeconds / windowSeconds) * windowSeconds;
-  const keyHash = (0, import_node_crypto.createHash)("sha256").update(key).digest("hex");
+  const keyHash = (0, import_node_crypto2.createHash)("sha256").update(key).digest("hex");
   const db = getDb();
   if (!_rateLimitSchemaReady) {
     await db.execute(`
@@ -7453,8 +7857,8 @@ async function checkRateLimit(scope, key, limit, windowSeconds) {
   };
 }
 async function createPasswordResetToken(userId) {
-  const token = (0, import_node_crypto.randomBytes)(32).toString("hex");
-  const tokenHash = (0, import_node_crypto.createHash)("sha256").update(token).digest("hex");
+  const token = (0, import_node_crypto2.randomBytes)(32).toString("hex");
+  const tokenHash = (0, import_node_crypto2.createHash)("sha256").update(token).digest("hex");
   const expiresAt = new Date(Date.now() + 60 * 60 * 1e3).toISOString();
   await getDb().execute({
     sql: "INSERT INTO password_reset_tokens (token, user_id, expires_at) VALUES (?, ?, ?)",
@@ -7464,7 +7868,7 @@ async function createPasswordResetToken(userId) {
 }
 async function consumePasswordResetToken(token) {
   const db = getDb();
-  const tokenHash = (0, import_node_crypto.createHash)("sha256").update(token).digest("hex");
+  const tokenHash = (0, import_node_crypto2.createHash)("sha256").update(token).digest("hex");
   const res = await db.execute({
     sql: "SELECT user_id, expires_at, used FROM password_reset_tokens WHERE token = ?",
     args: [tokenHash]
@@ -7552,7 +7956,7 @@ function deserialize(raw) {
   };
 }
 async function createJob(userId, query, options, callbackUrl) {
-  const id = (0, import_node_crypto.randomUUID)();
+  const id = (0, import_node_crypto2.randomUUID)();
   await getDb().execute({
     sql: "INSERT INTO jobs (id, user_id, query, options, callback_url) VALUES (?, ?, ?, ?, ?)",
     args: [id, userId, query, JSON.stringify(options), callbackUrl ?? null]
@@ -7560,7 +7964,7 @@ async function createJob(userId, query, options, callbackUrl) {
   return id;
 }
 async function createRunningJob(userId, query, options) {
-  const id = (0, import_node_crypto.randomUUID)();
+  const id = (0, import_node_crypto2.randomUUID)();
   await getDb().execute({
     sql: `INSERT INTO jobs (id, user_id, query, options, status, started_at) VALUES (?, ?, ?, ?, 'running', datetime('now'))`,
     args: [id, userId, query, JSON.stringify(options)]
@@ -7583,7 +7987,7 @@ async function startHarvestAttempt(input) {
         debug_json = NULL
     `,
     args: [
-      (0, import_node_crypto.randomUUID)(),
+      (0, import_node_crypto2.randomUUID)(),
       input.jobId,
       input.userId,
       input.attemptNumber,
@@ -7675,7 +8079,7 @@ function rowToRequestEvent(row) {
   };
 }
 async function logRequestEvent(input) {
-  const id = (0, import_node_crypto.randomUUID)();
+  const id = (0, import_node_crypto2.randomUUID)();
   await getDb().execute({
     sql: `
       INSERT INTO request_events (id, user_id, source, status, query, location, result_count, result, error)
@@ -7780,7 +8184,7 @@ function rowToWorkflowArtifact(row) {
   };
 }
 async function createWorkflowSchedule(input) {
-  const id = (0, import_node_crypto.randomUUID)();
+  const id = (0, import_node_crypto2.randomUUID)();
   await getDb().execute({
     sql: `
       INSERT INTO workflow_schedules (
@@ -7880,7 +8284,7 @@ async function markWorkflowScheduleRan(id, lastRunAt, nextRunAt) {
   });
 }
 async function createWorkflowRun(input) {
-  const id = input.id ?? (0, import_node_crypto.randomUUID)();
+  const id = input.id ?? (0, import_node_crypto2.randomUUID)();
   const args = [
     id,
     input.userId,
@@ -7956,7 +8360,7 @@ async function insertWorkflowArtifact(runId, userId, artifact) {
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `,
     args: [
-      (0, import_node_crypto.randomUUID)(),
+      (0, import_node_crypto2.randomUUID)(),
       runId,
       userId,
       artifact.kind,
@@ -8019,7 +8423,7 @@ async function recordWorkflowWebhookDelivery(input) {
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
     `,
     args: [
-      (0, import_node_crypto.randomUUID)(),
+      (0, import_node_crypto2.randomUUID)(),
       input.runId,
       input.scheduleId ?? null,
       input.userId,
@@ -8227,12 +8631,12 @@ async function reconcileBalanceMc(userId) {
   });
   return balanceMc;
 }
-var import_http, import_node_crypto, import_zod10, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
+var import_http, import_node_crypto2, import_zod10, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
 var init_db = __esm({
   "src/api/db.ts"() {
     "use strict";
     import_http = require("@libsql/client/http");
-    import_node_crypto = require("crypto");
+    import_node_crypto2 = require("crypto");
     import_zod10 = require("zod");
     DB_URL = process.env.TURSO_DATABASE_URL ?? "file:./paa-api.db";
     DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
@@ -8568,212 +8972,394 @@ var init_site_audit = __esm({
   }
 });
 
-// src/api/site-audit-middleware.ts
-var import_factory2, siteAuditAuth;
-var init_site_audit_middleware = __esm({
-  "src/api/site-audit-middleware.ts"() {
-    "use strict";
-    import_factory2 = require("hono/factory");
-    init_db();
-    siteAuditAuth = (0, import_factory2.createMiddleware)(async (c, next) => {
-      const apiKey = c.req.header("x-api-key");
-      if (apiKey) {
-        const user = await getUserByApiKey(apiKey);
-        if (!user) return c.json({ error: "Unauthorized" }, 401);
-        c.set("siteAuditUserId", user.id);
-        return next();
+// src/api/seo-link-graph.ts
+function normalize(u) {
+  return u.split("#")[0].replace(/\/$/, "");
+}
+function buildLinkGraph(pages, startUrl) {
+  const statusByUrl = /* @__PURE__ */ new Map();
+  for (const p of pages) statusByUrl.set(normalize(p.url), p.status);
+  const edges = [];
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
+      edges.push({
+        from: p.url,
+        to: l.href,
+        anchor: l.anchor,
+        rel: l.rel,
+        internal: l.internal,
+        nofollow,
+        targetStatus: statusByUrl.get(normalize(l.href)) ?? null
+      });
+    }
+  }
+  const inboundFrom = /* @__PURE__ */ new Map();
+  const inboundAnchors = /* @__PURE__ */ new Map();
+  const adjacency = /* @__PURE__ */ new Map();
+  for (const e of edges) {
+    if (!e.internal) continue;
+    const to = normalize(e.to);
+    const from = normalize(e.from);
+    if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
+    inboundFrom.get(to).add(from);
+    if (e.anchor) {
+      if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
+      inboundAnchors.get(to).push(e.anchor);
+    }
+    if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
+      if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
+      adjacency.get(from).add(to);
+    }
+  }
+  const depth = /* @__PURE__ */ new Map();
+  const start = normalize(startUrl);
+  const queue = [start];
+  depth.set(start, 0);
+  while (queue.length > 0) {
+    const cur = queue.shift();
+    const d = depth.get(cur);
+    for (const next of adjacency.get(cur) ?? []) {
+      if (!depth.has(next)) {
+        depth.set(next, d + 1);
+        queue.push(next);
       }
-      return c.json({ error: "Unauthorized" }, 401);
+    }
+  }
+  const topAnchorsFor = (url) => {
+    const counts = /* @__PURE__ */ new Map();
+    for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
+    return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
+  };
+  const metrics = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const key = normalize(p.url);
+    const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
+    metrics.set(p.url, {
+      url: p.url,
+      inlinks: inbound.size,
+      uniqueInlinks: inbound.size,
+      outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
+      outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
+      crawlDepth: depth.has(key) ? depth.get(key) : null,
+      orphan: inbound.size === 0 && key !== start,
+      topAnchors: topAnchorsFor(key)
     });
   }
+  return { edges, metrics };
+}
+var init_seo_link_graph = __esm({
+  "src/api/seo-link-graph.ts"() {
+    "use strict";
+  }
 });
 
-// src/api/site-audit-routes.ts
-function isPathInside(root, target) {
-  const relative = import_node_path2.default.relative(root, target);
-  return relative === "" || !relative.startsWith("..") && !import_node_path2.default.isAbsolute(relative);
+// src/api/seo-issues.ts
+function dupes(pages, key) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const k = key(p);
+    if (k === null || k === void 0 || k === "") continue;
+    const ks = String(k);
+    if (!groups.has(ks)) groups.set(ks, []);
+    groups.get(ks).push(p.url);
+  }
+  return [...groups.values()].filter((g) => g.length > 1).flat();
+}
+function computeIssues(pages, metrics) {
+  const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
+  const where = (fn) => pages.filter(fn).map((p) => p.url);
+  const ok = (p) => p.status === 200;
+  const pathname = (u) => {
+    try {
+      return new URL(u).pathname;
+    } catch {
+      return "";
+    }
+  };
+  const report = {
+    "title.missing": group(where((p) => ok(p) && !p.title)),
+    "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
+    "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
+    "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
+    "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
+    "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
+    "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
+    "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
+    "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
+    "h1.missing": group(where((p) => ok(p) && !p.h1)),
+    "h1.multiple": group(where((p) => !!p.h1_2)),
+    "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
+    "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
+    "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
+    "indexability.nonIndexable": group(where((p) => !p.indexable)),
+    "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
+    "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
+    "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
+    "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
+    "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
+    "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
+    "response.noResponse": group(where((p) => p.status === null)),
+    "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
+    "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
+    "images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
+    "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
+    "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
+    "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
+    "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
+    "links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
+  };
+  const normUrl2 = (u) => {
+    try {
+      const x = new URL(u);
+      x.hash = "";
+      return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
+    } catch {
+      return u.replace(/#.*$/, "").replace(/\/+$/, "");
+    }
+  };
+  const statusByUrl = new Map(pages.map((p) => [normUrl2(p.url), p.status]));
+  const brokenLinkPages = /* @__PURE__ */ new Set();
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      if (!l.internal) continue;
+      const st = statusByUrl.get(normUrl2(l.href));
+      if (st != null && st >= 400) brokenLinkPages.add(p.url);
+    }
+  }
+  report["links.brokenInternal"] = group([...brokenLinkPages]);
+  return report;
+}
+function renderIssueReport(siteUrl, pages, report, metrics) {
+  const total = pages.length;
+  const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
+  const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
+  const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
+  const maxDepth = depths.length ? Math.max(...depths) : 0;
+  const orphans = [...metrics.values()].filter((m) => m.orphan).length;
+  const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
+  return [
+    `# SEO Crawl Report: ${siteUrl}`,
+    `**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
+    `
+## Issues
+| | Issue | Count | Examples |
+|---|-------|-------|----------|
+${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
+    `
+## Most-linked pages
+${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
+    `
+_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
+  ].join("\n");
 }
-var import_node_path2, import_node_os2, import_hono, import_zod11, siteAuditApp;
-var init_site_audit_routes = __esm({
-  "src/api/site-audit-routes.ts"() {
+var THIN_WORDS, TITLE_MAX, TITLE_PX_MAX, TITLE_MIN, META_MAX, META_MIN, H1_MAX, URL_MAX;
+var init_seo_issues = __esm({
+  "src/api/seo-issues.ts"() {
     "use strict";
-    import_node_path2 = __toESM(require("path"), 1);
-    import_node_os2 = __toESM(require("os"), 1);
-    import_hono = require("hono");
-    import_zod11 = require("zod");
-    init_site_audit_middleware();
-    init_factory();
-    init_phases();
-    init_schemas();
-    siteAuditApp = new import_hono.Hono();
-    siteAuditApp.onError((err, c) => {
-      if (err instanceof import_zod11.ZodError) {
-        return c.json({ error: "Invalid request body", issues: err.issues }, 400);
+    THIN_WORDS = 200;
+    TITLE_MAX = 60;
+    TITLE_PX_MAX = 561;
+    TITLE_MIN = 30;
+    META_MAX = 155;
+    META_MIN = 70;
+    H1_MAX = 70;
+    URL_MAX = 115;
+  }
+});
+
+// src/api/image-audit.ts
+function formatBytes(n) {
+  if (n == null) return null;
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
+}
+function collectUrls(pages) {
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of pages) for (const raw of p.imageLinks || []) {
+    const u = decodeEntities(raw);
+    if (!/^https?:\/\//i.test(u)) continue;
+    if (/[{}]|%7[bd]/i.test(u)) continue;
+    const k = dedupKey(u);
+    const prev = byKey.get(k);
+    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
+  }
+  return [...byKey.values()];
+}
+async function sizeAndType(url, timeoutMs) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  const read = (res) => ({
+    len: res.headers.get("content-length"),
+    cr: res.headers.get("content-range"),
+    ct: res.headers.get("content-type"),
+    status: res.status
+  });
+  try {
+    let bytes = null;
+    let ct = null;
+    let status = null;
+    try {
+      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
+      status = h.status;
+      ct = h.ct;
+      if (h.len) bytes = Number(h.len);
+    } catch {
+    }
+    if (bytes == null) {
+      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
+      const r = read(g);
+      status = status ?? r.status;
+      ct = ct ?? r.ct;
+      if (r.cr && r.cr.includes("/")) {
+        const tail = r.cr.split("/")[1];
+        if (tail && tail !== "*") bytes = Number(tail);
+      } else if (r.len) bytes = Number(r.len);
+      try {
+        await g.body?.cancel();
+      } catch {
       }
-      const msg = err instanceof Error ? err.message : String(err);
-      return c.json({ error: msg }, 500);
-    });
-    siteAuditApp.use("*", siteAuditAuth);
-    siteAuditApp.post("/audit", async (c) => {
-      const body = SiteAuditStartRequestSchema.parse(await c.req.json());
-      const allowedRoot = import_node_path2.default.resolve(process.env["SESSION_ROOT"] ?? import_node_os2.default.tmpdir());
-      const sessionPath = import_node_path2.default.resolve(body.sessionPath);
-      if (!isPathInside(allowedRoot, sessionPath)) {
-        return c.json({ error: "sessionPath must be inside the configured session root" }, 400);
+    }
+    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
+  } catch (e) {
+    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+async function pool(items, n, fn) {
+  const out = new Array(items.length);
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) {
+      const idx = i++;
+      out[idx] = await fn(items[idx]);
+    }
+  }));
+  return out;
+}
+async function auditImages(pages, opts = {}) {
+  const concurrency = opts.concurrency ?? 12;
+  const timeoutMs = opts.timeoutMs ?? 12e3;
+  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
+  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
+  const rows = heads.map((r) => {
+    const format = formatOf(r.contentType, r.url);
+    return {
+      ...r,
+      size: formatBytes(r.bytes),
+      format,
+      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
+      legacyFormat: format !== "unknown" && !MODERN.has(format)
+    };
+  });
+  const sized = rows.filter((r) => r.bytes != null);
+  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
+  const formatCounts = {};
+  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
+  return {
+    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
+    summary: {
+      unique: rows.length,
+      sized: sized.length,
+      totalBytes,
+      totalSize: formatBytes(totalBytes) ?? "0 B",
+      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
+      over100kb: rows.filter((r) => r.over100kb).length,
+      legacyFormat: rows.filter((r) => r.legacyFormat).length,
+      formatCounts
+    }
+  };
+}
+function renderImageSection(audit) {
+  const s = audit.summary;
+  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
+  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
+  const lines = [
+    `## Images`,
+    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
+    `Formats: ${fmt}`
+  ];
+  if (heaviest.length) {
+    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
+    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
+  }
+  return lines.join("\n");
+}
+var OVER_BYTES, MODERN, decodeEntities, dedupKey, formatOf;
+var init_image_audit = __esm({
+  "src/api/image-audit.ts"() {
+    "use strict";
+    OVER_BYTES = 100 * 1024;
+    MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
+    decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
+    dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
+    formatOf = (ct, url) => {
+      if (ct) {
+        const m = ct.split(";")[0].trim().toLowerCase();
+        if (m.startsWith("image/")) return m.replace("image/", "");
+      }
+      return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
+    };
+  }
+});
+
+// src/api/blob-store.ts
+function byteLength(data) {
+  return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
+}
+function localBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
+}
+function getBlobStore() {
+  if (cached) return cached;
+  if (process.env.BLOB_READ_WRITE_TOKEN) {
+    cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
+  } else {
+    cached = new LocalBlobStore(localBaseDir());
+  }
+  return cached;
+}
+var import_node_fs2, import_node_os2, import_node_path2, LocalBlobStore, cached, VercelBlobStore;
+var init_blob_store = __esm({
+  "src/api/blob-store.ts"() {
+    "use strict";
+    import_node_fs2 = require("fs");
+    import_node_os2 = require("os");
+    import_node_path2 = require("path");
+    LocalBlobStore = class {
+      constructor(baseDir) {
+        this.baseDir = baseDir;
       }
-      const pathFields = [
-        body.sessionPath,
-        body.sfInternalPath,
-        body.sfInlinksPath,
-        body.sfOutlinksPath,
-        body.sitemapPath,
-        body.competitorDomainsCsvPath,
-        body.visualSitemapsPath,
-        body.gscPath,
-        body.ga4Path,
-        body.ahrefsPath
-      ];
-      for (const filePath of pathFields) {
-        if (filePath != null && !isPathInside(allowedRoot, import_node_path2.default.resolve(filePath))) {
-          return c.json({ error: "Path traversal attempt" }, 400);
-        }
+      baseDir;
+      kind = "local";
+      async put(key, data, contentType = "application/octet-stream") {
+        const path6 = (0, import_node_path2.join)(this.baseDir, "blobs", key);
+        (0, import_node_fs2.mkdirSync)((0, import_node_path2.dirname)(path6), { recursive: true });
+        (0, import_node_fs2.writeFileSync)(path6, data);
+        return { key, url: `file://${path6}`, bytes: byteLength(data), contentType };
       }
-      const userId = c.get("siteAuditUserId");
-      const service = makeSiteAuditService();
-      const jobId = await service.startAudit(userId, body);
-      void dispatchSiteAuditPhase(service, "phase1-ingest", { jobId, request: body }).catch(
-        (err) => {
-          console.error(
-            "[site-audit-routes] phase1-ingest dispatch failed for job",
-            jobId,
-            err instanceof Error ? err.message : String(err)
-          );
-        }
-      );
-      return c.json({ jobId }, 202);
-    });
-    siteAuditApp.post("/ingest", async (c) => {
-      const body = SiteAuditIngestRequestSchema.parse(await c.req.json());
-      const service = makeSiteAuditService();
-      const job = await service.getJob(body.jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
-      void dispatchSiteAuditPhase(service, "phase1-ingest", {
-        jobId: body.jobId,
-        request: JSON.parse(job.request)
-      }).catch((err) => {
-        console.error(
-          "[site-audit-routes] phase1-ingest re-dispatch failed for job",
-          body.jobId,
-          err instanceof Error ? err.message : String(err)
-        );
-      });
-      return c.json({ jobId: body.jobId }, 202);
-    });
-    siteAuditApp.post("/build-graph", async (c) => {
-      const body = SiteAuditBuildGraphRequestSchema.parse(await c.req.json());
-      const service = makeSiteAuditService();
-      const job = await service.getJob(body.jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
-      void dispatchSiteAuditPhase(service, "phase2-build-graph", { jobId: body.jobId }).catch(
-        (err) => {
-          console.error(
-            "[site-audit-routes] phase2-build-graph failed for job",
-            body.jobId,
-            err instanceof Error ? err.message : String(err)
-          );
-        }
-      );
-      return c.json({ jobId: body.jobId }, 202);
-    });
-    siteAuditApp.post("/classify", async (c) => {
-      const body = SiteAuditClassifyRequestSchema.parse(await c.req.json());
-      const service = makeSiteAuditService();
-      const job = await service.getJob(body.jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
-      void dispatchSiteAuditPhase(service, "phase3-classify", { jobId: body.jobId }).catch(
-        (err) => {
-          console.error(
-            "[site-audit-routes] phase3-classify failed for job",
-            body.jobId,
-            err instanceof Error ? err.message : String(err)
-          );
-        }
-      );
-      return c.json({ jobId: body.jobId }, 202);
-    });
-    siteAuditApp.post("/compare", async (c) => {
-      const body = SiteAuditCompareRequestSchema.parse(await c.req.json());
-      const service = makeSiteAuditService();
-      const job = await service.getJob(body.jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
-      void dispatchSiteAuditPhase(service, "phase4-compare", { jobId: body.jobId }).catch(
-        (err) => {
-          console.error(
-            "[site-audit-routes] phase4-compare failed for job",
-            body.jobId,
-            err instanceof Error ? err.message : String(err)
-          );
-        }
-      );
-      return c.json({ jobId: body.jobId }, 202);
-    });
-    siteAuditApp.post("/score", async (c) => {
-      const body = SiteAuditScoreRequestSchema.parse(await c.req.json());
-      const service = makeSiteAuditService();
-      const job = await service.getJob(body.jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
-      void dispatchSiteAuditPhase(service, "phase5-synthesize", {
-        jobId: body.jobId,
-        scoringOnly: true,
-        reportOnly: false
-      }).catch((err) => {
-        console.error(
-          "[site-audit-routes] phase5-synthesize(scoringOnly) failed for job",
-          body.jobId,
-          err instanceof Error ? err.message : String(err)
-        );
-      });
-      return c.json({ jobId: body.jobId }, 202);
-    });
-    siteAuditApp.post("/report", async (c) => {
-      const body = SiteAuditReportRequestSchema.parse(await c.req.json());
-      const service = makeSiteAuditService();
-      const job = await service.getJob(body.jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
-      void dispatchSiteAuditPhase(service, "phase5-synthesize", {
-        jobId: body.jobId,
-        scoringOnly: false,
-        reportOnly: true
-      }).catch((err) => {
-        console.error(
-          "[site-audit-routes] phase5-synthesize(reportOnly) failed for job",
-          body.jobId,
-          err instanceof Error ? err.message : String(err)
-        );
-      });
-      return c.json({ jobId: body.jobId }, 202);
-    });
-    siteAuditApp.get("/jobs/:id", async (c) => {
-      const jobId = c.req.param("id");
-      const userId = c.get("siteAuditUserId");
-      const service = makeSiteAuditService();
-      const job = await service.getJob(jobId);
-      if (!job) return c.json({ error: "Not found" }, 404);
-      if (job.user_id !== userId) return c.json({ error: "Not found" }, 404);
-      return c.json(job);
-    });
-    siteAuditApp.get("/jobs", async (c) => {
-      const userId = c.get("siteAuditUserId");
-      const service = makeSiteAuditService();
-      const jobs = await service.getJobsForUser(userId);
-      return c.json(jobs);
-    });
+    };
+    cached = null;
+    VercelBlobStore = class {
+      constructor(token) {
+        this.token = token;
+      }
+      token;
+      kind = "vercel-blob";
+      async put(key, data, contentType = "application/octet-stream") {
+        const { put } = await import("@vercel/blob");
+        const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
+        const result = await put(key, body, {
+          access: "public",
+          token: this.token,
+          contentType,
+          addRandomSuffix: true
+        });
+        return { key, url: result.url, bytes: byteLength(data), contentType };
+      }
+    };
   }
 });
 
@@ -9023,6 +9609,792 @@ var init_rates = __esm({
   }
 });
 
+// src/api/site-extract-repository.ts
+function rowToJob(r) {
+  return {
+    id: String(r.id),
+    userId: r.user_id != null ? Number(r.user_id) : null,
+    status: r.status != null ? String(r.status) : "pending",
+    startUrl: String(r.start_url ?? ""),
+    options: r.options ? JSON.parse(String(r.options)) : {},
+    totalUrls: Number(r.total_urls ?? 0),
+    doneUrls: Number(r.done_urls ?? 0),
+    artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,
+    error: r.error != null ? String(r.error) : null,
+    billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,
+    createdAt: String(r.created_at ?? ""),
+    updatedAt: String(r.updated_at ?? "")
+  };
+}
+async function createExtractJob(jobId, userId, startUrl, options) {
+  const db = getDb();
+  await db.execute({
+    sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,
+    args: [jobId, userId, startUrl, JSON.stringify(options)]
+  });
+}
+async function getExtractJob(jobId) {
+  const db = getDb();
+  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] });
+  return res.rows[0] ? rowToJob(res.rows[0]) : null;
+}
+async function setExtractJobTotal(jobId, totalUrls) {
+  const db = getDb();
+  await db.execute({
+    sql: `UPDATE site_extract_jobs SET status = 'running', total_urls = ?, updated_at = datetime('now') WHERE id = ?`,
+    args: [totalUrls, jobId]
+  });
+}
+async function saveExtractPages(jobId, pages) {
+  if (pages.length === 0) return;
+  const db = getDb();
+  await db.batch(
+    pages.map((p) => ({
+      sql: `INSERT OR REPLACE INTO site_extract_pages (job_id, url, page) VALUES (?, ?, ?)`,
+      args: [jobId, p.url, JSON.stringify({ ...p, bodyMarkdown: "", schema: [] })]
+    }))
+  );
+  await db.execute({
+    sql: `UPDATE site_extract_jobs
+            SET done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?), updated_at = datetime('now')
+          WHERE id = ?`,
+    args: [jobId, jobId]
+  });
+}
+async function getExtractedPages(jobId) {
+  const db = getDb();
+  const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
+  return res.rows.map((r) => JSON.parse(String(r.page)));
+}
+async function countSuccessfulPages(jobId) {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT COUNT(*) AS n FROM site_extract_pages
+            WHERE job_id = ? AND json_extract(page, '$.status') = 200 AND json_extract(page, '$.wordCount') > 0`,
+    args: [jobId]
+  });
+  return Number(res.rows[0]?.n ?? 0);
+}
+async function completeExtractJob(jobId, artifacts) {
+  const db = getDb();
+  await db.execute({
+    sql: `UPDATE site_extract_jobs SET status = 'complete', artifacts = ?, updated_at = datetime('now') WHERE id = ?`,
+    args: [JSON.stringify(artifacts ?? []), jobId]
+  });
+}
+async function failExtractJob(jobId, error) {
+  const db = getDb();
+  await db.execute({
+    sql: `UPDATE site_extract_jobs SET status = 'failed', error = ?, updated_at = datetime('now') WHERE id = ?`,
+    args: [sanitizeVendorName(error).slice(0, 2e3), jobId]
+  });
+}
+async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
+  const db = getDb();
+  const job = await getExtractJob(jobId);
+  if (!job || job.billedMc != null) return;
+  if (refundMc <= 0) {
+    await db.execute({ sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] });
+    return;
+  }
+  await db.batch([
+    { sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [refundMc, userId] },
+    { sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)", args: [userId, refundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference] },
+    { sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] }
+  ]);
+}
+var init_site_extract_repository = __esm({
+  "src/api/site-extract-repository.ts"() {
+    "use strict";
+    init_db();
+    init_errors();
+    init_rates();
+  }
+});
+
+// src/inngest/functions/site-extract.ts
+function chunk(items, size) {
+  const out = [];
+  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
+  return out;
+}
+function buildArtifacts(job, pages, branding, imageAudit) {
+  const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
+  const issues = computeIssues(pages, metrics);
+  let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  if (imageAudit) reportMd += `
+
+${renderImageSection(imageAudit)}`;
+  const pageRows = pages.map((p) => {
+    const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
+    const m = metrics.get(p.url);
+    return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
+  });
+  const jsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
+  const files = [
+    { name: "report.md", body: reportMd, type: "text/markdown" },
+    { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
+    { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
+    { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
+    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
+  ];
+  if (imageAudit) {
+    files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
+    files.push({ name: "images-summary.json", body: JSON.stringify(imageAudit.summary, null, 2), type: "application/json" });
+  }
+  if (branding) files.push({ name: "branding.json", body: JSON.stringify(branding, null, 2), type: "application/json" });
+  return files;
+}
+var BATCH, siteExtractFn;
+var init_site_extract = __esm({
+  "src/inngest/functions/site-extract.ts"() {
+    "use strict";
+    init_client();
+    init_site_extractor();
+    init_seo_link_graph();
+    init_seo_issues();
+    init_image_audit();
+    init_blob_store();
+    init_screenshot();
+    init_browser_service_env();
+    init_rates();
+    init_site_extract_repository();
+    BATCH = 30;
+    siteExtractFn = inngest.createFunction(
+      {
+        id: "site-extract",
+        retries: 2,
+        triggers: [{ event: "mcp-scraper/extract.requested" }],
+        onFailure: async ({ event }) => {
+          const jobId = event?.data?.event?.data?.jobId;
+          if (!jobId) return;
+          const current = await getExtractJob(jobId);
+          if (current && current.userId != null && current.billedMc == null) {
+            const heldMc = Number(current.options.heldMc ?? 0);
+            await settleExtractJob(jobId, current.userId, heldMc, 0, "crawl failed refund").catch(() => {
+            });
+          }
+          await failExtractJob(jobId, String(event?.data?.error?.message ?? "crawl failed")).catch(() => {
+          });
+        }
+      },
+      async ({ event, step }) => {
+        const jobId = event.data.jobId;
+        const job = await step.run("load-job", () => getExtractJob(jobId));
+        if (!job) return { jobId, status: "missing" };
+        const key = browserServiceApiKey();
+        if (!key) throw new Error("browser service key not configured");
+        const maxPages = Number(job.options.maxPages ?? 1e4);
+        const concurrency = Number(job.options.concurrency ?? 1);
+        const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
+        const urls = await step.run("discover", async () => {
+          const discovered = await discoverUrls(job.startUrl, maxPages, key);
+          await setExtractJobTotal(jobId, discovered.length);
+          return discovered;
+        });
+        const batches = chunk(urls, BATCH);
+        for (let i = 0; i < batches.length; i++) {
+          await step.run(`crawl-batch-${i}`, async () => {
+            const pages = await extractPagesRotating(batches[i], { kernelApiKey: key, concurrency, urlsPerBrowser });
+            await saveExtractPages(jobId, pages);
+            return pages.length;
+          });
+        }
+        const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
+        const imageAudit = Array.isArray(job.options.formats) && job.options.formats.includes("images") ? await step.run("image-audit", async () => {
+          const pages = await getExtractedPages(jobId);
+          return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+        }) : null;
+        const artifacts = await step.run("finalize", async () => {
+          const pages = await getExtractedPages(jobId);
+          const store = getBlobStore();
+          const files = buildArtifacts(job, pages, branding, imageAudit);
+          const stored = [];
+          for (const f of files) stored.push(await store.put(`extract/${jobId}/${f.name}`, f.body, f.type));
+          await completeExtractJob(jobId, stored);
+          return stored;
+        });
+        await step.run("settle", async () => {
+          const current = await getExtractJob(jobId);
+          if (!current || current.billedMc != null || current.userId == null) return;
+          const heldMc = Number(current.options.heldMc ?? 0);
+          const successful = await countSuccessfulPages(jobId);
+          const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
+          await settleExtractJob(jobId, current.userId, heldMc - usedMc, usedMc, new URL(job.startUrl).hostname);
+        });
+        return { jobId, status: "complete", artifacts };
+      }
+    );
+  }
+});
+
+// src/api/catalog-seed.ts
+var URL_PARAM, FORMATS_PARAM, CATALOG;
+var init_catalog_seed = __esm({
+  "src/api/catalog-seed.ts"() {
+    "use strict";
+    URL_PARAM = { key: "url", label: "URL", type: "url", required: true, placeholder: "https://example.com" };
+    FORMATS_PARAM = {
+      key: "formats",
+      label: "Formats",
+      type: "multi_enum",
+      default: ["markdown", "links"],
+      enumValues: [
+        { value: "markdown", label: "Markdown" },
+        { value: "links", label: "Links" },
+        { value: "json", label: "JSON / schema" },
+        { value: "images", label: "Image links" },
+        { value: "branding", label: "Branding" }
+      ]
+    };
+    CATALOG = {
+      groups: [
+        {
+          key: "youtube",
+          label: "YouTube",
+          icon: "youtube",
+          tools: [
+            {
+              key: "youtube_harvest",
+              label: "Search",
+              kind: "simple",
+              endpoint: "/youtube/harvest",
+              mcpName: "youtube_harvest",
+              tagline: "Find or list videos by topic or channel",
+              params: [
+                { key: "mode", label: "Mode", type: "enum", required: true, default: "search", enumValues: [{ value: "search", label: "Search" }, { value: "channel", label: "Channel" }] },
+                { key: "query", label: "Query", type: "string", placeholder: "best local SEO 2026" },
+                { key: "channelHandle", label: "Channel handle", type: "string", placeholder: "@mkbhd" },
+                { key: "maxVideos", label: "Max videos", type: "number", min: 1, max: 500, default: 50, advanced: true }
+              ],
+              examples: ["Find YouTube videos about local SEO", "Harvest videos from @mkbhd"]
+            },
+            {
+              key: "youtube_transcribe",
+              label: "Transcribe",
+              kind: "simple",
+              endpoint: "/youtube/transcribe",
+              mcpName: "youtube_transcribe",
+              tagline: "Transcribe one video to text",
+              params: [{ key: "url", label: "Video URL", type: "url", required: true, placeholder: "https://www.youtube.com/watch?v=\u2026" }],
+              examples: ["Transcribe this YouTube URL"]
+            }
+          ]
+        },
+        {
+          key: "facebook",
+          label: "Facebook",
+          icon: "facebook",
+          tools: [
+            {
+              key: "facebook_ad_search",
+              label: "Ad Search",
+              kind: "simple",
+              endpoint: "/facebook/search",
+              mcpName: "facebook_ad_search",
+              tagline: "Find ads in the Ad Library",
+              params: [
+                { key: "query", label: "Advertiser / query", type: "string", required: true },
+                { key: "country", label: "Country", type: "string", default: "US" },
+                { key: "maxResults", label: "Max results", type: "number", min: 1, max: 20, default: 10, advanced: true }
+              ],
+              examples: ["Find Facebook ads for this brand"]
+            },
+            {
+              key: "facebook_video_transcribe",
+              label: "Video Transcribe",
+              kind: "simple",
+              endpoint: "/facebook/video-transcribe",
+              mcpName: "facebook_video_transcribe",
+              tagline: "Transcribe a Facebook video/reel",
+              params: [{ ...URL_PARAM, label: "Video URL", placeholder: "https://www.facebook.com/share/v/\u2026" }]
+            },
+            {
+              key: "facebook_ad_transcribe",
+              label: "Ad Transcribe",
+              kind: "simple",
+              endpoint: "/facebook/transcribe",
+              mcpName: "facebook_ad_transcribe",
+              tagline: "Transcribe an ad creative",
+              params: [{ ...URL_PARAM, label: "Ad video URL" }]
+            },
+            {
+              key: "facebook_page_intel",
+              label: "Page Intel",
+              kind: "simple",
+              endpoint: "/facebook/page-intel",
+              mcpName: "facebook_page_intel",
+              tagline: "Advertiser page profile + ads",
+              params: [{ key: "libraryId", label: "Page / Library ID or URL", type: "string", required: true }]
+            }
+          ]
+        },
+        {
+          key: "instagram",
+          label: "Instagram",
+          icon: "instagram",
+          tools: [
+            {
+              key: "instagram_profile_content",
+              label: "Profile",
+              kind: "simple",
+              endpoint: "/instagram/profile-content",
+              mcpName: "instagram_profile_content",
+              tagline: "Inventory a profile grid",
+              params: [
+                { key: "url", label: "Profile URL or handle", type: "string", required: true, placeholder: "https://instagram.com/nasa" },
+                { key: "maxItems", label: "Max items", type: "number", min: 1, max: 2e3, default: 50, advanced: true },
+                { key: "maxScrolls", label: "Max scrolls", type: "number", min: 0, max: 250, default: 10, advanced: true }
+              ]
+            },
+            {
+              key: "instagram_media_download",
+              label: "Post / Reel",
+              kind: "simple",
+              endpoint: "/instagram/media-download",
+              mcpName: "instagram_media_download",
+              tagline: "Download a post or reel (+ transcript)",
+              params: [
+                { ...URL_PARAM, label: "Post / reel URL", placeholder: "https://www.instagram.com/reel/\u2026" },
+                { key: "includeTranscript", label: "Include transcript", type: "boolean", default: true }
+              ]
+            }
+          ]
+        },
+        {
+          key: "reddit",
+          label: "Reddit",
+          icon: "message-circle",
+          tools: [
+            {
+              key: "reddit_browser",
+              label: "Research (Browser)",
+              kind: "simple",
+              endpoint: "/agent",
+              mcpName: "browser_open",
+              tagline: "No dedicated tool yet \u2014 driven by the Browser agent",
+              params: [{ key: "query", label: "What to research", type: "string", required: true, placeholder: "ICP pain points in [niche]" }]
+            }
+          ]
+        },
+        {
+          key: "google_search",
+          label: "Google Search",
+          icon: "search",
+          tools: [
+            {
+              key: "search_serp",
+              label: "SERP",
+              kind: "simple",
+              endpoint: "/harvest/sync",
+              mcpName: "search_serp",
+              tagline: "Organic results only",
+              params: [
+                { key: "query", label: "Query", type: "string", required: true },
+                { key: "location", label: "Location", type: "string", placeholder: "Denver, CO" }
+              ],
+              examples: ["Top restaurants in San Francisco"]
+            },
+            {
+              key: "harvest_paa",
+              label: "PAA + SERP Detail",
+              kind: "simple",
+              endpoint: "/harvest/sync",
+              mcpName: "harvest_paa",
+              tagline: "People-Also-Ask, AI Overview, full SERP features",
+              params: [
+                { key: "query", label: "Query", type: "string", required: true },
+                { key: "maxQuestions", label: "Max questions", type: "number", min: 1, max: 200, default: 30, advanced: true },
+                { key: "location", label: "Location", type: "string" }
+              ]
+            }
+          ]
+        },
+        {
+          key: "google_maps",
+          label: "Google Maps",
+          icon: "map-pin",
+          tools: [
+            {
+              key: "maps_search",
+              label: "Search",
+              kind: "simple",
+              endpoint: "/maps/search",
+              mcpName: "maps_search",
+              tagline: "Find local businesses",
+              params: [
+                { key: "query", label: "Query", type: "string", required: true, placeholder: "roofers" },
+                { key: "location", label: "Location", type: "string", required: true, placeholder: "Denver, CO" },
+                { key: "maxResults", label: "Max results", type: "number", min: 1, max: 20, default: 10, advanced: true }
+              ]
+            },
+            {
+              key: "maps_place_intel",
+              label: "Place Intel",
+              kind: "simple",
+              endpoint: "/maps/place",
+              mcpName: "maps_place_intel",
+              tagline: "One place: details + reviews",
+              params: [
+                { key: "businessName", label: "Business name", type: "string", required: true },
+                { key: "location", label: "Location", type: "string", required: true },
+                { key: "includeReviews", label: "Include reviews", type: "boolean", default: false },
+                { key: "maxReviews", label: "Max reviews", type: "number", min: 1, max: 500, default: 50, advanced: true }
+              ]
+            }
+          ]
+        },
+        {
+          key: "websites",
+          label: "Websites",
+          icon: "globe",
+          tools: [
+            {
+              key: "scrape",
+              label: "Scrape",
+              kind: "modal",
+              endpoint: "/extract-url",
+              mcpName: "extract_url",
+              tagline: "Headless single or bulk page extraction",
+              modes: [
+                { groupKey: "scope", groupLabel: "Scope", optionKey: "single", optionLabel: "Headless Single", isDefault: true, endpointOverride: "/extract-url", mcpNameOverride: "extract_url" },
+                { groupKey: "scope", groupLabel: "Scope", optionKey: "bulk", optionLabel: "Headless Bulk", endpointOverride: "/extract-site", mcpNameOverride: "extract_site", paramOverrides: { rotateProxies: true } }
+              ],
+              params: [
+                URL_PARAM,
+                FORMATS_PARAM,
+                { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100, advanced: true, sourceKey: "scope:bulk" },
+                { key: "screenshot", label: "Screenshot", type: "boolean", advanced: true }
+              ],
+              examples: ["https://steenshoney.com/"]
+            },
+            {
+              key: "map_site_urls",
+              label: "Map",
+              kind: "simple",
+              endpoint: "/map-urls",
+              mcpName: "map_site_urls",
+              tagline: "URL inventory only",
+              params: [URL_PARAM, { key: "maxUrls", label: "Max URLs", type: "number", min: 1, max: 1e4, default: 100, advanced: true }]
+            },
+            {
+              key: "extract_site",
+              label: "Crawl",
+              kind: "simple",
+              endpoint: "/extract-site",
+              mcpName: "extract_site",
+              tagline: "Map + bulk scrape (content)",
+              params: [
+                URL_PARAM,
+                { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
+                FORMATS_PARAM,
+                { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true },
+                { key: "background", label: "Run in background", type: "boolean", default: true, advanced: true }
+              ],
+              examples: ["https://steenshoney.com/ (full-site content crawl)"]
+            },
+            {
+              key: "audit_site",
+              label: "Audit",
+              kind: "simple",
+              endpoint: "/extract-site",
+              mcpName: "audit_site",
+              tagline: "Technical SEO audit (Screaming-Frog style)",
+              params: [
+                URL_PARAM,
+                { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
+                { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true }
+              ],
+              examples: ["https://steenshoney.com/ (issues + link graph + image audit)"]
+            }
+          ]
+        },
+        {
+          key: "browser",
+          label: "Browser",
+          icon: "monitor",
+          tools: [
+            {
+              key: "browser_agent",
+              label: "Browser Agent",
+              kind: "simple",
+              endpoint: "/agent",
+              mcpName: "browser_open",
+              tagline: "Drive a live browser (open, navigate, read, click)",
+              params: [
+                { key: "url", label: "Start URL", type: "url" },
+                { key: "profile", label: "Use Chrome profile", type: "string", placeholder: "saved profile name", advanced: true }
+              ]
+            },
+            {
+              key: "query_fanout_workflow",
+              label: "AI Visibility (Fan-out)",
+              kind: "simple",
+              mcpName: "query_fanout_workflow",
+              tagline: "Capture what ChatGPT/Claude search & cite (AEO)",
+              params: [
+                { key: "prompt", label: "Prompt", type: "string" },
+                { key: "first_party_domain", label: "Your domain", type: "string", advanced: true },
+                { key: "export", label: "Export artifacts", type: "boolean", advanced: true }
+              ]
+            }
+          ]
+        },
+        {
+          key: "workflows",
+          label: "Workflows",
+          icon: "workflow",
+          tools: [
+            {
+              key: "directory_workflow",
+              label: "Directory",
+              kind: "simple",
+              endpoint: "/directory/run",
+              mcpName: "directory_workflow",
+              isSpecial: true,
+              tagline: "Build a local business directory",
+              params: [
+                { key: "query", label: "Business type", type: "string", required: true, placeholder: "roofers" },
+                { key: "state", label: "State", type: "string", required: true, placeholder: "TN" },
+                { key: "minPopulation", label: "Min city population", type: "number", default: 1e5, advanced: true }
+              ]
+            },
+            {
+              key: "rank_tracker_workflow",
+              label: "Rank Tracker",
+              kind: "simple",
+              mcpName: "rank_tracker_workflow",
+              isSpecial: true,
+              tagline: "Generate a rank-tracker blueprint",
+              params: [{ key: "domain", label: "Domain", type: "string", required: true }]
+            },
+            {
+              key: "deep_research_workflow",
+              label: "Deep Research",
+              kind: "simple",
+              endpoint: "/workflows/run",
+              mcpName: "workflow_run",
+              isSpecial: true,
+              tagline: "Multi-source, fact-checked research report",
+              params: [{ key: "question", label: "Question", type: "string", required: true }]
+            }
+          ]
+        }
+      ]
+    };
+  }
+});
+
+// src/api/site-audit-middleware.ts
+var import_factory2, siteAuditAuth;
+var init_site_audit_middleware = __esm({
+  "src/api/site-audit-middleware.ts"() {
+    "use strict";
+    import_factory2 = require("hono/factory");
+    init_db();
+    siteAuditAuth = (0, import_factory2.createMiddleware)(async (c, next) => {
+      const apiKey = c.req.header("x-api-key");
+      if (apiKey) {
+        const user = await getUserByApiKey(apiKey);
+        if (!user) return c.json({ error: "Unauthorized" }, 401);
+        c.set("siteAuditUserId", user.id);
+        return next();
+      }
+      return c.json({ error: "Unauthorized" }, 401);
+    });
+  }
+});
+
+// src/api/site-audit-routes.ts
+function isPathInside(root, target) {
+  const relative = import_node_path3.default.relative(root, target);
+  return relative === "" || !relative.startsWith("..") && !import_node_path3.default.isAbsolute(relative);
+}
+var import_node_path3, import_node_os3, import_hono, import_zod11, siteAuditApp;
+var init_site_audit_routes = __esm({
+  "src/api/site-audit-routes.ts"() {
+    "use strict";
+    import_node_path3 = __toESM(require("path"), 1);
+    import_node_os3 = __toESM(require("os"), 1);
+    import_hono = require("hono");
+    import_zod11 = require("zod");
+    init_site_audit_middleware();
+    init_factory();
+    init_phases();
+    init_schemas();
+    siteAuditApp = new import_hono.Hono();
+    siteAuditApp.onError((err, c) => {
+      if (err instanceof import_zod11.ZodError) {
+        return c.json({ error: "Invalid request body", issues: err.issues }, 400);
+      }
+      const msg = err instanceof Error ? err.message : String(err);
+      return c.json({ error: msg }, 500);
+    });
+    siteAuditApp.use("*", siteAuditAuth);
+    siteAuditApp.post("/audit", async (c) => {
+      const body = SiteAuditStartRequestSchema.parse(await c.req.json());
+      const allowedRoot = import_node_path3.default.resolve(process.env["SESSION_ROOT"] ?? import_node_os3.default.tmpdir());
+      const sessionPath = import_node_path3.default.resolve(body.sessionPath);
+      if (!isPathInside(allowedRoot, sessionPath)) {
+        return c.json({ error: "sessionPath must be inside the configured session root" }, 400);
+      }
+      const pathFields = [
+        body.sessionPath,
+        body.sfInternalPath,
+        body.sfInlinksPath,
+        body.sfOutlinksPath,
+        body.sitemapPath,
+        body.competitorDomainsCsvPath,
+        body.visualSitemapsPath,
+        body.gscPath,
+        body.ga4Path,
+        body.ahrefsPath
+      ];
+      for (const filePath of pathFields) {
+        if (filePath != null && !isPathInside(allowedRoot, import_node_path3.default.resolve(filePath))) {
+          return c.json({ error: "Path traversal attempt" }, 400);
+        }
+      }
+      const userId = c.get("siteAuditUserId");
+      const service = makeSiteAuditService();
+      const jobId = await service.startAudit(userId, body);
+      void dispatchSiteAuditPhase(service, "phase1-ingest", { jobId, request: body }).catch(
+        (err) => {
+          console.error(
+            "[site-audit-routes] phase1-ingest dispatch failed for job",
+            jobId,
+            err instanceof Error ? err.message : String(err)
+          );
+        }
+      );
+      return c.json({ jobId }, 202);
+    });
+    siteAuditApp.post("/ingest", async (c) => {
+      const body = SiteAuditIngestRequestSchema.parse(await c.req.json());
+      const service = makeSiteAuditService();
+      const job = await service.getJob(body.jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
+      void dispatchSiteAuditPhase(service, "phase1-ingest", {
+        jobId: body.jobId,
+        request: JSON.parse(job.request)
+      }).catch((err) => {
+        console.error(
+          "[site-audit-routes] phase1-ingest re-dispatch failed for job",
+          body.jobId,
+          err instanceof Error ? err.message : String(err)
+        );
+      });
+      return c.json({ jobId: body.jobId }, 202);
+    });
+    siteAuditApp.post("/build-graph", async (c) => {
+      const body = SiteAuditBuildGraphRequestSchema.parse(await c.req.json());
+      const service = makeSiteAuditService();
+      const job = await service.getJob(body.jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
+      void dispatchSiteAuditPhase(service, "phase2-build-graph", { jobId: body.jobId }).catch(
+        (err) => {
+          console.error(
+            "[site-audit-routes] phase2-build-graph failed for job",
+            body.jobId,
+            err instanceof Error ? err.message : String(err)
+          );
+        }
+      );
+      return c.json({ jobId: body.jobId }, 202);
+    });
+    siteAuditApp.post("/classify", async (c) => {
+      const body = SiteAuditClassifyRequestSchema.parse(await c.req.json());
+      const service = makeSiteAuditService();
+      const job = await service.getJob(body.jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
+      void dispatchSiteAuditPhase(service, "phase3-classify", { jobId: body.jobId }).catch(
+        (err) => {
+          console.error(
+            "[site-audit-routes] phase3-classify failed for job",
+            body.jobId,
+            err instanceof Error ? err.message : String(err)
+          );
+        }
+      );
+      return c.json({ jobId: body.jobId }, 202);
+    });
+    siteAuditApp.post("/compare", async (c) => {
+      const body = SiteAuditCompareRequestSchema.parse(await c.req.json());
+      const service = makeSiteAuditService();
+      const job = await service.getJob(body.jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
+      void dispatchSiteAuditPhase(service, "phase4-compare", { jobId: body.jobId }).catch(
+        (err) => {
+          console.error(
+            "[site-audit-routes] phase4-compare failed for job",
+            body.jobId,
+            err instanceof Error ? err.message : String(err)
+          );
+        }
+      );
+      return c.json({ jobId: body.jobId }, 202);
+    });
+    siteAuditApp.post("/score", async (c) => {
+      const body = SiteAuditScoreRequestSchema.parse(await c.req.json());
+      const service = makeSiteAuditService();
+      const job = await service.getJob(body.jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
+      void dispatchSiteAuditPhase(service, "phase5-synthesize", {
+        jobId: body.jobId,
+        scoringOnly: true,
+        reportOnly: false
+      }).catch((err) => {
+        console.error(
+          "[site-audit-routes] phase5-synthesize(scoringOnly) failed for job",
+          body.jobId,
+          err instanceof Error ? err.message : String(err)
+        );
+      });
+      return c.json({ jobId: body.jobId }, 202);
+    });
+    siteAuditApp.post("/report", async (c) => {
+      const body = SiteAuditReportRequestSchema.parse(await c.req.json());
+      const service = makeSiteAuditService();
+      const job = await service.getJob(body.jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== c.get("siteAuditUserId")) return c.json({ error: "Not found" }, 404);
+      void dispatchSiteAuditPhase(service, "phase5-synthesize", {
+        jobId: body.jobId,
+        scoringOnly: false,
+        reportOnly: true
+      }).catch((err) => {
+        console.error(
+          "[site-audit-routes] phase5-synthesize(reportOnly) failed for job",
+          body.jobId,
+          err instanceof Error ? err.message : String(err)
+        );
+      });
+      return c.json({ jobId: body.jobId }, 202);
+    });
+    siteAuditApp.get("/jobs/:id", async (c) => {
+      const jobId = c.req.param("id");
+      const userId = c.get("siteAuditUserId");
+      const service = makeSiteAuditService();
+      const job = await service.getJob(jobId);
+      if (!job) return c.json({ error: "Not found" }, 404);
+      if (job.user_id !== userId) return c.json({ error: "Not found" }, 404);
+      return c.json(job);
+    });
+    siteAuditApp.get("/jobs", async (c) => {
+      const userId = c.get("siteAuditUserId");
+      const service = makeSiteAuditService();
+      const jobs = await service.getJobsForUser(userId);
+      return c.json(jobs);
+    });
+  }
+});
+
 // src/youtube/schemas.ts
 var import_zod12, YouTubeHarvestOptionsSchema;
 var init_schemas2 = __esm({
@@ -9184,7 +10556,7 @@ function rankCheckContextOptions(config) {
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
   };
 }
-async function withTimeout(promise, timeoutMs, label) {
+async function withTimeout2(promise, timeoutMs, label) {
   let timeout;
   try {
     return await Promise.race([
@@ -9224,14 +10596,14 @@ function buildYouTubeChannelVideosUrl(channelInput) {
   }
   return `https://www.youtube.com/${handle}/videos`;
 }
-var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright3, import_sdk4, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS, SERP_CHALLENGE_SOLVE_POLL_MS, BrowserDriver;
+var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS, SERP_CHALLENGE_SOLVE_POLL_MS, BrowserDriver;
 var init_BrowserDriver = __esm({
   "src/driver/BrowserDriver.ts"() {
     "use strict";
     import_playwright_extra = require("playwright-extra");
     import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
-    import_playwright3 = require("playwright");
-    import_sdk4 = __toESM(require("@onkernel/sdk"), 1);
+    import_playwright4 = require("playwright");
+    import_sdk5 = __toESM(require("@onkernel/sdk"), 1);
     init_selectors();
     init_errors();
     import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
@@ -9274,7 +10646,7 @@ var init_BrowserDriver = __esm({
           serpNavigation: null
         };
         if (config.kernelApiKey) {
-          this.kernelClient = new import_sdk4.default({ apiKey: config.kernelApiKey });
+          this.kernelClient = new import_sdk5.default({ apiKey: config.kernelApiKey });
           const timeoutSeconds = positiveIntFromEnv("KERNEL_BROWSER_TIMEOUT_SECONDS", DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS);
           if (config.kernelProfileName && config.kernelProfileSaveChanges === true) {
             await ensureKernelProfile(this.kernelClient, config.kernelProfileName);
@@ -9295,7 +10667,7 @@ var init_BrowserDriver = __esm({
           let defaultProxyDisableError = null;
           if (proxyMode === "none") {
             try {
-              await withTimeout(
+              await withTimeout2(
                 this.kernelClient.browsers.update(this.kernelSessionId, { disable_default_proxy: true }),
                 5e3,
                 `Kernel session ${this.kernelSessionId} disable default proxy`
@@ -9338,7 +10710,7 @@ var init_BrowserDriver = __esm({
           if (this.debugEnabled) {
             await this.populateKernelRetrieveDebug(kernelDebug, config.kernelProxyId);
           }
-          this.browser = await import_playwright3.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
+          this.browser = await import_playwright4.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
           this.context = await this.browser.newContext(rankCheckContextOptions(config));
           await this.installEsbuildHelperShims(this.context);
           this.page = await this.context.newPage();
@@ -9375,7 +10747,7 @@ var init_BrowserDriver = __esm({
       async populateKernelRetrieveDebug(kernelDebug, requestedProxyId) {
         if (!this.kernelClient || !this.kernelSessionId) return;
         try {
-          const retrieved = await withTimeout(
+          const retrieved = await withTimeout2(
             this.kernelClient.browsers.retrieve(this.kernelSessionId),
             5e3,
             `Kernel session ${this.kernelSessionId} retrieve`
@@ -9433,7 +10805,7 @@ var init_BrowserDriver = __esm({
               error: null
             };
           }
-          await withTimeout(
+          await withTimeout2(
             debugPage.goto("https://ipapi.co/json/", { waitUntil: "domcontentloaded", timeout: 7e3 }),
             8e3,
             "browser network location navigation"
@@ -9459,7 +10831,7 @@ var init_BrowserDriver = __esm({
       }
       async loadJsonInDebugPage(debugPage, url) {
         try {
-          await withTimeout(
+          await withTimeout2(
             debugPage.goto(url, { waitUntil: "domcontentloaded", timeout: 7e3 }),
             8e3,
             `browser network location navigation ${url}`
@@ -9654,12 +11026,12 @@ var init_BrowserDriver = __esm({
               event: "kernel_browser_delete_started",
               kernel_session_id: sessionId
             }));
-            const deleteSession = withTimeout(
+            const deleteSession = withTimeout2(
               client2.browsers.deleteByID(sessionId),
               KERNEL_SESSION_DELETE_TIMEOUT_MS,
               `Kernel session ${sessionId} delete`
             );
-            const closeBrowser = withTimeout(
+            const closeBrowser = withTimeout2(
               b.close(),
               KERNEL_BROWSER_CLOSE_TIMEOUT_MS,
               `Kernel browser ${sessionId} close`
@@ -10060,14 +11432,14 @@ var init_YouTubeExtractor = __esm({
 });
 
 // src/youtube/MP3Downloader.ts
-var import_node_child_process, import_node_util, import_promises3, import_node_path3, execFileAsync, MP3Downloader;
+var import_node_child_process, import_node_util, import_promises3, import_node_path4, execFileAsync, MP3Downloader;
 var init_MP3Downloader = __esm({
   "src/youtube/MP3Downloader.ts"() {
     "use strict";
     import_node_child_process = require("child_process");
     import_node_util = require("util");
     import_promises3 = require("fs/promises");
-    import_node_path3 = __toESM(require("path"), 1);
+    import_node_path4 = __toESM(require("path"), 1);
     execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
     MP3Downloader = class {
       constructor(outputDir, reporter, concurrency = 3) {
@@ -10089,7 +11461,7 @@ var init_MP3Downloader = __esm({
       }
       async downloadOne(video) {
         const url = video.url;
-        const outputTemplate = import_node_path3.default.join(this.outputDir, "%(title)s.%(ext)s");
+        const outputTemplate = import_node_path4.default.join(this.outputDir, "%(title)s.%(ext)s");
         try {
           const { stdout } = await execFileAsync("yt-dlp", [
             "--extract-audio",
@@ -10167,24 +11539,24 @@ async function extractOnce(options) {
   return extractor.extract(options);
 }
 async function writeOutputs(result, outputDir) {
-  await import_node_fs2.promises.mkdir(outputDir, { recursive: true });
+  await import_node_fs3.promises.mkdir(outputDir, { recursive: true });
   const slug = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
   const ts = Date.now();
-  await import_node_fs2.promises.writeFile(
-    import_node_path4.default.join(outputDir, `${slug}-${ts}.json`),
+  await import_node_fs3.promises.writeFile(
+    import_node_path5.default.join(outputDir, `${slug}-${ts}.json`),
     JSON.stringify(result, null, 2),
     "utf8"
   );
   if (result.videos.length > 0) {
-    await import_node_fs2.promises.writeFile(
-      import_node_path4.default.join(outputDir, `${slug}-videos-${ts}.csv`),
+    await import_node_fs3.promises.writeFile(
+      import_node_path5.default.join(outputDir, `${slug}-videos-${ts}.csv`),
       import_papaparse.default.unparse(result.videos, { header: true }),
       "utf8"
     );
   }
   if (result.downloads.length > 0) {
-    await import_node_fs2.promises.writeFile(
-      import_node_path4.default.join(outputDir, `${slug}-downloads-${ts}.csv`),
+    await import_node_fs3.promises.writeFile(
+      import_node_path5.default.join(outputDir, `${slug}-downloads-${ts}.csv`),
       import_papaparse.default.unparse(result.downloads, { header: true }),
       "utf8"
     );
@@ -10224,13 +11596,13 @@ async function ytHarvest(rawOptions) {
     `YouTube blocked all ${MAX_ATTEMPTS} attempts. Try again in a few minutes.`
   );
 }
-var import_node_fs2, import_node_path4, import_papaparse, MAX_ATTEMPTS, RETRY_DELAY_MS;
+var import_node_fs3, import_node_path5, import_papaparse, MAX_ATTEMPTS, RETRY_DELAY_MS;
 var init_youtube_harvest = __esm({
   "src/youtube/youtube-harvest.ts"() {
     "use strict";
-    import_node_fs2 = require("fs");
+    import_node_fs3 = require("fs");
     init_browser_service_env();
-    import_node_path4 = __toESM(require("path"), 1);
+    import_node_path5 = __toESM(require("path"), 1);
     import_papaparse = __toESM(require("papaparse"), 1);
     init_schemas2();
     init_BrowserDriver();
@@ -10428,11 +11800,11 @@ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey
     const audioInfo = await captureAudioViaMediaRecorder(page);
     await driver.close();
     if (!audioInfo || audioInfo.bytes.length < 1e4) return null;
-    import_client2.fal.config({ credentials: falKey });
+    import_client3.fal.config({ credentials: falKey });
     const ext = audioInfo.mimeType.includes("ogg") ? "ogg" : "webm";
     const audioFile = new File([new Uint8Array(audioInfo.bytes)], `audio.${ext}`, { type: audioInfo.mimeType });
-    const uploadedUrl = await import_client2.fal.storage.upload(audioFile);
-    const result = await import_client2.fal.subscribe("fal-ai/wizper", {
+    const uploadedUrl = await import_client3.fal.storage.upload(audioFile);
+    const result = await import_client3.fal.subscribe("fal-ai/wizper", {
       input: { audio_url: uploadedUrl, task: "transcribe", language: "en" },
       logs: false,
       pollInterval: 3e3
@@ -10487,13 +11859,13 @@ async function fetchCaptions(videoId) {
   if (whisper) return finalizeCaptions(whisper);
   throw new Error(`No captions available for ${videoId} \u2014 all three tiers failed`);
 }
-var import_client2, WHISPER_RECORD_SECONDS;
+var import_client3, WHISPER_RECORD_SECONDS;
 var init_CaptionFetcher = __esm({
   "src/youtube/CaptionFetcher.ts"() {
     "use strict";
     init_BrowserDriver();
     init_browser_service_env();
-    import_client2 = require("@fal-ai/client");
+    import_client3 = require("@fal-ai/client");
     WHISPER_RECORD_SECONDS = 90;
   }
 });
@@ -10557,9 +11929,13 @@ var init_server_schemas = __esm({
     });
     ExtractSiteBodySchema = import_zod13.z.object({
       url: import_zod13.z.string().min(1, "url is required"),
-      maxPages: import_zod13.z.number().int().min(1).max(200).optional(),
+      maxPages: import_zod13.z.number().int().min(1).max(1e4).optional(),
       browserFallback: import_zod13.z.boolean().optional(),
-      kernelFallback: import_zod13.z.boolean().optional()
+      kernelFallback: import_zod13.z.boolean().optional(),
+      rotateProxies: import_zod13.z.boolean().optional(),
+      rotateProxyEvery: import_zod13.z.number().int().min(1).max(100).optional(),
+      background: import_zod13.z.boolean().optional(),
+      formats: import_zod13.z.array(import_zod13.z.enum(["markdown", "links", "json", "images", "branding"])).optional()
     });
     YoutubeHarvestBodySchema = import_zod13.z.object({
       mode: import_zod13.z.enum(["search", "channel"]),
@@ -10686,7 +12062,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
     return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
   }
   await expireConcurrencyLocksForUser(user.id);
-  const lockId = `cl_${(0, import_node_crypto2.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
+  const lockId = `cl_${(0, import_node_crypto3.randomUUID)().replace(/-/g, "").slice(0, 24)}`;
   const res = await getDb().execute({
     sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
           SELECT ?, ?, ?, 'active', datetime('now', ?), ?
@@ -10740,11 +12116,11 @@ async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECON
     args: [lockTtlModifier(ttlSeconds), lockId]
   });
 }
-var import_node_crypto2, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS, schemaReady;
+var import_node_crypto3, DEFAULT_LOCK_TTL_SECONDS, DEFAULT_RETRY_AFTER_SECONDS, MAX_LOCK_TTL_SECONDS, schemaReady;
 var init_concurrency_gates = __esm({
   "src/api/concurrency-gates.ts"() {
     "use strict";
-    import_node_crypto2 = require("crypto");
+    import_node_crypto3 = require("crypto");
     init_db();
     init_rates();
     DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
@@ -10768,10 +12144,10 @@ function buildTranscriptMarkdown(result) {
   if (result.chunks?.length) {
     lines.push("## Timestamped Segments");
     lines.push("");
-    for (const chunk of result.chunks) {
-      const start = fmtTs(chunk.timestamp[0]);
-      const end = fmtTs(chunk.timestamp[1]);
-      lines.push(`**[${start} \u2192 ${end}]** ${chunk.text.trim()}`);
+    for (const chunk2 of result.chunks) {
+      const start = fmtTs(chunk2.timestamp[0]);
+      const end = fmtTs(chunk2.timestamp[1]);
+      lines.push(`**[${start} \u2192 ${end}]** ${chunk2.text.trim()}`);
       lines.push("");
     }
   }
@@ -11409,20 +12785,20 @@ var init_FacebookAdExtractor = __esm({
           if (splitRe.lastIndex === m.index) splitRe.lastIndex++;
         }
         if (last < bodyText.length) adChunks.push(bodyText.slice(last));
-        const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk) => {
-          const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
+        const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk2) => {
+          const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
           const libraryId = lidM ? lidM[1] : null;
-          const statusM = chunk.match(/^(Active|Inactive)/i);
+          const statusM = chunk2.match(/^(Active|Inactive)/i);
           const status = statusM ? statusM[1] : null;
-          const startM = chunk.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
+          const startM = chunk2.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
           const started = startM ? startM[1].replace(/\s+/g, " ").trim() : null;
           const visual = adVisuals[libraryId ?? ""] ?? { imageSrc: null, videoSrc: null, videoPoster: null };
-          const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk);
+          const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk2);
           const creativeType = visual.videoSrc || hasVideoText ? "video" : visual.imageSrc ? "image" : "unknown";
           let domain = null;
           const domainRe = /\b([A-Z0-9]{2,}(?:\.[A-Z]{2,})+)\b/g;
           let dm;
-          while ((dm = domainRe.exec(chunk)) !== null) {
+          while ((dm = domainRe.exec(chunk2)) !== null) {
             const c = dm[1];
             if (c.length >= 6 && c !== "LIBRARY") {
               domain = c;
@@ -11431,9 +12807,9 @@ var init_FacebookAdExtractor = __esm({
           }
           let primaryText = null;
           let headline = null;
-          const sponsoredIdx = chunk.search(/\bSponsored\b/i);
+          const sponsoredIdx = chunk2.search(/\bSponsored\b/i);
           if (sponsoredIdx >= 0) {
-            let afterSponsored = chunk.slice(sponsoredIdx + "Sponsored".length).trim();
+            let afterSponsored = chunk2.slice(sponsoredIdx + "Sponsored".length).trim();
             afterSponsored = afterSponsored.replace(/\s*\d*:?\d+\s*\/\s*\d*:?\d+.*$/, "");
             if (domain) {
               const di = afterSponsored.indexOf(domain);
@@ -11444,18 +12820,18 @@ var init_FacebookAdExtractor = __esm({
           }
           const ctaRe2 = new RegExp("\\b(" + CTA_LABELS.map((l) => l.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|") + ")\\b", "i");
           if (domain) {
-            const di = chunk.indexOf(domain);
+            const di = chunk2.indexOf(domain);
             if (di >= 0) {
-              let afterDomain = chunk.slice(di + domain.length).trim();
+              let afterDomain = chunk2.slice(di + domain.length).trim();
               const ctaMatch = afterDomain.search(ctaRe2);
               if (ctaMatch >= 0) afterDomain = afterDomain.slice(0, ctaMatch).trim();
               if (afterDomain.length > 3 && afterDomain.length < 120) headline = afterDomain;
             }
           }
           if (!headline) {
-            const ctaIdx = chunk.search(ctaRe2);
+            const ctaIdx = chunk2.search(ctaRe2);
             if (ctaIdx > 0) {
-              const beforeCta = chunk.slice(0, ctaIdx).trimEnd();
+              const beforeCta = chunk2.slice(0, ctaIdx).trimEnd();
               const sentenceEnd = Math.max(beforeCta.lastIndexOf(". "), beforeCta.lastIndexOf("! "), beforeCta.lastIndexOf("? "));
               if (sentenceEnd >= 0) {
                 const candidate = beforeCta.slice(sentenceEnd + 2).trim();
@@ -11467,12 +12843,12 @@ var init_FacebookAdExtractor = __esm({
           }
           let cta = null;
           for (const lbl of CTA_LABELS) {
-            if (chunk.toLowerCase().includes(lbl.toLowerCase())) {
+            if (chunk2.toLowerCase().includes(lbl.toLowerCase())) {
               cta = lbl;
               break;
             }
           }
-          const multiM = chunk.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
+          const multiM = chunk2.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
           const clusterCount = multiM ? parseInt(multiM[1]) : null;
           return {
             libraryId,
@@ -11495,8 +12871,8 @@ var init_FacebookAdExtractor = __esm({
         const unknownCreativeCount = ads.filter((a) => a.creativeType === "unknown").length;
         const advertiserPageId = new URL(page.url()).searchParams.get("view_all_page_id") ?? null;
         const nameFreq = /* @__PURE__ */ new Map();
-        for (const chunk of adChunks) {
-          const detailsParts = chunk.split("See ad details ");
+        for (const chunk2 of adChunks) {
+          const detailsParts = chunk2.split("See ad details ");
           if (detailsParts.length < 2) continue;
           const sponsoredIdx = detailsParts[1].indexOf(" Sponsored");
           if (sponsoredIdx < 0) continue;
@@ -12221,7 +13597,7 @@ async function createFreshLocationProxy(kernel, options, target) {
 }
 async function deleteKernelProxyId(kernelApiKey, proxyId) {
   if (!kernelApiKey || !proxyId) return;
-  const kernel = new import_sdk5.default({ apiKey: kernelApiKey });
+  const kernel = new import_sdk6.default({ apiKey: kernelApiKey });
   await kernel.proxies.delete(proxyId);
 }
 async function resolveKernelProxyId(options) {
@@ -12235,7 +13611,7 @@ async function resolveKernelProxyId(options) {
   if (!target || !options.kernelApiKey) {
     return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
   }
-  const kernel = new import_sdk5.default({ apiKey: options.kernelApiKey });
+  const kernel = new import_sdk6.default({ apiKey: options.kernelApiKey });
   try {
     const attemptIndex = options.attemptIndex ?? 0;
     if (options.fresh) {
@@ -12341,11 +13717,11 @@ async function resolveKernelProxyId(options) {
     return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
   }
 }
-var import_sdk5, US_STATE_CODES, US_CITY_CENTER_ZIPS;
+var import_sdk6, US_STATE_CODES, US_CITY_CENTER_ZIPS;
 var init_kernel_proxy_resolver = __esm({
   "src/kernel-proxy-resolver.ts"() {
     "use strict";
-    import_sdk5 = __toESM(require("@onkernel/sdk"), 1);
+    import_sdk6 = __toESM(require("@onkernel/sdk"), 1);
     init_uule();
     US_STATE_CODES = {
       alabama: "AL",
@@ -12561,7 +13937,7 @@ function transcriptMarkdown(title, text, chunks, durationMs) {
 }
 async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript") {
   const startMs = Date.now();
-  const result = await import_client3.fal.subscribe("fal-ai/wizper", {
+  const result = await import_client4.fal.subscribe("fal-ai/wizper", {
     input: { audio_url: mediaUrl, task: "transcribe", language: "en" },
     logs: false,
     pollInterval: 3e3
@@ -12577,11 +13953,11 @@ async function transcribeMediaUrl(mediaUrl, markdownTitle = "# Media Transcript"
     markdown: transcriptMarkdown(markdownTitle, text, chunks, durationMs)
   };
 }
-var import_client3;
+var import_client4;
 var init_media_transcription = __esm({
   "src/services/media-transcription.ts"() {
     "use strict";
-    import_client3 = require("@fal-ai/client");
+    import_client4 = require("@fal-ai/client");
   }
 });
 
@@ -12624,7 +14000,7 @@ async function kernelLaunchOptsResidential() {
   }
   return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
-var import_hono4, import_zod16, import_client4, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
+var import_hono4, import_zod16, import_client5, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
 var init_facebook_ad_routes = __esm({
   "src/api/facebook-ad-routes.ts"() {
     "use strict";
@@ -12639,7 +14015,7 @@ var init_facebook_ad_routes = __esm({
     init_FacebookOrganicVideoExtractor();
     init_kernel_proxy_resolver();
     init_schemas3();
-    import_client4 = require("@fal-ai/client");
+    import_client5 = require("@fal-ai/client");
     init_api_auth();
     init_url_utils();
     init_concurrency_gates();
@@ -12782,7 +14158,7 @@ var init_facebook_ad_routes = __esm({
         metadata: { videoUrl }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client4.fal.config({ credentials: process.env.FAL_KEY });
+      import_client5.fal.config({ credentials: process.env.FAL_KEY });
       let debited = false;
       try {
         const { ok, balance_mc } = await debitMc(fbUser.id, MC_COSTS.fb_transcribe, LedgerOperation.FB_TRANSCRIBE, videoUrl);
@@ -12821,7 +14197,7 @@ var init_facebook_ad_routes = __esm({
         metadata: { url: sourceUrl.href }
       });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
-      import_client4.fal.config({ credentials: process.env.FAL_KEY });
+      import_client5.fal.config({ credentials: process.env.FAL_KEY });
       const driver = new BrowserDriver();
       let debited = false;
       try {
@@ -12915,11 +14291,11 @@ var init_facebook_ad_routes = __esm({
         }
         if (last < bodyText.length) adChunks.push(bodyText.slice(last));
         const advertiserMap = /* @__PURE__ */ new Map();
-        for (const chunk of adChunks) {
-          const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
+        for (const chunk2 of adChunks) {
+          const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
           if (!lidM) continue;
           const libraryId = lidM[1];
-          const detailsParts = chunk.split("See ad details ");
+          const detailsParts = chunk2.split("See ad details ");
           if (detailsParts.length < 2) continue;
           const afterDetails = detailsParts[1];
           const sponsoredIdx = afterDetails.indexOf(" Sponsored");
@@ -13396,7 +14772,7 @@ async function resolveInstagramLaunch(body) {
   };
 }
 function outputBaseDir() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path5.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
 }
 function safeFilePart(input) {
   return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
@@ -13404,8 +14780,8 @@ function safeFilePart(input) {
 function mediaOutputDir(shortcode, sourceUrl) {
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
-  const dir = (0, import_node_path5.join)(outputBaseDir(), "instagram", `${stamp}-${slug}`);
-  (0, import_node_fs3.mkdirSync)(dir, { recursive: true });
+  const dir = (0, import_node_path6.join)(outputBaseDir(), "instagram", `${stamp}-${slug}`);
+  (0, import_node_fs4.mkdirSync)(dir, { recursive: true });
   return dir;
 }
 async function downloadToFile(url, destPath, referer) {
@@ -13418,10 +14794,10 @@ async function downloadToFile(url, destPath, referer) {
   });
   if (!res.ok) throw new Error(`HTTP ${res.status}`);
   if (!res.body) throw new Error("Empty response body");
-  await (0, import_promises4.pipeline)(import_node_stream2.Readable.fromWeb(res.body), (0, import_node_fs3.createWriteStream)(destPath));
+  await (0, import_promises4.pipeline)(import_node_stream2.Readable.fromWeb(res.body), (0, import_node_fs4.createWriteStream)(destPath));
   return {
     savedPath: destPath,
-    sizeBytes: (0, import_node_fs3.statSync)(destPath).size,
+    sizeBytes: (0, import_node_fs4.statSync)(destPath).size,
     mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
   };
 }
@@ -13443,8 +14819,8 @@ function runFfmpegMux(videoPath, audioPath, outPath) {
       stdio: ["ignore", "ignore", "pipe"]
     });
     let stderr = "";
-    child.stderr.on("data", (chunk) => {
-      stderr += String(chunk);
+    child.stderr.on("data", (chunk2) => {
+      stderr += String(chunk2);
     });
     child.on("error", (err) => resolve({ ok: false, error: err.message }));
     child.on("close", (code) => resolve({ ok: code === 0, error: code === 0 ? null : stderr.trim() || `ffmpeg exited ${code}` }));
@@ -13456,15 +14832,15 @@ function trackFilename(track, index, shortcode) {
   const bitrate = track.bitrate ? `-${track.bitrate}` : "";
   return `${label}-${type}-${index}${bitrate}.mp4`;
 }
-var import_hono5, import_zod17, import_node_fs3, import_node_os3, import_node_path5, import_node_stream2, import_promises4, import_node_child_process2, InstagramProfileContentBodySchema, InstagramMediaTypeSchema, InstagramMediaDownloadBodySchema, instagramApp;
+var import_hono5, import_zod17, import_node_fs4, import_node_os4, import_node_path6, import_node_stream2, import_promises4, import_node_child_process2, InstagramProfileContentBodySchema, InstagramMediaTypeSchema, InstagramMediaDownloadBodySchema, instagramApp;
 var init_instagram_routes = __esm({
   "src/api/instagram-routes.ts"() {
     "use strict";
     import_hono5 = require("hono");
     import_zod17 = require("zod");
-    import_node_fs3 = require("fs");
-    import_node_os3 = require("os");
-    import_node_path5 = require("path");
+    import_node_fs4 = require("fs");
+    import_node_os4 = require("os");
+    import_node_path6 = require("path");
     import_node_stream2 = require("stream");
     import_promises4 = require("stream/promises");
     import_node_child_process2 = require("child_process");
@@ -13592,16 +14968,16 @@ var init_instagram_routes = __esm({
         let outputDir = null;
         if (body.downloadMedia) {
           outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
-          const textPath = (0, import_node_path5.join)(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
-          (0, import_node_fs3.writeFileSync)(textPath, [
+          const textPath = (0, import_node_path6.join)(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
+          (0, import_node_fs4.writeFileSync)(textPath, [
             `URL: ${extraction.pageUrl}`,
             extraction.caption ? `Caption: ${extraction.caption}` : "",
             "",
             extraction.bodyText
           ].filter(Boolean).join("\n"), "utf8");
-          downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: (0, import_node_fs3.statSync)(textPath).size, mimeType: "text/plain", error: null });
+          downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: (0, import_node_fs4.statSync)(textPath).size, mimeType: "text/plain", error: null });
           if (mediaTypes.has("image") && extraction.imageUrl) {
-            const imagePathBase = (0, import_node_path5.join)(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
+            const imagePathBase = (0, import_node_path6.join)(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
             try {
               const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
               const ext = extFromMime(downloaded.mimeType, "jpg");
@@ -13610,7 +14986,7 @@ var init_instagram_routes = __esm({
                 const { renameSync } = await import("fs");
                 renameSync(downloaded.savedPath, finalPath);
               }
-              downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: (0, import_node_fs3.statSync)(finalPath).size, mimeType: downloaded.mimeType, error: null });
+              downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: (0, import_node_fs4.statSync)(finalPath).size, mimeType: downloaded.mimeType, error: null });
             } catch (err) {
               downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
             }
@@ -13621,7 +14997,7 @@ var init_instagram_routes = __esm({
             if (track.streamType === "video" && !mediaTypes.has("video")) continue;
             if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
             const kind = track.streamType === "audio" ? "audio" : "video";
-            const target = (0, import_node_path5.join)(outputDir, trackFilename(track, index, extraction.shortcode));
+            const target = (0, import_node_path6.join)(outputDir, trackFilename(track, index, extraction.shortcode));
             try {
               const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
               downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
@@ -13631,10 +15007,10 @@ var init_instagram_routes = __esm({
             }
           }
           if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
-            const outPath = (0, import_node_path5.join)(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
+            const outPath = (0, import_node_path6.join)(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
             const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
             if (muxed.ok) {
-              downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: (0, import_node_fs3.statSync)(outPath).size, mimeType: "video/mp4", error: null });
+              downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: (0, import_node_fs4.statSync)(outPath).size, mimeType: "video/mp4", error: null });
             } else {
               warnings.push(`Mux skipped/failed: ${muxed.error}`);
             }
@@ -14669,13 +16045,13 @@ var init_maps_routes = __esm({
 });
 
 // src/mcp/workflow-catalog.ts
-function normalize(value) {
+function normalize2(value) {
   return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
 }
 function suggestWorkflowRecipes(goal, limit = 3) {
-  const normalized = normalize(goal);
+  const normalized = normalize2(goal);
   const scored = WORKFLOW_RECIPES.map((recipe) => {
-    const haystack = normalize([
+    const haystack = normalize2([
       recipe.id,
       recipe.title,
       recipe.description,
@@ -14806,16 +16182,119 @@ function reportTitle(full) {
   return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
 }
 function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path7.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
 }
 function saveFullReport(full) {
   if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
   const outDir = outputBaseDir2();
   try {
-    (0, import_node_fs4.mkdirSync)(outDir, { recursive: true });
+    (0, import_node_fs5.mkdirSync)(outDir, { recursive: true });
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const file = (0, import_node_path7.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
+    (0, import_node_fs5.writeFileSync)(file, full, "utf8");
+    return file;
+  } catch {
+    return null;
+  }
+}
+function reportSavingActive() {
+  return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
+}
+function saveBulkSite(siteUrl, pages, seo, imageAudit) {
+  if (!reportSavingActive()) return null;
+  try {
+    const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+    const dir = (0, import_node_path7.join)(outputBaseDir2(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
+    const pagesDir = (0, import_node_path7.join)(dir, "pages");
+    (0, import_node_fs5.mkdirSync)(pagesDir, { recursive: true });
+    const indexRows = pages.map((p, i) => {
+      const num = String(i + 1).padStart(4, "0");
+      const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
+      const fname = `${num}-${slug}.md`;
+      const body = (p.bodyMarkdown ?? "").trim();
+      const content = [
+        `# ${p.title ?? "Untitled"}`,
+        `- **URL:** ${p.url}`,
+        p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
+        p.schemaTypes?.length ? `- **Schema:** ${p.schemaTypes.join(", ")}` : "",
+        "",
+        body || "_(no content extracted)_"
+      ].filter(Boolean).join("\n");
+      (0, import_node_fs5.writeFileSync)((0, import_node_path7.join)(pagesDir, fname), content, "utf8");
+      return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
+    });
+    const dataFilesSection = seo ? [
+      `
+## Data files (in this folder)`,
+      `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
+      `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
+      `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
+      `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
+      `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      ...imageAudit ? [
+        `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
+        `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
+      ] : [],
+      ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
+    ].join("\n") : "";
+    const index = [
+      `# Site Extract: ${siteUrl}`,
+      `**${pages.length} pages** crawled. Everything is in this folder.`,
+      `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
+      seo ? `- SEO crawl data: the data files listed below.` : "",
+      dataFilesSection,
+      `
+## Pages
+| # | Title | URL | File |
+|---|-------|-----|------|
+${indexRows.join("\n")}`
+    ].filter(Boolean).join("\n");
+    const indexFile = (0, import_node_path7.join)(dir, "index.md");
+    (0, import_node_fs5.writeFileSync)(indexFile, index, "utf8");
+    let seoFiles;
+    if (seo) {
+      const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
+      const pagesJsonl = (0, import_node_path7.join)(dir, "pages.jsonl");
+      const linksJsonl = (0, import_node_path7.join)(dir, "links.jsonl");
+      const metricsJsonl = (0, import_node_path7.join)(dir, "link-metrics.jsonl");
+      const issuesFile = (0, import_node_path7.join)(dir, "issues.json");
+      const reportFile = (0, import_node_path7.join)(dir, "report.md");
+      (0, import_node_fs5.writeFileSync)(pagesJsonl, toJsonl(seo.pageRows), "utf8");
+      (0, import_node_fs5.writeFileSync)(linksJsonl, toJsonl(seo.edges), "utf8");
+      (0, import_node_fs5.writeFileSync)(metricsJsonl, toJsonl(seo.metrics), "utf8");
+      (0, import_node_fs5.writeFileSync)(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
+      (0, import_node_fs5.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
+
+${renderImageSection(imageAudit)}` : ""), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      if (imageAudit) {
+        const imagesJsonl = (0, import_node_path7.join)(dir, "images.jsonl");
+        const imagesSummary = (0, import_node_path7.join)(dir, "images-summary.json");
+        (0, import_node_fs5.writeFileSync)(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
+        (0, import_node_fs5.writeFileSync)(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
+        seoFiles.push(imagesJsonl, imagesSummary);
+      }
+      if (seo.branding) {
+        const brandingFile = (0, import_node_path7.join)(dir, "branding.json");
+        (0, import_node_fs5.writeFileSync)(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
+        seoFiles.push(brandingFile);
+      }
+    }
+    return { dir, indexFile, fileCount: pages.length, seoFiles };
+  } catch {
+    return null;
+  }
+}
+function saveUrlInventory(siteUrl, urls) {
+  if (!reportSavingActive()) return null;
+  try {
+    const outDir = outputBaseDir2();
+    (0, import_node_fs5.mkdirSync)(outDir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-    const file = (0, import_node_path6.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
-    (0, import_node_fs4.writeFileSync)(file, full, "utf8");
+    const file = (0, import_node_path7.join)(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
+    const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
+    const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
+    (0, import_node_fs5.writeFileSync)(file, rows.join("\n"), "utf8");
     return file;
   } catch {
     return null;
@@ -14824,19 +16303,19 @@ function saveFullReport(full) {
 function persistScreenshotLocally(base64, url) {
   if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
   try {
-    const dir = (0, import_node_path6.join)(outputBaseDir2(), "screenshots");
-    (0, import_node_fs4.mkdirSync)(dir, { recursive: true });
+    const dir = (0, import_node_path7.join)(outputBaseDir2(), "screenshots");
+    (0, import_node_fs5.mkdirSync)(dir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
     const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
-    const filePath = (0, import_node_path6.join)(dir, `${stamp}-${slug}.png`);
-    (0, import_node_fs4.writeFileSync)(filePath, Buffer.from(base64, "base64"));
+    const filePath = (0, import_node_path7.join)(dir, `${stamp}-${slug}.png`);
+    (0, import_node_fs5.writeFileSync)(filePath, Buffer.from(base64, "base64"));
     return filePath;
   } catch {
     return null;
   }
 }
-function oneBlock(content) {
-  const filePath = saveFullReport(content);
+function oneBlock(content, diskContent) {
+  const filePath = saveFullReport(diskContent ?? content);
   const text = filePath ? `${content}
 
 \u{1F4C4} Saved: \`${filePath}\`` : content;
@@ -15105,7 +16584,12 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
   ].filter(Boolean).join("\n") : "";
   const bodySection = bodyMd ? `
 ## Page Content
-${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
+${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? `
+
+*(truncated to 3,000 of ${bodyMd.length.toLocaleString()} chars \u2014 full content in the saved report)*` : ""}` : "";
+  const bodySectionFull = bodyMd ? `
+## Page Content
+${bodyMd}` : "";
   const screenshotSection = screenshotMeta ? `
 ## Screenshot
 - **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
@@ -15136,7 +16620,10 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
   const full = `# URL Extract: ${url}
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
-  const textResult = oneBlock(full);
+  const diskReport = `# URL Extract: ${url}
+**${title}**
+${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${screenshotSection}${mediaSection}${tips}`;
+  const textResult = oneBlock(full, diskReport);
   const structuredContent = {
     url,
     title: d.title ?? null,
@@ -15169,7 +16656,16 @@ function formatMapSiteUrls(raw, input) {
   const ok = urls.filter((u) => (u.status ?? 0) >= 200 && (u.status ?? 0) < 300);
   const broken = urls.filter((u) => u.status !== null && u.status >= 400);
   const redirects = urls.filter((u) => u.status !== null && u.status >= 300 && u.status < 400);
-  const urlRows = urls.slice(0, 200).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
+  const isBulk = urls.length > BULK_URL_THRESHOLD;
+  const inventoryFile = isBulk ? saveUrlInventory(input.url, urls.map((u) => ({ url: u.url, status: u.status ?? null }))) : null;
+  const inlineCount = isBulk ? BULK_URL_THRESHOLD : 200;
+  const urlRows = urls.slice(0, inlineCount).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
+  const inventoryNote = isBulk ? inventoryFile ? `
+## \u{1F4C1} Full inventory saved
+- **File:** \`${inventoryFile}\` (CSV: url, status \u2014 all ${urls.length} URLs)
+Showing the first ${inlineCount} below; read the file for the complete list.` : `
+## \u26A0\uFE0F Full inventory not saved
+Report saving is disabled, so only the first ${inlineCount} of ${urls.length} URLs are shown. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the full inventory.` : "";
   const full = [
     `# URL Map: ${input.url}`,
     `**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s${d.truncated ? " \xB7 *truncated*" : ""}`,
@@ -15178,12 +16674,13 @@ function formatMapSiteUrls(raw, input) {
 - \u2705 2xx: ${ok.length}
 - \u{1F500} 3xx: ${redirects.length}
 - \u274C 4xx+: ${broken.length}`,
+    inventoryNote,
     `
-## URL Inventory
+## URL Inventory${isBulk ? ` (first ${inlineCount} of ${urls.length})` : ""}
 | # | URL | Status |
 |---|-----|--------|
 ${urlRows}`,
-    broken.length ? `
+    !isBulk && broken.length ? `
 ## Broken URLs
 ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
     `
@@ -15201,47 +16698,164 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
       okCount: ok.length,
       redirectCount: redirects.length,
       brokenCount: broken.length,
+      inventoryFile: inventoryFile ?? null,
       urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
       durationMs: d.durationMs ?? 0
     }
   };
 }
+function buildSeoExport(siteUrl, pages, branding) {
+  const { edges, metrics } = buildLinkGraph(pages, siteUrl);
+  const pageRows = pages.map((p) => {
+    const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
+    const m = metrics.get(p.url);
+    return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
+  });
+  const issues = computeIssues(pages, metrics);
+  return {
+    pageRows,
+    edges,
+    metrics: [...metrics.values()],
+    issues,
+    reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    branding: branding ?? void 0
+  };
+}
 function formatExtractSite(raw, input) {
   const parsed = parseData(raw);
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
   const d = parsed.data;
   const pages = d.pages ?? [];
-  const pageRows = pages.map((p, i) => {
-    const schemaInfo = p.kpo?.type?.join(", ") ?? (Array.isArray(p.schema) && p.schema.length ? `${p.schema.length} block(s)` : "\u2014");
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const pageRow = (p, i) => {
+    const schemaInfo = schemaTypesOf(p).join(", ") || "\u2014";
     return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | ${schemaInfo} |`;
-  }).join("\n");
-  const full = [
+  };
+  const tips = `
+---
+\u{1F4A1} **Tips**
+- Map URLs first: use \`map_site_urls\`
+- Inspect a single page: use \`extract_url\``;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    pages: pages.map((p) => ({
+      url: String(p.url ?? ""),
+      title: p.title ?? null,
+      schemaTypes: schemaTypesOf(p)
+    })),
+    durationMs: d.durationMs ?? 0
+  };
+  if (pages.length > BULK_PAGE_THRESHOLD) {
+    const bulk = saveBulkSite(input.url, pages.map((p) => ({
+      url: String(p.url ?? ""),
+      title: p.title ?? null,
+      bodyMarkdown: p.bodyMarkdown ?? null,
+      metaDescription: p.metaDescription ?? null,
+      schemaTypes: schemaTypesOf(p)
+    })));
+    const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
+    const location2 = bulk ? `
+## \u{1F4C1} Bulk scrape saved
+- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
+- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
+- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
+
+The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
+
+\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
+## \u26A0\uFE0F Bulk scrape not saved
+Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
+    const full2 = [
+      `# Site Extract: ${input.url}`,
+      durationLine,
+      location2,
+      `
+## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
+| # | Title | URL | Schema |
+|---|-------|-----|--------|
+${preview}`,
+      tips
+    ].join("\n");
+    return { content: [{ type: "text", text: full2 }], structuredContent: { ...structuredContent, bulkFolder: bulk?.dir ?? null } };
+  }
+  const header = [
     `# Site Extract: ${input.url}`,
-    `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`,
+    durationLine,
     `
 ## Pages
 | # | Title | URL | Schema |
 |---|-------|-----|--------|
-${pageRows}`,
-    `
----
-\u{1F4A1} **Tips**
-- Map URLs first: use \`map_site_urls\`
-- Inspect a single page: use \`extract_url\``
+${pages.map(pageRow).join("\n")}`
   ].join("\n");
-  return {
-    ...oneBlock(full),
-    structuredContent: {
-      url: input.url,
-      pageCount: pages.length,
-      pages: pages.map((p) => ({
-        url: String(p.url ?? ""),
-        title: p.title ?? null,
-        schemaTypes: p.kpo?.type ?? []
-      })),
-      durationMs: d.durationMs ?? 0
-    }
+  const full = `${header}${tips}`;
+  const pageDetails = pages.map((p, i) => {
+    const body = (p.bodyMarkdown ?? "").trim();
+    return [
+      `
+## ${i + 1}. ${p.title ?? "Untitled"}`,
+      `- **URL:** ${p.url}`,
+      p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
+      body ? `
+${body}` : "_(no content extracted)_"
+    ].filter(Boolean).join("\n");
+  }).join("\n");
+  const diskReport = `${header}
+
+---
+# Full Page Content
+${pageDetails}${tips}`;
+  return { ...oneBlock(full, diskReport), structuredContent };
+}
+async function formatAuditSite(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const pages = d.pages ?? [];
+  const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
+  const seo = buildSeoExport(input.url, pages, d.branding);
+  const imageAudit = await auditImages(pages, { concurrency: 12 });
+  const bulk = saveBulkSite(input.url, pages.map((p) => ({
+    url: String(p.url ?? ""),
+    title: p.title ?? null,
+    bodyMarkdown: p.bodyMarkdown ?? null,
+    metaDescription: p.metaDescription ?? null,
+    schemaTypes: schemaTypesOf(p)
+  })), seo, imageAudit);
+  const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
+  const img = imageAudit.summary;
+  const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
+  const structuredContent = {
+    url: input.url,
+    pageCount: pages.length,
+    durationMs: d.durationMs ?? 0,
+    bulkFolder: bulk?.dir ?? null,
+    issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
   };
+  const location2 = bulk ? `
+## \u{1F4C1} Technical audit saved
+- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
+- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
+- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
+
+Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
+## \u26A0\uFE0F Audit not saved
+Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
+  const full = [
+    `# Technical SEO Audit: ${input.url}`,
+    durationLine,
+    location2,
+    `
+## Top issues
+${topIssues || "_none found_"}`,
+    `
+## Images
+${imgLine}`
+  ].join("\n");
+  return { content: [{ type: "text", text: full }], structuredContent };
 }
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
@@ -16417,16 +18031,21 @@ ${rows}` : ""
     }
   };
 }
-var import_node_fs4, import_node_os4, import_node_path6, reportSavingEnabled;
+var import_node_fs5, import_node_os5, import_node_path7, reportSavingEnabled, BULK_PAGE_THRESHOLD, BULK_URL_THRESHOLD;
 var init_mcp_response_formatter = __esm({
   "src/mcp/mcp-response-formatter.ts"() {
     "use strict";
-    import_node_fs4 = require("fs");
-    import_node_os4 = require("os");
-    import_node_path6 = require("path");
+    import_node_fs5 = require("fs");
+    import_node_os5 = require("os");
+    import_node_path7 = require("path");
     init_errors();
     init_workflow_catalog();
+    init_seo_link_graph();
+    init_seo_issues();
+    init_image_audit();
     reportSavingEnabled = true;
+    BULK_PAGE_THRESHOLD = 25;
+    BULK_URL_THRESHOLD = 500;
   }
 });
 
@@ -16819,11 +18438,11 @@ function csvRowsFor(result) {
   return rows;
 }
 async function saveDirectoryCsv(result) {
-  const outDir = (0, import_node_path7.join)(outputBaseDir2(), "directory-workflows");
+  const outDir = (0, import_node_path8.join)(outputBaseDir2(), "directory-workflows");
   await (0, import_promises6.mkdir)(outDir, { recursive: true });
   const stamp = result.extractedAt.replace(/[:.]/g, "-");
   const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
-  const path6 = (0, import_node_path7.join)(outDir, `${stamp}-${slug}-directory-workflow.csv`);
+  const path6 = (0, import_node_path8.join)(outDir, `${stamp}-${slug}-directory-workflow.csv`);
   const headers = [
     "source_query",
     "source_location",
@@ -16881,12 +18500,12 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
   const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
   return { ...base, csvPath };
 }
-var import_promises6, import_node_path7, import_zod19, DirectoryWorkflowOptionsSchema;
+var import_promises6, import_node_path8, import_zod19, DirectoryWorkflowOptionsSchema;
 var init_directory_workflow = __esm({
   "src/directory/directory-workflow.ts"() {
     "use strict";
     import_promises6 = require("fs/promises");
-    import_node_path7 = require("path");
+    import_node_path8 = require("path");
     import_zod19 = require("zod");
     init_mcp_response_formatter();
     init_browser_service_env();
@@ -17013,7 +18632,7 @@ var init_slugify = __esm({
 
 // src/workflows/artifact-writer.ts
 function workflowOutputBaseDir(outputDir) {
-  return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path8.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
+  return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path9.join)((0, import_node_os6.homedir)(), "Downloads", "mcp-scraper");
 }
 function timestamp() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -17022,11 +18641,11 @@ function safeSlug(value) {
   return slugify(value).replace(/^-+|-+$/g, "").slice(0, 80) || "run";
 }
 function indexPath(baseDir = workflowOutputBaseDir()) {
-  return (0, import_node_path8.join)(baseDir, "workflows", "index.json");
+  return (0, import_node_path9.join)(baseDir, "workflows", "index.json");
 }
 async function readIndex(baseDir) {
   const path6 = indexPath(baseDir);
-  if (!(0, import_node_fs5.existsSync)(path6)) return { runs: [] };
+  if (!(0, import_node_fs6.existsSync)(path6)) return { runs: [] };
   try {
     return JSON.parse(await (0, import_promises7.readFile)(path6, "utf8"));
   } catch {
@@ -17048,17 +18667,17 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
     summary: `${manifest.title} (${manifest.status})`
   };
   index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
-  await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
+  await (0, import_promises7.mkdir)((0, import_node_path9.dirname)(path6), { recursive: true });
   await (0, import_promises7.writeFile)(path6, JSON.stringify(index, null, 2), "utf8");
 }
-var import_promises7, import_node_fs5, import_node_os5, import_node_path8, import_node_child_process3, ArtifactWriter;
+var import_promises7, import_node_fs6, import_node_os6, import_node_path9, import_node_child_process3, ArtifactWriter;
 var init_artifact_writer = __esm({
   "src/workflows/artifact-writer.ts"() {
     "use strict";
     import_promises7 = require("fs/promises");
-    import_node_fs5 = require("fs");
-    import_node_os5 = require("os");
-    import_node_path8 = require("path");
+    import_node_fs6 = require("fs");
+    import_node_os6 = require("os");
+    import_node_path9 = require("path");
     import_node_child_process3 = require("child_process");
     init_csv();
     init_slugify();
@@ -17085,7 +18704,7 @@ var init_artifact_writer = __esm({
         const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
         const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
         const baseDir = workflowOutputBaseDir(outputDir);
-        const runDir = (0, import_node_path8.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
+        const runDir = (0, import_node_path9.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
         await (0, import_promises7.mkdir)(runDir, { recursive: true });
         return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
       }
@@ -17095,20 +18714,20 @@ var init_artifact_writer = __esm({
         return path6;
       }
       async writeJson(label, relativePath, data) {
-        const path6 = (0, import_node_path8.join)(this.runDir, relativePath);
-        await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
+        const path6 = (0, import_node_path9.join)(this.runDir, relativePath);
+        await (0, import_promises7.mkdir)((0, import_node_path9.dirname)(path6), { recursive: true });
         await (0, import_promises7.writeFile)(path6, JSON.stringify(data, null, 2), "utf8");
         return this.remember("json", label, path6);
       }
       async writeText(label, relativePath, text, kind = "markdown") {
-        const path6 = (0, import_node_path8.join)(this.runDir, relativePath);
-        await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
+        const path6 = (0, import_node_path9.join)(this.runDir, relativePath);
+        await (0, import_promises7.mkdir)((0, import_node_path9.dirname)(path6), { recursive: true });
         await (0, import_promises7.writeFile)(path6, text, "utf8");
         return this.remember(kind, label, path6);
       }
       async writeCsv(label, relativePath, headers, rows) {
-        const path6 = (0, import_node_path8.join)(this.runDir, relativePath);
-        await (0, import_promises7.mkdir)((0, import_node_path8.dirname)(path6), { recursive: true });
+        const path6 = (0, import_node_path9.join)(this.runDir, relativePath);
+        await (0, import_promises7.mkdir)((0, import_node_path9.dirname)(path6), { recursive: true });
         await (0, import_promises7.writeFile)(path6, rowsToCsv(headers, rows), "utf8");
         return this.remember("csv", label, path6, rows.length);
       }
@@ -17129,7 +18748,7 @@ var init_artifact_writer = __esm({
           errors,
           counts
         };
-        const path6 = (0, import_node_path8.join)(this.runDir, "manifest.json");
+        const path6 = (0, import_node_path9.join)(this.runDir, "manifest.json");
         await (0, import_promises7.writeFile)(path6, JSON.stringify(manifest, null, 2), "utf8");
         await updateWorkflowIndex(manifest, path6, this.baseDir);
         return path6;
@@ -18709,7 +20328,7 @@ async function readManifestFromSummary(summary) {
 function webhookSignature(body, timestamp2) {
   const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
   if (!secret2) return null;
-  return (0, import_node_crypto3.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
+  return (0, import_node_crypto4.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
 }
 async function deliverWorkflowWebhook(input) {
   if (!input.webhookUrl) return;
@@ -18916,11 +20535,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
   }
   return { dispatched: results.length, results };
 }
-var import_node_crypto3, import_promises9, import_hono8, import_zod25, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
+var import_node_crypto4, import_promises9, import_hono8, import_zod25, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
 var init_workflow_routes = __esm({
   "src/api/workflow-routes.ts"() {
     "use strict";
-    import_node_crypto3 = require("crypto");
+    import_node_crypto4 = require("crypto");
     import_promises9 = require("fs/promises");
     import_hono8 = require("hono");
     import_zod25 = require("zod");
@@ -19201,7 +20820,7 @@ var init_workflow_routes = __esm({
 // src/serp-intelligence/page-snapshot-extractor.ts
 function sha256(value) {
   if (!value) return null;
-  return (0, import_node_crypto4.createHash)("sha256").update(value).digest("hex");
+  return (0, import_node_crypto5.createHash)("sha256").update(value).digest("hex");
 }
 function countWords(markdown) {
   const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
@@ -19250,7 +20869,7 @@ function countLinks(html, baseUrl) {
     totalLinkCount: internalLinkCount + externalLinkCount
   };
 }
-function withTimeout2(operation, timeoutMs) {
+function withTimeout3(operation, timeoutMs) {
   return new Promise((resolve, reject) => {
     const timer = setTimeout(() => reject(new Error(`Page snapshot timed out after ${timeoutMs}ms`)), timeoutMs);
     operation.then(
@@ -19367,7 +20986,7 @@ async function capturePageSnapshot(target, options = {}) {
     });
   }
   try {
-    const extraction = await withTimeout2(
+    const extraction = await withTimeout3(
       extractKpo({ url: checked.parsed.href, kernelApiKey }),
       timeoutMs
     );
@@ -19501,11 +21120,11 @@ async function capturePageSnapshots(targets, options = {}) {
     }
   };
 }
-var import_node_crypto4, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
+var import_node_crypto5, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
 var init_page_snapshot_extractor = __esm({
   "src/serp-intelligence/page-snapshot-extractor.ts"() {
     "use strict";
-    import_node_crypto4 = require("crypto");
+    import_node_crypto5 = require("crypto");
     import_p_limit3 = __toESM(require("p-limit"), 1);
     init_kpo_extractor();
     init_url_utils();
@@ -20750,52 +22369,52 @@ var init_PAAExtractor = __esm({
 });
 
 // src/output/OutputSerializer.ts
-var import_node_fs6, import_node_path9, import_papaparse2, OutputSerializer;
+var import_node_fs7, import_node_path10, import_papaparse2, OutputSerializer;
 var init_OutputSerializer = __esm({
   "src/output/OutputSerializer.ts"() {
     "use strict";
-    import_node_fs6 = require("fs");
-    import_node_path9 = __toESM(require("path"), 1);
+    import_node_fs7 = require("fs");
+    import_node_path10 = __toESM(require("path"), 1);
     import_papaparse2 = __toESM(require("papaparse"), 1);
     OutputSerializer = class {
       async writeJSON(result, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const filename = `${slug}-${Date.now()}.json`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
         return fullPath;
       }
       async writeCSV(rows, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const seedRaw = rows[0]?.seed_query ?? "paa";
         const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const csv = import_papaparse2.default.unparse(rows, { header: true });
         const filename = `${slug}-${Date.now()}.csv`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
         return fullPath;
       }
       async writeVideoCSV(videos, seed, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const csv = import_papaparse2.default.unparse(videos, { header: true });
         const filename = `${slug}-videos-${Date.now()}.csv`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
         return fullPath;
       }
       async writeForumCSV(forums, seed, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const csv = import_papaparse2.default.unparse(forums, { header: true });
         const filename = `${slug}-forums-${Date.now()}.csv`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
         return fullPath;
       }
       async writeAIOverviewCSV(citations, text, seed, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const rows = citations.map((c, i) => ({
           seed_query: seed,
@@ -20805,12 +22424,12 @@ var init_OutputSerializer = __esm({
         }));
         const csv = import_papaparse2.default.unparse(rows, { header: true });
         const filename = `${slug}-ai-overview-${Date.now()}.csv`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
         return fullPath;
       }
       async writeAIModeCSV(citations, text, seed, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const rows = citations.map((c, i) => ({
           seed_query: seed,
@@ -20820,18 +22439,18 @@ var init_OutputSerializer = __esm({
         }));
         const csv = import_papaparse2.default.unparse(rows, { header: true });
         const filename = `${slug}-ai-mode-${Date.now()}.csv`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
         return fullPath;
       }
       async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
-        await import_node_fs6.promises.mkdir(outputDir, { recursive: true });
+        await import_node_fs7.promises.mkdir(outputDir, { recursive: true });
         const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
         const rows = cards.map((c) => ({ seed_query: seed, ...c }));
         const csv = import_papaparse2.default.unparse(rows, { header: true });
         const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
-        const fullPath = import_node_path9.default.join(outputDir, filename);
-        await import_node_fs6.promises.writeFile(fullPath, csv, "utf8");
+        const fullPath = import_node_path10.default.join(outputDir, filename);
+        await import_node_fs7.promises.writeFile(fullPath, csv, "utf8");
         return fullPath;
       }
     };
@@ -21958,12 +23577,45 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.14";
+    PACKAGE_VERSION = "0.3.16";
+  }
+});
+
+// src/mcp/server-instructions.ts
+var SERVER_INSTRUCTIONS;
+var init_server_instructions = __esm({
+  "src/mcp/server-instructions.ts"() {
+    "use strict";
+    SERVER_INSTRUCTIONS = `
+This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
+this routing map, then load its schema before calling.
+
+ROUTING
+- One web page -> extract_url. Whole site (crawl + SEO report) -> extract_site. Just the URL list -> map_site_urls.
+- Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail -> harvest_paa.
+- Google Maps: find places -> maps_search; one place deep-dive + reviews -> maps_place_intel.
+- YouTube: find or list videos -> youtube_harvest; transcribe one video -> youtube_transcribe.
+- Facebook: find ads -> facebook_ad_search; transcribe an ad -> facebook_ad_transcribe; transcribe a
+  video -> facebook_video_transcribe; page profile/intel -> facebook_page_intel.
+- Instagram: profile inventory -> instagram_profile_content; one post or reel -> instagram_media_download.
+- Workflows (multi-step): local directory build -> directory_workflow; rank blueprint -> rank_tracker_workflow;
+  AI-answer citation fan-out (AEO) -> query_fanout_workflow; deep research -> deep_research_workflow.
+- Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) -> the browser_* agent
+  (browser_open, then navigate/read); save/reuse logins via browser_profile_*.
+
+NOTES
+- Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
+  It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
+- Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
+  cheapest tool that answers the question (plain search/extract before browser agents).
+- Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
+  for full detail rather than expecting the whole payload inline.
+`.trim();
   }
 });
 
 // src/mcp/mcp-tool-schemas.ts
-var import_zod27, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
+var import_zod27, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
 var init_mcp_tool_schemas = __esm({
   "src/mcp/mcp-tool-schemas.ts"() {
     "use strict";
@@ -21991,11 +23643,20 @@ var init_mcp_tool_schemas = __esm({
     };
     MapSiteUrlsInputSchema = {
       url: import_zod27.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
-      maxUrls: import_zod27.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
+      maxUrls: import_zod27.z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
     };
     ExtractSiteInputSchema = {
-      url: import_zod27.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
-      maxPages: import_zod27.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
+      url: import_zod27.z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
+      maxPages: import_zod27.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
+      rotateProxies: import_zod27.z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+      rotateProxyEvery: import_zod27.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
+      formats: import_zod27.z.array(import_zod27.z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
+    };
+    AuditSiteInputSchema = {
+      url: import_zod27.z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
+      maxPages: import_zod27.z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
+      rotateProxies: import_zod27.z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
+      rotateProxyEvery: import_zod27.z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
     };
     YoutubeHarvestInputSchema = {
       mode: import_zod27.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
@@ -22297,6 +23958,19 @@ var init_mcp_tool_schemas = __esm({
       })),
       durationMs: import_zod27.z.number().min(0)
     };
+    AuditSiteOutputSchema = {
+      url: import_zod27.z.string(),
+      pageCount: import_zod27.z.number().int().min(0),
+      durationMs: import_zod27.z.number().min(0),
+      bulkFolder: import_zod27.z.string().nullable(),
+      issues: import_zod27.z.record(import_zod27.z.string(), import_zod27.z.number()),
+      images: import_zod27.z.object({
+        unique: import_zod27.z.number().int().min(0),
+        totalBytes: import_zod27.z.number().min(0),
+        over100kb: import_zod27.z.number().int().min(0),
+        legacyFormat: import_zod27.z.number().int().min(0)
+      })
+    };
     MapsPlaceIntelOutputSchema = {
       name: import_zod27.z.string(),
       rating: NullableString,
@@ -23076,7 +24750,7 @@ function localPlanningToolAnnotations(title) {
 function listSavedReports() {
   try {
     const dir = outputBaseDir2();
-    return (0, import_node_fs7.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs7.statSync)((0, import_node_path10.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
+    return (0, import_node_fs8.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs8.statSync)((0, import_node_path11.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
   } catch {
     return [];
   }
@@ -23100,15 +24774,15 @@ function registerSavedReportResources(server) {
     },
     async (uri, variables) => {
       const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
-      const filename = (0, import_node_path10.basename)(decodeURIComponent(String(requested ?? "")));
+      const filename = (0, import_node_path11.basename)(decodeURIComponent(String(requested ?? "")));
       if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
-      const text = (0, import_node_fs7.readFileSync)((0, import_node_path10.join)(outputBaseDir2(), filename), "utf8");
+      const text = (0, import_node_fs8.readFileSync)((0, import_node_path11.join)(outputBaseDir2(), filename), "utf8");
       return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
     }
   );
 }
 function buildPaaExtractorMcpServer(executor, options = {}) {
-  const server = new import_mcp.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION });
+  const server = new import_mcp.McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
   registerPaaExtractorMcpTools(server, executor, options);
   return server;
 }
@@ -23140,18 +24814,25 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
   server.registerTool("map_site_urls", {
     title: "Site URL Map",
-    description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
+    description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Supports up to maxUrls=10000; maps over 500 URLs write the complete inventory to a local CSV file and return a summary plus the file path instead of the full list inline. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
     inputSchema: MapSiteUrlsInputSchema,
     outputSchema: MapSiteUrlsOutputSchema,
     annotations: liveWebToolAnnotations("Site URL Map")
   }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
   server.registerTool("extract_site", {
-    title: "Multi-Page Site Extract",
-    description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
+    title: "Multi-Page Site Content Crawl",
+    description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
     inputSchema: ExtractSiteInputSchema,
     outputSchema: ExtractSiteOutputSchema,
-    annotations: liveWebToolAnnotations("Multi-Page Site Extract")
+    annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
+  server.registerTool("audit_site", {
+    title: "Technical SEO Audit",
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    inputSchema: AuditSiteInputSchema,
+    outputSchema: AuditSiteOutputSchema,
+    annotations: liveWebToolAnnotations("Technical SEO Audit")
+  }, async (input) => formatAuditSite(await executor.auditSite(input), input));
   server.registerTool("youtube_harvest", {
     title: "YouTube Video Harvest",
     description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -23271,7 +24952,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     outputSchema: WorkflowArtifactReadOutputSchema,
     annotations: liveWebToolAnnotations("Read Workflow Artifact")
   }, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
-  server.registerTool("rank_tracker_blueprint", {
+  server.registerTool("rank_tracker_workflow", {
     title: "Rank Tracker Blueprint Builder",
     description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
     inputSchema: RankTrackerBlueprintInputSchema,
@@ -23292,15 +24973,16 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     }
   }, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
 }
-var import_mcp, import_node_fs7, import_node_path10;
+var import_mcp, import_node_fs8, import_node_path11;
 var init_paa_mcp_server = __esm({
   "src/mcp/paa-mcp-server.ts"() {
     "use strict";
     import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
-    import_node_fs7 = require("fs");
-    import_node_path10 = require("path");
+    import_node_fs8 = require("fs");
+    import_node_path11 = require("path");
     init_version();
     init_mcp_response_formatter();
+    init_server_instructions();
     init_mcp_tool_schemas();
     init_rank_tracker_blueprint();
     init_mcp_response_formatter();
@@ -23488,6 +25170,9 @@ var init_http_mcp_tool_executor = __esm({
       extractSite(input) {
         return this.call("/extract-site", input);
       }
+      auditSite(input) {
+        return this.call("/extract-site", input);
+      }
       youtubeHarvest(input) {
         return this.call("/youtube/harvest", input);
       }
@@ -23709,7 +25394,7 @@ async function migrateBrowserAgent() {
 }
 async function createSessionRow(input) {
   const db = getDb();
-  const id = `bas_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
+  const id = `bas_${(0, import_node_crypto6.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
   await db.execute({
     sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
           VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
@@ -23770,7 +25455,7 @@ async function recordAction(input) {
     sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
           VALUES (?, ?, ?, ?, ?, ?)`,
     args: [
-      `baa_${(0, import_node_crypto5.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
+      `baa_${(0, import_node_crypto6.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
       input.sessionId,
       input.type,
       input.params == null ? null : JSON.stringify(input.params),
@@ -23810,11 +25495,11 @@ async function listReplayRows(sessionId) {
   });
   return res.rows;
 }
-var import_node_crypto5, _ready;
+var import_node_crypto6, _ready;
 var init_browser_agent_db = __esm({
   "src/api/browser-agent-db.ts"() {
     "use strict";
-    import_node_crypto5 = require("crypto");
+    import_node_crypto6 = require("crypto");
     init_db();
     _ready = false;
   }
@@ -24419,7 +26104,7 @@ function buildFanoutResult(ctx) {
       interceptorReady: ready,
       rawBytes,
       unmappedKeys: parsed.unmappedKeys,
-      note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. Open a direct/no-proxy hosted browser session, navigate to chatgpt.com or claude.ai after opening, run a new prompt in that session, then call browser_capture_fanout again."
+      note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call query_fanout_workflow again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. Open a direct/no-proxy hosted browser session, navigate to chatgpt.com or claude.ai after opening, run a new prompt in that session, then call query_fanout_workflow again."
     };
   }
   return result;
@@ -24496,20 +26181,20 @@ var init_classify = __esm({
 
 // src/services/fanout/export.ts
 function outputBaseDir3() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path11.join)((0, import_node_os6.homedir)(), "Downloads", "mcp-scraper");
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path12.join)((0, import_node_os7.homedir)(), "Downloads", "mcp-scraper");
 }
 function safe(value) {
   return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
 }
 function writeTable(path6, rows, delimiter) {
-  (0, import_node_fs8.writeFileSync)(path6, import_papaparse3.default.unparse(rows.length ? rows : [{}], { delimiter }));
+  (0, import_node_fs9.writeFileSync)(path6, import_papaparse3.default.unparse(rows.length ? rows : [{}], { delimiter }));
 }
 function exportFanout(enriched) {
   const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
   const outputDir = outputBaseDir3();
-  const relativeDir = (0, import_node_path11.join)("fanout", `${stamp}-${safe(enriched.platform)}`);
-  const dir = (0, import_node_path11.join)(outputDir, relativeDir);
-  (0, import_node_fs8.mkdirSync)(dir, { recursive: true });
+  const relativeDir = (0, import_node_path12.join)("fanout", `${stamp}-${safe(enriched.platform)}`);
+  const dir = (0, import_node_path12.join)(outputDir, relativeDir);
+  (0, import_node_fs9.mkdirSync)(dir, { recursive: true });
   const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
   const citationRows = enriched.aggregates.citationOrder.map((c) => {
     const src = enriched.citedUrls.find((s) => s.url === c.url);
@@ -24522,25 +26207,25 @@ function exportFanout(enriched) {
   const relativePaths = {
     relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
     dir: relativeDir,
-    json: (0, import_node_path11.join)(relativeDir, "fanout.json"),
-    queriesCsv: (0, import_node_path11.join)(relativeDir, "queries.csv"),
-    queriesTsv: (0, import_node_path11.join)(relativeDir, "queries.tsv"),
-    citationsCsv: (0, import_node_path11.join)(relativeDir, "citations.csv"),
-    sourcesCsv: (0, import_node_path11.join)(relativeDir, "sources.csv"),
-    browsedOnlyCsv: (0, import_node_path11.join)(relativeDir, "browsed-only.csv"),
-    snippetsCsv: (0, import_node_path11.join)(relativeDir, "snippets.csv"),
-    domainsCsv: (0, import_node_path11.join)(relativeDir, "domains.csv"),
-    report: (0, import_node_path11.join)(relativeDir, "report.html")
+    json: (0, import_node_path12.join)(relativeDir, "fanout.json"),
+    queriesCsv: (0, import_node_path12.join)(relativeDir, "queries.csv"),
+    queriesTsv: (0, import_node_path12.join)(relativeDir, "queries.tsv"),
+    citationsCsv: (0, import_node_path12.join)(relativeDir, "citations.csv"),
+    sourcesCsv: (0, import_node_path12.join)(relativeDir, "sources.csv"),
+    browsedOnlyCsv: (0, import_node_path12.join)(relativeDir, "browsed-only.csv"),
+    snippetsCsv: (0, import_node_path12.join)(relativeDir, "snippets.csv"),
+    domainsCsv: (0, import_node_path12.join)(relativeDir, "domains.csv"),
+    report: (0, import_node_path12.join)(relativeDir, "report.html")
   };
-  (0, import_node_fs8.writeFileSync)((0, import_node_path11.join)(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.queriesCsv), queryRows, ",");
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.queriesTsv), queryRows, "	");
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.citationsCsv), citationRows, ",");
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.sourcesCsv), sourceRows2, ",");
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
-  writeTable((0, import_node_path11.join)(outputDir, relativePaths.domainsCsv), domainRows, ",");
-  (0, import_node_fs8.writeFileSync)((0, import_node_path11.join)(outputDir, relativePaths.report), renderReportHtml(enriched));
+  (0, import_node_fs9.writeFileSync)((0, import_node_path12.join)(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.queriesCsv), queryRows, ",");
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.queriesTsv), queryRows, "	");
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.citationsCsv), citationRows, ",");
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.sourcesCsv), sourceRows2, ",");
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
+  writeTable((0, import_node_path12.join)(outputDir, relativePaths.domainsCsv), domainRows, ",");
+  (0, import_node_fs9.writeFileSync)((0, import_node_path12.join)(outputDir, relativePaths.report), renderReportHtml(enriched));
   return relativePaths;
 }
 function esc(s) {
@@ -24618,13 +26303,13 @@ render();
 </script>
 </body></html>`;
 }
-var import_node_fs8, import_node_os6, import_node_path11, import_papaparse3;
+var import_node_fs9, import_node_os7, import_node_path12, import_papaparse3;
 var init_export = __esm({
   "src/services/fanout/export.ts"() {
     "use strict";
-    import_node_fs8 = require("fs");
-    import_node_os6 = require("os");
-    import_node_path11 = require("path");
+    import_node_fs9 = require("fs");
+    import_node_os7 = require("os");
+    import_node_path12 = require("path");
     import_papaparse3 = __toESM(require("papaparse"), 1);
   }
 });
@@ -24672,7 +26357,7 @@ async function runFanoutCapture(page, input) {
     }
     const adapter = adapterForHost(hostname);
     if (!adapter) {
-      throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
+      throw new Error(`query_fanout_workflow supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
     }
     const text = cap.bestAnswerText();
     const parsed = adapter.parse(text, input.prompt || "");
@@ -24710,7 +26395,7 @@ function keepHostedBrowserAlive(_browser) {
 function client() {
   const apiKey = browserServiceApiKey();
   if (!apiKey) throw new Error("Browser backend API key is required");
-  return new import_sdk6.default({ apiKey });
+  return new import_sdk7.default({ apiKey });
 }
 function isProfileConflict(err) {
   const message = err instanceof Error ? err.message : String(err);
@@ -24862,7 +26547,7 @@ async function pressKeys(runtimeSessionId, keys) {
   await k.browsers.computer.pressKey(runtimeSessionId, { keys });
 }
 async function goto(cdpWsUrl, url) {
-  const browser = await import_playwright4.chromium.connectOverCDP(cdpWsUrl);
+  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
     const context = browser.contexts()[0] ?? await browser.newContext();
     const page = context.pages()[0] ?? await context.newPage();
@@ -24873,7 +26558,7 @@ async function goto(cdpWsUrl, url) {
   }
 }
 async function captureFanout(cdpWsUrl, input) {
-  const browser = await import_playwright4.chromium.connectOverCDP(cdpWsUrl);
+  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
     const context = browser.contexts()[0] ?? await browser.newContext();
     const page = context.pages()[0] ?? await context.newPage();
@@ -24883,7 +26568,7 @@ async function captureFanout(cdpWsUrl, input) {
   }
 }
 async function readPage(cdpWsUrl) {
-  const browser = await import_playwright4.chromium.connectOverCDP(cdpWsUrl);
+  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
     const context = browser.contexts()[0] ?? await browser.newContext();
     const page = context.pages()[0] ?? await context.newPage();
@@ -24938,7 +26623,7 @@ async function readPage(cdpWsUrl) {
   }
 }
 async function locatePageTargets(cdpWsUrl, targets) {
-  const browser = await import_playwright4.chromium.connectOverCDP(cdpWsUrl);
+  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
     const context = browser.contexts()[0] ?? await browser.newContext();
     const page = context.pages()[0] ?? await context.newPage();
@@ -24950,16 +26635,16 @@ async function locatePageTargets(cdpWsUrl, targets) {
     }).catch(() => {
     });
     const data = await page.evaluate((rawTargets) => {
-      const normalize2 = (value) => value.replace(/\s+/g, " ").trim();
+      const normalize3 = (value) => value.replace(/\s+/g, " ").trim();
       const isVisibleRect = (r) => r.width >= 1 && r.height >= 1 && r.bottom >= 0 && r.right >= 0 && r.top <= window.innerHeight && r.left <= window.innerWidth;
       const roleOf = (el2) => el2.getAttribute("role") || el2.tagName.toLowerCase();
       const nameOf = (el2) => {
         const e = el2;
-        return normalize2(
+        return normalize3(
           e.getAttribute("aria-label") || e.placeholder || e.innerText || e.getAttribute("value") || e.getAttribute("title") || ""
         ).slice(0, 160);
       };
-      const textOf = (el2) => normalize2(el2.innerText || el2.textContent || "").slice(0, 500);
+      const textOf = (el2) => normalize3(el2.innerText || el2.textContent || "").slice(0, 500);
       const unionRects = (rects) => {
         const visible = rects.filter(isVisibleRect);
         if (!visible.length) return null;
@@ -24998,7 +26683,7 @@ async function locatePageTargets(cdpWsUrl, targets) {
         return nodes.map((el2) => ({ el: el2, rect: el2.getBoundingClientRect() })).filter((item) => isVisibleRect(item.rect));
       };
       const textMatches = (needle, match) => {
-        const wanted = normalize2(needle);
+        const wanted = normalize3(needle);
         const wantedLower = wanted.toLowerCase();
         if (!wantedLower) return [];
         const matches = [];
@@ -25006,7 +26691,7 @@ async function locatePageTargets(cdpWsUrl, targets) {
         let node = walker.nextNode();
         while (node) {
           const raw = node.textContent || "";
-          const normalizedRaw = normalize2(raw);
+          const normalizedRaw = normalize3(raw);
           const rawLower = raw.toLowerCase();
           const normalizedLower = normalizedRaw.toLowerCase();
           const ok = match === "exact" ? normalizedLower === wantedLower : normalizedLower.includes(wantedLower);
@@ -25102,12 +26787,12 @@ async function replayList(runtimeSessionId) {
     finishedAt: r.finished_at ?? null
   }));
 }
-var import_sdk6, import_playwright4, DEFAULT_TIMEOUT_SECONDS;
+var import_sdk7, import_playwright5, DEFAULT_TIMEOUT_SECONDS;
 var init_browser_agent_service = __esm({
   "src/services/browser-agent/browser-agent-service.ts"() {
     "use strict";
-    import_sdk6 = __toESM(require("@onkernel/sdk"), 1);
-    import_playwright4 = require("playwright");
+    import_sdk7 = __toESM(require("@onkernel/sdk"), 1);
+    import_playwright5 = require("playwright");
     init_browser_service_env();
     init_run_capture();
     DEFAULT_TIMEOUT_SECONDS = 600;
@@ -26108,14 +27793,14 @@ function getSessionSecret() {
 function safeEqualHex(a, b) {
   if (a.length !== b.length) return false;
   try {
-    return (0, import_node_crypto6.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
+    return (0, import_node_crypto7.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
   } catch {
     return false;
   }
 }
 function signSession(userId) {
   const payload = String(userId);
-  const sig = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
+  const sig = (0, import_node_crypto7.createHmac)("sha256", secret()).update(payload).digest("hex");
   return `${payload}.${sig}`;
 }
 function verifySession(token) {
@@ -26123,16 +27808,16 @@ function verifySession(token) {
   if (dot === -1) return null;
   const payload = token.slice(0, dot);
   const sig = token.slice(dot + 1);
-  const expected = (0, import_node_crypto6.createHmac)("sha256", secret()).update(payload).digest("hex");
+  const expected = (0, import_node_crypto7.createHmac)("sha256", secret()).update(payload).digest("hex");
   if (!safeEqualHex(sig, expected)) return null;
   const id = parseInt(payload);
   return isNaN(id) ? null : id;
 }
-var import_node_crypto6, isProduction, secret;
+var import_node_crypto7, isProduction, secret;
 var init_session = __esm({
   "src/api/session.ts"() {
     "use strict";
-    import_node_crypto6 = require("crypto");
+    import_node_crypto7 = require("crypto");
     isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
     secret = () => getSessionSecret();
   }
@@ -26379,6 +28064,9 @@ var init_server = __esm({
     import_hono14 = require("inngest/hono");
     init_client();
     init_site_audit();
+    init_site_extract();
+    init_site_extract_repository();
+    init_catalog_seed();
     init_site_audit_routes();
     init_youtube_routes();
     init_screenshot_routes();
@@ -26563,6 +28251,10 @@ var init_server = __esm({
       const taken = await countActiveUsers();
       return c.json({ taken, open: true });
     });
+    app.get("/catalog", (c) => {
+      c.header("Cache-Control", "public, max-age=60");
+      return c.json(CATALOG);
+    });
     app.get("/me", async (c) => {
       const token = (0, import_cookie.getCookie)(c, "session");
       if (!token) return c.json({ authenticated: false });
@@ -26797,7 +28489,7 @@ var init_server = __esm({
       const raw = await c.req.json().catch(() => ({}));
       const bodyResult = ExtractUrlBodySchema.safeParse(raw);
       if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
-      const { url, screenshot: screenshot2, screenshotDevice, extractBranding, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
+      const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
       if (!allowLocal) {
         const checked = await validatePublicHttpUrl(url, { field: "URL" });
         if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
@@ -26830,7 +28522,7 @@ var init_server = __esm({
         const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
         const [result, pageData] = await Promise.all([
           extractKpo({ url: canonicalUrl, kernelApiKey }),
-          screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch((err) => {
+          screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding2 }).catch((err) => {
             console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
             return null;
           }) : null
@@ -26874,7 +28566,7 @@ var init_server = __esm({
         debited = true;
         const result = await spiderSite({
           startUrl: parsed.href,
-          maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
+          maxUrls: Math.min(1e4, Math.max(1, body.maxUrls ?? 500)),
           concurrency: Math.min(20, Math.max(1, body.concurrency ?? 12)),
           kernelApiKey: body.browserFallback ?? body.kernelFallback ? browserServiceApiKey() : void 0
         });
@@ -26911,7 +28603,43 @@ var init_server = __esm({
       if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
       const parsed = checked.parsed;
       const user = c.get("user");
-      const siteHoldMc = MC_COSTS.page_scrape * 10;
+      if (body.background === true) {
+        const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
+        if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
+        const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
+        const heldMc = maxPages * MC_COSTS.page_scrape;
+        const { ok: holdOk, balance_mc: holdBal } = await debitMc(user.id, heldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
+        if (!holdOk) return c.json(insufficientBalanceResponse(holdBal, heldMc), 402);
+        const jobId = `ext_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
+        let jobCreated = false;
+        try {
+          await createExtractJob(jobId, user.id, parsed.href, {
+            maxPages,
+            concurrency: concurrencyLimitForUser(user),
+            urlsPerBrowser: body.rotateProxyEvery ?? 10,
+            formats: body.formats,
+            heldMc
+          });
+          jobCreated = true;
+          await inngest.send({ name: "mcp-scraper/extract.requested", data: { jobId } });
+        } catch (err) {
+          if (jobCreated) {
+            await settleExtractJob(jobId, user.id, heldMc, 0, "enqueue failed").catch(() => {
+            });
+          } else {
+            await creditMc(user.id, heldMc, LedgerOperation.EXTRACT_SITE_REFUND, "enqueue failed").catch(() => {
+            });
+          }
+          await failExtractJob(jobId, err instanceof Error ? err.message : "enqueue failed").catch(() => {
+          });
+          return c.json({ error: "Failed to enqueue background crawl" }, 503);
+        }
+        return c.json({ jobId, status: "pending", statusUrl: `/extract-site/status/${jobId}` });
+      }
+      const affordablePagesSync = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
+      if (affordablePagesSync < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
+      const siteMaxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 100)), affordablePagesSync);
+      const siteHoldMc = siteMaxPages * MC_COSTS.page_scrape;
       const gate = await acquireConcurrencyGate(user, "extract_site", {
         reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
         metadata: { url: parsed.href }
@@ -26922,10 +28650,17 @@ var init_server = __esm({
         const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
         if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
         debited = true;
+        const rotateProxies = body.rotateProxies === true;
+        const wantsBranding = body.formats?.includes("branding") === true;
         const result = await extractSite({
           startUrl: parsed.href,
-          maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
-          kernelApiKey: body.browserFallback ?? body.kernelFallback ? browserServiceApiKey() : void 0
+          maxPages: siteMaxPages,
+          kernelApiKey: rotateProxies || wantsBranding || body.browserFallback || body.kernelFallback ? browserServiceApiKey() : void 0,
+          formats: body.formats,
+          ...rotateProxies ? {
+            rotateProxyEvery: body.rotateProxyEvery ?? 30,
+            parallelism: concurrencyLimitForUser(user)
+          } : {}
         });
         const pageCount = result.pages?.length ?? 1;
         const actualSiteMc = pageCount * MC_COSTS.page_scrape;
@@ -26956,6 +28691,21 @@ var init_server = __esm({
         await releaseConcurrencyGate(gate.lockId);
       }
     });
+    app.get("/extract-site/status/:id", auth2, async (c) => {
+      const user = c.get("user");
+      const job = await getExtractJob(c.req.param("id"));
+      if (!job || job.userId !== user.id) return c.json({ error: "Job not found" }, 404);
+      return c.json({
+        jobId: job.id,
+        status: job.status,
+        startUrl: job.startUrl,
+        totalUrls: job.totalUrls,
+        doneUrls: job.doneUrls,
+        artifacts: job.artifacts ?? [],
+        error: job.error,
+        updatedAt: job.updatedAt
+      });
+    });
     app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
       try {
         const user = c.get("sessionUser");
@@ -27144,7 +28894,7 @@ var init_server = __esm({
       ]);
       return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
     });
-    app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono14.serve)({ client: inngest, functions: [siteAuditFn] }));
+    app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono14.serve)({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
     app.route("/api/internal/site-architecture-auditor", siteAuditApp);
     app.route("/youtube", youtubeApp);
     app.route("/screenshot", screenshotApp);
@@ -27266,10 +29016,10 @@ ${ATTRIBUTION_PIXEL_SCRIPT}
 });
 
 // bin/api-server.ts
-var import_node_fs9 = require("fs");
+var import_node_fs10 = require("fs");
 function loadDotEnv() {
   try {
-    for (const line of (0, import_node_fs9.readFileSync)(".env", "utf8").split("\n")) {
+    for (const line of (0, import_node_fs10.readFileSync)(".env", "utf8").split("\n")) {
       const eq = line.indexOf("=");
       if (eq < 1 || line.trimStart().startsWith("#")) continue;
       const k = line.slice(0, eq).trim();