mcp-scraper 0.3.13 → 0.3.15

package/dist/{server-LX65PB7D.js → server-UOQ2JIUQ.js}

@@ -16,7 +16,7 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-AZ5PKU4F.js";
+} from "./chunk-ZAUMSBV3.js";
 import {
   SiteAuditJobRowSchema,
   advanceWorkflowRunStep,
@@ -77,7 +77,7 @@ import {
   setPassword,
   setStripeCustomerId,
   verifyPassword
-} from "./chunk-DBQDG7EH.js";
+} from "./chunk-5H22TOXQ.js";
 import {
   BrowserDriver,
   MapsSelectors,
@@ -92,14 +92,17 @@ import {
   CaptureSerpSnapshotInputSchema,
   CaptureSerpSnapshotOutputSchema,
   HttpMcpToolExecutor,
+  buildLinkGraph,
   buildPaaExtractorMcpServer,
+  computeIssues,
   configureReportSaving,
   formatCaptureSerpPageSnapshots,
   formatCaptureSerpSnapshot,
   harvestTimeoutBudget,
   liveWebToolAnnotations,
-  outputBaseDir
-} from "./chunk-YGV2XZDT.js";
+  outputBaseDir,
+  renderIssueReport
+} from "./chunk-WQAIOY2D.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -135,7 +138,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-LFATOGDF.js";
-import "./chunk-BEZJ4G32.js";
+import "./chunk-2MX5WOII.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {
@@ -4195,6 +4198,10 @@ async function captureScreenshot(url, kernelApiKey, device = "desktop") {
   const result = await capturePageData(url, { kernelApiKey, device, screenshot: true, branding: false });
   return result.screenshot;
 }
+async function extractBranding(url, kernelApiKey, device = "desktop") {
+  const result = await capturePageData(url, { kernelApiKey, device, screenshot: false, branding: true });
+  return result.branding;
+}
 async function capturePageData(url, opts = {}) {
   const device = opts.device ?? "desktop";
   const wantShot = opts.screenshot ?? false;
@@ -4698,9 +4705,10 @@ function baseDomain(url) {
 }
 async function spiderSite(opts) {
   const startMs = Date.now();
-  const maxUrls = Math.min(opts.maxUrls ?? 500, 2e3);
+  const maxUrls = Math.min(opts.maxUrls ?? 500, 1e4);
   const concurrency = Math.min(opts.concurrency ?? 12, 20);
   const timeoutMs = opts.timeoutMs ?? 8e3;
+  const budgetMs = opts.budgetMs ?? 24e4;
   const startNorm = normalizeUrl(opts.startUrl, opts.startUrl) ?? opts.startUrl;
   const startDomain = baseDomain(opts.startUrl);
   const visited = /* @__PURE__ */ new Set();
@@ -4714,7 +4722,7 @@ async function spiderSite(opts) {
       queue.push({ url: loc, from: "sitemap" });
     }
   }
-  while (queue.length > 0 && results.length < maxUrls) {
+  while (queue.length > 0 && results.length < maxUrls && Date.now() - startMs < budgetMs) {
     const batch = queue.splice(0, concurrency);
     const settled = await Promise.all(
       batch.map(async ({ url, from }) => {
@@ -4774,12 +4782,193 @@ async function spiderSite(opts) {
 
 // src/api/site-extractor.ts
 import TurndownService2 from "turndown";
+import { createHash } from "crypto";
+
+// src/api/rotating-proxy-crawl.ts
+import Kernel3 from "@onkernel/sdk";
+import { chromium as chromium3 } from "playwright";
+var DEFAULT_CONCURRENCY = 1;
+var DEFAULT_URLS_PER_BROWSER = 10;
+var DEFAULT_HEADLESS = true;
+async function discoverSitemapUrls2(startUrl, maxUrls) {
+  const domain = new URL(startUrl).hostname.replace(/^www\./, "");
+  const urls = await discoverSitemapUrls(startUrl, domain);
+  return urls.slice(0, maxUrls);
+}
+function withTimeout(p, ms) {
+  return Promise.race([p, new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), ms))]);
+}
+async function createBrowserWithRetry(client2, headless) {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      return await client2.browsers.create({ stealth: true, headless, timeout_seconds: 120 });
+    } catch (err) {
+      if (attempt === 2) throw err;
+      await new Promise((r) => setTimeout(r, 5e3));
+    }
+  }
+  throw new Error("unreachable");
+}
+var PAGE_HARD_TIMEOUT_MS = 35e3;
+async function crawlPage(context, url) {
+  const startedAt = Date.now();
+  let page = null;
+  try {
+    page = await context.newPage();
+    const work = (async (p) => {
+      const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
+      const html = await p.content();
+      const headers = resp ? Object.fromEntries(Object.entries(await resp.headers()).map(([k, v]) => [k.toLowerCase(), v])) : void 0;
+      return { url, html, status: resp?.status() ?? 200, via: "browser", headers, responseTimeMs: Date.now() - startedAt, redirectUrl: headers?.["location"] ?? null };
+    })(page);
+    const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
+    return await Promise.race([work, timeout]);
+  } catch {
+    return { url, html: null, status: 0, via: "browser", responseTimeMs: Date.now() - startedAt };
+  } finally {
+    if (page) void page.close().catch(() => {
+    });
+  }
+}
+async function crawlWithRotation(urls, opts) {
+  const client2 = new Kernel3({ apiKey: opts.apiKey });
+  const concurrency = Math.max(1, opts.concurrency ?? DEFAULT_CONCURRENCY);
+  const urlsPerBrowser = Math.max(1, opts.urlsPerBrowser ?? DEFAULT_URLS_PER_BROWSER);
+  const headless = opts.headless ?? DEFAULT_HEADLESS;
+  const uniqueUrls = [...new Set(urls)];
+  const byUrl = /* @__PURE__ */ new Map();
+  const runPass = async (targets, phase) => {
+    if (targets.length === 0) return;
+    let next = 0;
+    let done = 0;
+    const workers = Math.min(concurrency, targets.length);
+    const worker = async () => {
+      let kb = null;
+      let browser = null;
+      let inBrowser = 0;
+      const retire = () => {
+        const b = browser, k = kb;
+        browser = null;
+        kb = null;
+        if (b) void b.close().catch(() => {
+        });
+        if (k) void client2.browsers.deleteByID(k.session_id).catch(() => {
+        });
+      };
+      try {
+        while (true) {
+          const i = next++;
+          if (i >= targets.length) break;
+          const url = targets[i];
+          let r;
+          try {
+            if (!browser || inBrowser >= urlsPerBrowser) {
+              retire();
+              kb = await createBrowserWithRetry(client2, headless);
+              browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
+              inBrowser = 0;
+            }
+            const context = browser.contexts()[0] ?? await browser.newContext();
+            r = await crawlPage(context, url);
+            inBrowser++;
+          } catch {
+            r = { url, html: null, status: 0, via: "browser" };
+            retire();
+          }
+          byUrl.set(url, r);
+          done++;
+          opts.onPage?.({ url, status: r.status, ok: !!(r.html && r.status >= 200 && r.status < 300), ms: r.responseTimeMs ?? 0, done, total: targets.length, phase });
+        }
+      } finally {
+        retire();
+      }
+    };
+    await Promise.all(Array.from({ length: workers }, () => worker()));
+  };
+  await runPass(uniqueUrls, "main");
+  const failed = uniqueUrls.filter((u) => {
+    const r = byUrl.get(u);
+    return !r || !r.html || r.status < 200 || r.status >= 300;
+  });
+  if (failed.length > 0) await runPass(failed, "retry");
+  return urls.map((u) => byUrl.get(u) ?? { url: u, html: null, status: 0, via: "browser" });
+}
+
+// src/api/site-extractor.ts
+var PIXEL_WIDTHS = { i: 4, l: 4, j: 4, ".": 4, ",": 4, "'": 4, t: 6, f: 6, r: 6, " ": 4, m: 14, w: 13, W: 16, M: 16 };
+function estimatePixels(s) {
+  let px = 0;
+  for (const ch of s) px += PIXEL_WIDTHS[ch] ?? (ch === ch.toUpperCase() && ch !== ch.toLowerCase() ? 10 : 8);
+  return px;
+}
+function emptyPageData(url, status, via) {
+  return {
+    url,
+    status,
+    via,
+    contentType: null,
+    responseTimeMs: null,
+    sizeBytes: null,
+    redirectUrl: null,
+    lastModified: null,
+    xRobotsTag: null,
+    title: null,
+    titleLength: null,
+    titlePixels: null,
+    metaDescription: null,
+    metaDescLength: null,
+    metaKeywords: null,
+    h1: null,
+    h1_2: null,
+    h2Count: 0,
+    headings: [],
+    wordCount: 0,
+    textRatio: 0,
+    contentHash: "",
+    metaRobots: null,
+    relNext: null,
+    relPrev: null,
+    hreflang: [],
+    og: null,
+    twitter: null,
+    ampHref: null,
+    imageCount: 0,
+    imagesMissingAlt: 0,
+    imageLinks: [],
+    indexable: status === 200,
+    indexabilityReason: status === 200 ? null : "non-200",
+    schemaTypes: [],
+    canonicalUrl: null,
+    internalLinks: 0,
+    externalLinks: 0,
+    outlinks: [],
+    bodyMarkdown: "",
+    schema: []
+  };
+}
+function makeSeedSpider(startUrl, urls) {
+  return {
+    startUrl,
+    urls: urls.map((url) => ({ url, status: 200, linkedFrom: "seed", via: "fetch" })),
+    totalFound: urls.length,
+    durationMs: 0,
+    truncated: false,
+    browserRetries: 0
+  };
+}
 var UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
 var EXTRACT_CONCURRENCY = 6;
 var MAX_PAGE_MARKDOWN = 4e4;
 var turndown = new TurndownService2({ headingStyle: "atx", bulletListMarker: "-", codeBlockStyle: "fenced" });
-function parsePageData(url, html, status, via) {
+function parsePageData(url, html, status, via, resp) {
   const origin = new URL(url).origin;
+  const headers = resp?.headers ?? {};
+  const contentType = headers["content-type"] ?? null;
+  const lastModified = headers["last-modified"] ?? null;
+  const xRobotsTag = headers["x-robots-tag"] ?? null;
+  const responseTimeMs = resp?.responseTimeMs ?? null;
+  const redirectUrl = resp?.redirectUrl ?? headers["location"] ?? null;
+  const sizeBytes = headers["content-length"] ? Number(headers["content-length"]) : Buffer.byteLength(html);
   const title = html.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.trim() ?? null;
   const metaDescription = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i)?.[1]?.trim() ?? html.match(/<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i)?.[1]?.trim() ?? null;
   const canonicalUrl = html.match(/<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? html.match(/<link[^>]+href=["']([^"']*)["'][^>]+rel=["']canonical["']/i)?.[1]?.trim() ?? null;
@@ -4816,38 +5005,170 @@ function parsePageData(url, html, status, via) {
   } catch {
     bodyMarkdown = bodyText.slice(0, MAX_PAGE_MARKDOWN);
   }
-  let internalLinks = 0;
-  let externalLinks = 0;
-  for (const m of html.matchAll(/href\s*=\s*["']([^"'\s>]+)/gi)) {
+  const outlinks = [];
+  for (const m of html.matchAll(/<a\b([^>]*?)href\s*=\s*["']([^"'#][^"']*)["']([^>]*)>([\s\S]*?)<\/a>/gi)) {
     try {
-      const abs = new URL(m[1], url);
-      if (abs.origin === origin) internalLinks++;
-      else externalLinks++;
+      const abs = new URL(m[2], url);
+      if (abs.protocol !== "http:" && abs.protocol !== "https:") continue;
+      const attrs = `${m[1]} ${m[3]}`;
+      const rel = attrs.match(/\brel\s*=\s*["']([^"']*)["']/i)?.[1]?.toLowerCase() ?? null;
+      const anchor = m[4].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 160);
+      outlinks.push({ href: abs.href, anchor, rel, internal: abs.origin === origin });
     } catch {
     }
   }
-  return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
+  const internalLinks = outlinks.filter((l) => l.internal).length;
+  const externalLinks = outlinks.length - internalLinks;
+  const titleLength = title?.length ?? null;
+  const titlePixels = title ? estimatePixels(title) : null;
+  const metaDescLength = metaDescription?.length ?? null;
+  const metaKeywords = html.match(/<meta[^>]+name=["']keywords["'][^>]+content=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  const h1List = headings.filter((h) => h.level === 1);
+  const h1_2 = h1List[1]?.text ?? null;
+  const h2Count = headings.filter((h) => h.level === 2).length;
+  const textRatio = Number((bodyText.length / Math.max(1, html.length)).toFixed(3));
+  const contentHash = createHash("sha1").update(bodyText).digest("hex");
+  const metaRobots = html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']*)["']/i)?.[1]?.trim().toLowerCase() ?? null;
+  const relNext = html.match(/<link[^>]+rel=["']next["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  const relPrev = html.match(/<link[^>]+rel=["']prev["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  const hreflang = [];
+  for (const m of html.matchAll(/<link[^>]+rel=["']alternate["'][^>]*>/gi)) {
+    const lang = m[0].match(/hreflang=["']([^"']*)["']/i)?.[1];
+    const href = m[0].match(/href=["']([^"']*)["']/i)?.[1];
+    if (lang && href) hreflang.push({ lang, href });
+  }
+  const ogTag = (p) => html.match(new RegExp(`<meta[^>]+property=["']og:${p}["'][^>]+content=["']([^"']*)["']`, "i"))?.[1]?.trim();
+  const og = ogTag("title") || ogTag("description") || ogTag("image") || ogTag("type") ? { title: ogTag("title"), description: ogTag("description"), image: ogTag("image"), type: ogTag("type") } : null;
+  const twTag = (p) => html.match(new RegExp(`<meta[^>]+name=["']twitter:${p}["'][^>]+content=["']([^"']*)["']`, "i"))?.[1]?.trim();
+  const twitter = twTag("card") || twTag("title") || twTag("description") ? { card: twTag("card"), title: twTag("title"), description: twTag("description") } : null;
+  const ampHref = html.match(/<link[^>]+rel=["']amphtml["'][^>]+href=["']([^"']*)["']/i)?.[1]?.trim() ?? null;
+  let imageCount = 0;
+  let imagesMissingAlt = 0;
+  const imageLinkSet = /* @__PURE__ */ new Set();
+  for (const m of html.matchAll(/<img\b([^>]*)>/gi)) {
+    imageCount++;
+    const alt = m[1].match(/\balt\s*=\s*["']([^"']*)["']/i)?.[1];
+    if (!alt || !alt.trim()) imagesMissingAlt++;
+    const src = m[1].match(/\bsrc\s*=\s*["']([^"']+)["']/i)?.[1];
+    if (src) {
+      try {
+        const abs = new URL(src, url);
+        if (abs.protocol === "http:" || abs.protocol === "https:") imageLinkSet.add(abs.href);
+      } catch {
+      }
+    }
+  }
+  if (og?.image) {
+    try {
+      imageLinkSet.add(new URL(og.image, url).href);
+    } catch {
+    }
+  }
+  const imageLinks = [...imageLinkSet].slice(0, 300);
+  const canonNorm = (u) => {
+    try {
+      const x = new URL(u, url);
+      return (x.origin + x.pathname).replace(/\/+$/, "");
+    } catch {
+      return u.replace(/\/+$/, "");
+    }
+  };
+  const canonicalSelf = canonicalUrl ? canonNorm(canonicalUrl) === canonNorm(url) : true;
+  let indexable = true;
+  let indexabilityReason = null;
+  if (status !== 200) {
+    indexable = false;
+    indexabilityReason = "non-200";
+  } else if (metaRobots?.includes("noindex")) {
+    indexable = false;
+    indexabilityReason = "noindex";
+  } else if (xRobotsTag?.toLowerCase().includes("noindex")) {
+    indexable = false;
+    indexabilityReason = "x-robots-noindex";
+  } else if (canonicalUrl && !canonicalSelf) {
+    indexable = false;
+    indexabilityReason = "canonicalised";
+  }
+  return {
+    url,
+    status,
+    via,
+    contentType,
+    responseTimeMs,
+    sizeBytes,
+    redirectUrl,
+    lastModified,
+    xRobotsTag,
+    title,
+    titleLength,
+    titlePixels,
+    metaDescription,
+    metaDescLength,
+    metaKeywords,
+    h1,
+    h1_2,
+    h2Count,
+    headings,
+    wordCount,
+    textRatio,
+    contentHash,
+    metaRobots,
+    relNext,
+    relPrev,
+    hreflang,
+    og,
+    twitter,
+    ampHref,
+    imageCount,
+    imagesMissingAlt,
+    imageLinks,
+    indexable,
+    indexabilityReason,
+    schemaTypes,
+    canonicalUrl,
+    internalLinks,
+    externalLinks,
+    outlinks,
+    bodyMarkdown,
+    schema
+  };
+}
+async function extractPagesRotating(urls, opts) {
+  const fetched = await crawlWithRotation(urls, { apiKey: opts.kernelApiKey, concurrency: opts.concurrency, urlsPerBrowser: opts.urlsPerBrowser });
+  return fetched.map((r) => r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser"));
+}
+async function discoverUrls(startUrl, maxPages, kernelApiKey) {
+  const sitemapUrls = await discoverSitemapUrls2(startUrl, maxPages);
+  if (sitemapUrls.length > 0) return sitemapUrls;
+  const spider = await spiderSite({ startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey });
+  return spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
 }
 async function fetchAndParse(url, kernelApiKey) {
+  const startedAt = Date.now();
   try {
     const res = await fetch(url, {
       headers: { "User-Agent": UA2, "Accept": "text/html,application/xhtml+xml" },
       signal: AbortSignal.timeout(1e4),
       redirect: "manual"
     });
+    const meta = {
+      headers: Object.fromEntries([...res.headers.entries()].map(([k, v]) => [k.toLowerCase(), v])),
+      responseTimeMs: Date.now() - startedAt,
+      redirectUrl: res.headers.get("location")
+    };
     if (res.ok) {
       const ct = res.headers.get("content-type") ?? "";
       if (ct.includes("html")) {
         const html = await res.text();
-        return parsePageData(url, html, res.status, "fetch");
+        return parsePageData(url, html, res.status, "fetch", meta);
       }
-      return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
+      return { ...emptyPageData(url, res.status, "fetch"), contentType: ct || null, responseTimeMs: meta.responseTimeMs ?? null };
     }
     if ((res.status === 403 || res.status === 429) && kernelApiKey) {
       const html = await fetchWithKernel(url);
       return parsePageData(url, html, 200, "browser");
     }
-    return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
+    return { ...emptyPageData(url, res.status, "fetch"), responseTimeMs: meta.responseTimeMs ?? null, redirectUrl: meta.redirectUrl ?? null };
   } catch {
     if (kernelApiKey) {
       try {
@@ -4856,7 +5177,7 @@ async function fetchAndParse(url, kernelApiKey) {
       } catch {
       }
     }
-    return { url, status: null, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
+    return emptyPageData(url, null, "fetch");
   }
 }
 async function runWithConcurrency(items, concurrency, fn) {
@@ -4871,30 +5192,82 @@ async function runWithConcurrency(items, concurrency, fn) {
 }
 async function extractSite(opts) {
   const startMs = Date.now();
-  const maxPages = Math.min(opts.maxPages ?? 100, 200);
+  const maxPages = Math.min(opts.maxPages ?? 100, 1e4);
   const concurrency = Math.min(opts.concurrency ?? EXTRACT_CONCURRENCY, 10);
-  const spider = await spiderSite({
-    startUrl: opts.startUrl,
-    maxUrls: maxPages,
-    concurrency: 12,
-    kernelApiKey: opts.kernelApiKey
-  });
-  const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+  const rotating = Boolean(opts.rotateProxyEvery && opts.kernelApiKey);
+  const seeded = opts.seedUrls && opts.seedUrls.length > 0;
   const pages = [];
-  let browserRetries = spider.browserRetries;
-  await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
-    const data = await fetchAndParse(url, opts.kernelApiKey);
-    if (data.via === "browser") browserRetries++;
-    pages.push(data);
-  });
+  let browserRetries = 0;
+  let truncated = false;
+  let spider;
+  if (rotating) {
+    const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
+    const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
+    const followLinks = !seeded;
+    const seen = /* @__PURE__ */ new Set();
+    const frontier = [];
+    const enqueue = (u) => {
+      const n = normalizeUrl(u, opts.startUrl);
+      if (n && !seen.has(n)) {
+        seen.add(n);
+        frontier.push(n);
+      }
+    };
+    if (seeded) {
+      for (const u of opts.seedUrls) enqueue(u);
+    } else {
+      enqueue(opts.startUrl);
+      for (const u of await discoverSitemapUrls2(opts.startUrl, maxPages)) enqueue(u);
+    }
+    const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
+    let done = 0;
+    while (frontier.length > 0 && pages.length < maxPages) {
+      const capacity = maxPages - pages.length;
+      const wave = frontier.splice(0, Math.min(capacity, waveCap));
+      const fetched = await crawlWithRotation(wave, {
+        apiKey: opts.kernelApiKey,
+        concurrency: browserConcurrency,
+        urlsPerBrowser,
+        onPage: (p) => {
+          done++;
+          opts.onProgress?.({ done, discovered: seen.size, frontier: frontier.length, lastOk: p.ok ? 1 : 0, lastFail: p.ok ? 0 : 1, phase: p.phase });
+        }
+      });
+      for (const r of fetched) {
+        const page = r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser");
+        pages.push(page);
+        browserRetries++;
+        if (followLinks && r.html) {
+          for (const l of page.outlinks) if (l.internal) enqueue(l.href);
+        }
+      }
+    }
+    truncated = frontier.length > 0;
+    spider = makeSeedSpider(opts.startUrl, [...seen]);
+  } else {
+    spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
+    const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+    browserRetries = spider.browserRetries;
+    await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
+      const data = await fetchAndParse(url, opts.kernelApiKey);
+      if (data.via === "browser") browserRetries++;
+      pages.push(data);
+    });
+    truncated = spider.truncated || pages.length >= maxPages;
+  }
+  let branding = null;
+  if (opts.formats?.includes("branding") && opts.kernelApiKey) {
+    branding = await extractBranding(opts.startUrl, opts.kernelApiKey).catch(() => null);
+  }
   return {
     startUrl: opts.startUrl,
     pages,
     totalPages: pages.length,
     spider,
     durationMs: Date.now() - startMs,
-    truncated: spider.truncated || pages.length >= maxPages,
-    browserRetries
+    truncated,
+    browserRetries,
+    branding
   };
 }
 
@@ -7059,6 +7432,580 @@ var siteAuditFn = inngest.createFunction(
   }
 );
 
+// src/api/blob-store.ts
+import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
+import { homedir as homedir2 } from "os";
+import { join as join4, dirname as dirname2 } from "path";
+function byteLength(data) {
+  return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
+}
+var LocalBlobStore = class {
+  constructor(baseDir) {
+    this.baseDir = baseDir;
+  }
+  baseDir;
+  kind = "local";
+  async put(key, data, contentType = "application/octet-stream") {
+    const path5 = join4(this.baseDir, "blobs", key);
+    mkdirSync2(dirname2(path5), { recursive: true });
+    writeFileSync(path5, data);
+    return { key, url: `file://${path5}`, bytes: byteLength(data), contentType };
+  }
+};
+function localBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
+}
+var cached = null;
+function getBlobStore() {
+  if (cached) return cached;
+  if (process.env.BLOB_READ_WRITE_TOKEN) {
+    cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
+  } else {
+    cached = new LocalBlobStore(localBaseDir());
+  }
+  return cached;
+}
+var VercelBlobStore = class {
+  constructor(token) {
+    this.token = token;
+  }
+  token;
+  kind = "vercel-blob";
+  async put(key, data, contentType = "application/octet-stream") {
+    const { put } = await import("@vercel/blob");
+    const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
+    const result = await put(key, body, {
+      access: "public",
+      token: this.token,
+      contentType,
+      addRandomSuffix: true
+    });
+    return { key, url: result.url, bytes: byteLength(data), contentType };
+  }
+};
+
+// src/api/site-extract-repository.ts
+function rowToJob(r) {
+  return {
+    id: String(r.id),
+    userId: r.user_id != null ? Number(r.user_id) : null,
+    status: r.status != null ? String(r.status) : "pending",
+    startUrl: String(r.start_url ?? ""),
+    options: r.options ? JSON.parse(String(r.options)) : {},
+    totalUrls: Number(r.total_urls ?? 0),
+    doneUrls: Number(r.done_urls ?? 0),
+    artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,
+    error: r.error != null ? String(r.error) : null,
+    billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,
+    createdAt: String(r.created_at ?? ""),
+    updatedAt: String(r.updated_at ?? "")
+  };
+}
+async function createExtractJob(jobId, userId, startUrl, options) {
+  const db = getDb();
+  await db.execute({
+    sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
+          VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,
+    args: [jobId, userId, startUrl, JSON.stringify(options)]
+  });
+}
+async function getExtractJob(jobId) {
+  const db = getDb();
+  const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] });
+  return res.rows[0] ? rowToJob(res.rows[0]) : null;
+}
+async function setExtractJobTotal(jobId, totalUrls) {
+  const db = getDb();
+  await db.execute({
+    sql: `UPDATE site_extract_jobs SET status = 'running', total_urls = ?, updated_at = datetime('now') WHERE id = ?`,
+    args: [totalUrls, jobId]
+  });
+}
+async function saveExtractPages(jobId, pages) {
+  if (pages.length === 0) return;
+  const db = getDb();
+  await db.batch(
+    pages.map((p) => ({
+      sql: `INSERT OR REPLACE INTO site_extract_pages (job_id, url, page) VALUES (?, ?, ?)`,
+      args: [jobId, p.url, JSON.stringify({ ...p, bodyMarkdown: "", schema: [] })]
+    }))
+  );
+  await db.execute({
+    sql: `UPDATE site_extract_jobs
+            SET done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?), updated_at = datetime('now')
+          WHERE id = ?`,
+    args: [jobId, jobId]
+  });
+}
+async function getExtractedPages(jobId) {
+  const db = getDb();
+  const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
+  return res.rows.map((r) => JSON.parse(String(r.page)));
+}
+async function countSuccessfulPages(jobId) {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT COUNT(*) AS n FROM site_extract_pages
+            WHERE job_id = ? AND json_extract(page, '$.status') = 200 AND json_extract(page, '$.wordCount') > 0`,
+    args: [jobId]
+  });
+  return Number(res.rows[0]?.n ?? 0);
+}
+async function completeExtractJob(jobId, artifacts) {
+  const db = getDb();
+  await db.execute({
+    sql: `UPDATE site_extract_jobs SET status = 'complete', artifacts = ?, updated_at = datetime('now') WHERE id = ?`,
+    args: [JSON.stringify(artifacts ?? []), jobId]
+  });
+}
+async function failExtractJob(jobId, error) {
+  const db = getDb();
+  await db.execute({
+    sql: `UPDATE site_extract_jobs SET status = 'failed', error = ?, updated_at = datetime('now') WHERE id = ?`,
+    args: [sanitizeVendorName(error).slice(0, 2e3), jobId]
+  });
+}
+async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
+  const db = getDb();
+  const job = await getExtractJob(jobId);
+  if (!job || job.billedMc != null) return;
+  if (refundMc <= 0) {
+    await db.execute({ sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] });
+    return;
+  }
+  await db.batch([
+    { sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [refundMc, userId] },
+    { sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)", args: [userId, refundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference] },
+    { sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] }
+  ]);
+}
+
+// src/inngest/functions/site-extract.ts
+var BATCH = 30;
+function chunk(items, size) {
+  const out = [];
+  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
+  return out;
+}
+function buildArtifacts(job, pages, branding) {
+  const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
+  const issues = computeIssues(pages, metrics);
+  const reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  const pageRows = pages.map((p) => {
+    const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
+    const m = metrics.get(p.url);
+    return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
+  });
+  const jsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
+  const files = [
+    { name: "report.md", body: reportMd, type: "text/markdown" },
+    { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
+    { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
+    { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
+    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
+  ];
+  if (branding) files.push({ name: "branding.json", body: JSON.stringify(branding, null, 2), type: "application/json" });
+  return files;
+}
+var siteExtractFn = inngest.createFunction(
+  {
+    id: "site-extract",
+    retries: 2,
+    triggers: [{ event: "mcp-scraper/extract.requested" }],
+    onFailure: async ({ event }) => {
+      const jobId = event?.data?.event?.data?.jobId;
+      if (!jobId) return;
+      const current = await getExtractJob(jobId);
+      if (current && current.userId != null && current.billedMc == null) {
+        const heldMc = Number(current.options.heldMc ?? 0);
+        await settleExtractJob(jobId, current.userId, heldMc, 0, "crawl failed refund").catch(() => {
+        });
+      }
+      await failExtractJob(jobId, String(event?.data?.error?.message ?? "crawl failed")).catch(() => {
+      });
+    }
+  },
+  async ({ event, step }) => {
+    const jobId = event.data.jobId;
+    const job = await step.run("load-job", () => getExtractJob(jobId));
+    if (!job) return { jobId, status: "missing" };
+    const key = browserServiceApiKey();
+    if (!key) throw new Error("browser service key not configured");
+    const maxPages = Number(job.options.maxPages ?? 1e4);
+    const concurrency = Number(job.options.concurrency ?? 1);
+    const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
+    const urls = await step.run("discover", async () => {
+      const discovered = await discoverUrls(job.startUrl, maxPages, key);
+      await setExtractJobTotal(jobId, discovered.length);
+      return discovered;
+    });
+    const batches = chunk(urls, BATCH);
+    for (let i = 0; i < batches.length; i++) {
+      await step.run(`crawl-batch-${i}`, async () => {
+        const pages = await extractPagesRotating(batches[i], { kernelApiKey: key, concurrency, urlsPerBrowser });
+        await saveExtractPages(jobId, pages);
+        return pages.length;
+      });
+    }
+    const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
+    const artifacts = await step.run("finalize", async () => {
+      const pages = await getExtractedPages(jobId);
+      const store = getBlobStore();
+      const files = buildArtifacts(job, pages, branding);
+      const stored = [];
+      for (const f of files) stored.push(await store.put(`extract/${jobId}/${f.name}`, f.body, f.type));
+      await completeExtractJob(jobId, stored);
+      return stored;
+    });
+    await step.run("settle", async () => {
+      const current = await getExtractJob(jobId);
+      if (!current || current.billedMc != null || current.userId == null) return;
+      const heldMc = Number(current.options.heldMc ?? 0);
+      const successful = await countSuccessfulPages(jobId);
+      const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
+      await settleExtractJob(jobId, current.userId, heldMc - usedMc, usedMc, new URL(job.startUrl).hostname);
+    });
+    return { jobId, status: "complete", artifacts };
+  }
+);
+
+// src/api/catalog-seed.ts
+var URL_PARAM = { key: "url", label: "URL", type: "url", required: true, placeholder: "https://example.com" };
+var FORMATS_PARAM = {
+  key: "formats",
+  label: "Formats",
+  type: "multi_enum",
+  default: ["markdown", "links"],
+  enumValues: [
+    { value: "markdown", label: "Markdown" },
+    { value: "links", label: "Links" },
+    { value: "json", label: "JSON / schema" },
+    { value: "images", label: "Image links" },
+    { value: "branding", label: "Branding" }
+  ]
+};
+var CATALOG = {
+  groups: [
+    {
+      key: "youtube",
+      label: "YouTube",
+      icon: "youtube",
+      tools: [
+        {
+          key: "youtube_harvest",
+          label: "Search",
+          kind: "simple",
+          endpoint: "/youtube/harvest",
+          mcpName: "youtube_harvest",
+          tagline: "Find or list videos by topic or channel",
+          params: [
+            { key: "mode", label: "Mode", type: "enum", required: true, default: "search", enumValues: [{ value: "search", label: "Search" }, { value: "channel", label: "Channel" }] },
+            { key: "query", label: "Query", type: "string", placeholder: "best local SEO 2026" },
+            { key: "channelHandle", label: "Channel handle", type: "string", placeholder: "@mkbhd" },
+            { key: "maxVideos", label: "Max videos", type: "number", min: 1, max: 500, default: 50, advanced: true }
+          ],
+          examples: ["Find YouTube videos about local SEO", "Harvest videos from @mkbhd"]
+        },
+        {
+          key: "youtube_transcribe",
+          label: "Transcribe",
+          kind: "simple",
+          endpoint: "/youtube/transcribe",
+          mcpName: "youtube_transcribe",
+          tagline: "Transcribe one video to text",
+          params: [{ key: "url", label: "Video URL", type: "url", required: true, placeholder: "https://www.youtube.com/watch?v=\u2026" }],
+          examples: ["Transcribe this YouTube URL"]
+        }
+      ]
+    },
+    {
+      key: "facebook",
+      label: "Facebook",
+      icon: "facebook",
+      tools: [
+        {
+          key: "facebook_ad_search",
+          label: "Ad Search",
+          kind: "simple",
+          endpoint: "/facebook/search",
+          mcpName: "facebook_ad_search",
+          tagline: "Find ads in the Ad Library",
+          params: [
+            { key: "query", label: "Advertiser / query", type: "string", required: true },
+            { key: "country", label: "Country", type: "string", default: "US" },
+            { key: "maxResults", label: "Max results", type: "number", min: 1, max: 20, default: 10, advanced: true }
+          ],
+          examples: ["Find Facebook ads for this brand"]
+        },
+        {
+          key: "facebook_video_transcribe",
+          label: "Video Transcribe",
+          kind: "simple",
+          endpoint: "/facebook/video-transcribe",
+          mcpName: "facebook_video_transcribe",
+          tagline: "Transcribe a Facebook video/reel",
+          params: [{ ...URL_PARAM, label: "Video URL", placeholder: "https://www.facebook.com/share/v/\u2026" }]
+        },
+        {
+          key: "facebook_ad_transcribe",
+          label: "Ad Transcribe",
+          kind: "simple",
+          endpoint: "/facebook/transcribe",
+          mcpName: "facebook_ad_transcribe",
+          tagline: "Transcribe an ad creative",
+          params: [{ ...URL_PARAM, label: "Ad video URL" }]
+        },
+        {
+          key: "facebook_page_intel",
+          label: "Page Intel",
+          kind: "simple",
+          endpoint: "/facebook/page-intel",
+          mcpName: "facebook_page_intel",
+          tagline: "Advertiser page profile + ads",
+          params: [{ key: "libraryId", label: "Page / Library ID or URL", type: "string", required: true }]
+        }
+      ]
+    },
+    {
+      key: "instagram",
+      label: "Instagram",
+      icon: "instagram",
+      tools: [
+        {
+          key: "instagram_profile_content",
+          label: "Profile",
+          kind: "simple",
+          endpoint: "/instagram/profile-content",
+          mcpName: "instagram_profile_content",
+          tagline: "Inventory a profile grid",
+          params: [
+            { key: "url", label: "Profile URL or handle", type: "string", required: true, placeholder: "https://instagram.com/nasa" },
+            { key: "maxItems", label: "Max items", type: "number", min: 1, max: 2e3, default: 50, advanced: true },
+            { key: "maxScrolls", label: "Max scrolls", type: "number", min: 0, max: 250, default: 10, advanced: true }
+          ]
+        },
+        {
+          key: "instagram_media_download",
+          label: "Post / Reel",
+          kind: "simple",
+          endpoint: "/instagram/media-download",
+          mcpName: "instagram_media_download",
+          tagline: "Download a post or reel (+ transcript)",
+          params: [
+            { ...URL_PARAM, label: "Post / reel URL", placeholder: "https://www.instagram.com/reel/\u2026" },
+            { key: "includeTranscript", label: "Include transcript", type: "boolean", default: true }
+          ]
+        }
+      ]
+    },
+    {
+      key: "reddit",
+      label: "Reddit",
+      icon: "message-circle",
+      tools: [
+        {
+          key: "reddit_browser",
+          label: "Research (Browser)",
+          kind: "simple",
+          endpoint: "/agent",
+          mcpName: "browser_open",
+          tagline: "No dedicated tool yet \u2014 driven by the Browser agent",
+          params: [{ key: "query", label: "What to research", type: "string", required: true, placeholder: "ICP pain points in [niche]" }]
+        }
+      ]
+    },
+    {
+      key: "google_search",
+      label: "Google Search",
+      icon: "search",
+      tools: [
+        {
+          key: "search_serp",
+          label: "SERP",
+          kind: "simple",
+          endpoint: "/harvest/sync",
+          mcpName: "search_serp",
+          tagline: "Organic results only",
+          params: [
+            { key: "query", label: "Query", type: "string", required: true },
+            { key: "location", label: "Location", type: "string", placeholder: "Denver, CO" }
+          ],
+          examples: ["Top restaurants in San Francisco"]
+        },
+        {
+          key: "harvest_paa",
+          label: "PAA + SERP Detail",
+          kind: "simple",
+          endpoint: "/harvest/sync",
+          mcpName: "harvest_paa",
+          tagline: "People-Also-Ask, AI Overview, full SERP features",
+          params: [
+            { key: "query", label: "Query", type: "string", required: true },
+            { key: "maxQuestions", label: "Max questions", type: "number", min: 1, max: 200, default: 30, advanced: true },
+            { key: "location", label: "Location", type: "string" }
+          ]
+        }
+      ]
+    },
+    {
+      key: "google_maps",
+      label: "Google Maps",
+      icon: "map-pin",
+      tools: [
+        {
+          key: "maps_search",
+          label: "Search",
+          kind: "simple",
+          endpoint: "/maps/search",
+          mcpName: "maps_search",
+          tagline: "Find local businesses",
+          params: [
+            { key: "query", label: "Query", type: "string", required: true, placeholder: "roofers" },
+            { key: "location", label: "Location", type: "string", required: true, placeholder: "Denver, CO" },
+            { key: "maxResults", label: "Max results", type: "number", min: 1, max: 20, default: 10, advanced: true }
+          ]
+        },
+        {
+          key: "maps_place_intel",
+          label: "Place Intel",
+          kind: "simple",
+          endpoint: "/maps/place",
+          mcpName: "maps_place_intel",
+          tagline: "One place: details + reviews",
+          params: [
+            { key: "businessName", label: "Business name", type: "string", required: true },
+            { key: "location", label: "Location", type: "string", required: true },
+            { key: "includeReviews", label: "Include reviews", type: "boolean", default: false },
+            { key: "maxReviews", label: "Max reviews", type: "number", min: 1, max: 500, default: 50, advanced: true }
+          ]
+        }
+      ]
+    },
+    {
+      key: "websites",
+      label: "Websites",
+      icon: "globe",
+      tools: [
+        {
+          key: "scrape",
+          label: "Scrape",
+          kind: "modal",
+          endpoint: "/extract-url",
+          mcpName: "extract_url",
+          tagline: "Headless single or bulk page extraction",
+          modes: [
+            { groupKey: "scope", groupLabel: "Scope", optionKey: "single", optionLabel: "Headless Single", isDefault: true, endpointOverride: "/extract-url", mcpNameOverride: "extract_url" },
+            { groupKey: "scope", groupLabel: "Scope", optionKey: "bulk", optionLabel: "Headless Bulk", endpointOverride: "/extract-site", mcpNameOverride: "extract_site", paramOverrides: { rotateProxies: true } }
+          ],
+          params: [
+            URL_PARAM,
+            FORMATS_PARAM,
+            { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100, advanced: true, sourceKey: "scope:bulk" },
+            { key: "screenshot", label: "Screenshot", type: "boolean", advanced: true }
+          ],
+          examples: ["https://steenshoney.com/"]
+        },
+        {
+          key: "map_site_urls",
+          label: "Map",
+          kind: "simple",
+          endpoint: "/map-urls",
+          mcpName: "map_site_urls",
+          tagline: "URL inventory only",
+          params: [URL_PARAM, { key: "maxUrls", label: "Max URLs", type: "number", min: 1, max: 1e4, default: 100, advanced: true }]
+        },
+        {
+          key: "extract_site",
+          label: "Crawl",
+          kind: "simple",
+          endpoint: "/extract-site",
+          mcpName: "extract_site",
+          tagline: "Map + bulk scrape + SEO report",
+          params: [
+            URL_PARAM,
+            { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
+            FORMATS_PARAM,
+            { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true },
+            { key: "background", label: "Run in background", type: "boolean", default: true, advanced: true }
+          ],
+          examples: ["https://steenshoney.com/ (full-site SEO crawl)"]
+        }
+      ]
+    },
+    {
+      key: "browser",
+      label: "Browser",
+      icon: "monitor",
+      tools: [
+        {
+          key: "browser_agent",
+          label: "Browser Agent",
+          kind: "simple",
+          endpoint: "/agent",
+          mcpName: "browser_open",
+          tagline: "Drive a live browser (open, navigate, read, click)",
+          params: [
+            { key: "url", label: "Start URL", type: "url" },
+            { key: "profile", label: "Use Chrome profile", type: "string", placeholder: "saved profile name", advanced: true }
+          ]
+        },
+        {
+          key: "query_fanout_workflow",
+          label: "AI Visibility (Fan-out)",
+          kind: "simple",
+          mcpName: "query_fanout_workflow",
+          tagline: "Capture what ChatGPT/Claude search & cite (AEO)",
+          params: [
+            { key: "prompt", label: "Prompt", type: "string" },
+            { key: "first_party_domain", label: "Your domain", type: "string", advanced: true },
+            { key: "export", label: "Export artifacts", type: "boolean", advanced: true }
+          ]
+        }
+      ]
+    },
+    {
+      key: "workflows",
+      label: "Workflows",
+      icon: "workflow",
+      tools: [
+        {
+          key: "directory_workflow",
+          label: "Directory",
+          kind: "simple",
+          endpoint: "/directory/run",
+          mcpName: "directory_workflow",
+          isSpecial: true,
+          tagline: "Build a local business directory",
+          params: [
+            { key: "query", label: "Business type", type: "string", required: true, placeholder: "roofers" },
+            { key: "state", label: "State", type: "string", required: true, placeholder: "TN" },
+            { key: "minPopulation", label: "Min city population", type: "number", default: 1e5, advanced: true }
+          ]
+        },
+        {
+          key: "rank_tracker_workflow",
+          label: "Rank Tracker",
+          kind: "simple",
+          mcpName: "rank_tracker_workflow",
+          isSpecial: true,
+          tagline: "Generate a rank-tracker blueprint",
+          params: [{ key: "domain", label: "Domain", type: "string", required: true }]
+        },
+        {
+          key: "deep_research_workflow",
+          label: "Deep Research",
+          kind: "simple",
+          endpoint: "/workflows/run",
+          mcpName: "workflow_run",
+          isSpecial: true,
+          tagline: "Multi-source, fact-checked research report",
+          params: [{ key: "question", label: "Question", type: "string", required: true }]
+        }
+      ]
+    }
+  ]
+};
+
 // src/api/site-audit-routes.ts
 import path2 from "path";
 import os from "os";
@@ -8049,9 +8996,13 @@ var MapUrlsBodySchema = z11.object({
 });
 var ExtractSiteBodySchema = z11.object({
   url: z11.string().min(1, "url is required"),
-  maxPages: z11.number().int().min(1).max(200).optional(),
+  maxPages: z11.number().int().min(1).max(1e4).optional(),
   browserFallback: z11.boolean().optional(),
-  kernelFallback: z11.boolean().optional()
+  kernelFallback: z11.boolean().optional(),
+  rotateProxies: z11.boolean().optional(),
+  rotateProxyEvery: z11.number().int().min(1).max(100).optional(),
+  background: z11.boolean().optional(),
+  formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding"])).optional()
 });
 var YoutubeHarvestBodySchema = z11.object({
   mode: z11.enum(["search", "channel"]),
@@ -8359,10 +9310,10 @@ function buildTranscriptMarkdown(result) {
   if (result.chunks?.length) {
     lines.push("## Timestamped Segments");
     lines.push("");
-    for (const chunk of result.chunks) {
-      const start = fmtTs(chunk.timestamp[0]);
-      const end = fmtTs(chunk.timestamp[1]);
-      lines.push(`**[${start} \u2192 ${end}]** ${chunk.text.trim()}`);
+    for (const chunk2 of result.chunks) {
+      const start = fmtTs(chunk2.timestamp[0]);
+      const end = fmtTs(chunk2.timestamp[1]);
+      lines.push(`**[${start} \u2192 ${end}]** ${chunk2.text.trim()}`);
       lines.push("");
     }
   }
@@ -8863,20 +9814,20 @@ var FacebookAdExtractor = class {
       if (splitRe.lastIndex === m.index) splitRe.lastIndex++;
     }
     if (last < bodyText.length) adChunks.push(bodyText.slice(last));
-    const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk) => {
-      const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
+    const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk2) => {
+      const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
       const libraryId = lidM ? lidM[1] : null;
-      const statusM = chunk.match(/^(Active|Inactive)/i);
+      const statusM = chunk2.match(/^(Active|Inactive)/i);
       const status = statusM ? statusM[1] : null;
-      const startM = chunk.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
+      const startM = chunk2.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
       const started = startM ? startM[1].replace(/\s+/g, " ").trim() : null;
       const visual = adVisuals[libraryId ?? ""] ?? { imageSrc: null, videoSrc: null, videoPoster: null };
-      const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk);
+      const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk2);
       const creativeType = visual.videoSrc || hasVideoText ? "video" : visual.imageSrc ? "image" : "unknown";
       let domain = null;
       const domainRe = /\b([A-Z0-9]{2,}(?:\.[A-Z]{2,})+)\b/g;
       let dm;
-      while ((dm = domainRe.exec(chunk)) !== null) {
+      while ((dm = domainRe.exec(chunk2)) !== null) {
         const c = dm[1];
         if (c.length >= 6 && c !== "LIBRARY") {
           domain = c;
@@ -8885,9 +9836,9 @@ var FacebookAdExtractor = class {
       }
       let primaryText = null;
       let headline = null;
-      const sponsoredIdx = chunk.search(/\bSponsored\b/i);
+      const sponsoredIdx = chunk2.search(/\bSponsored\b/i);
       if (sponsoredIdx >= 0) {
-        let afterSponsored = chunk.slice(sponsoredIdx + "Sponsored".length).trim();
+        let afterSponsored = chunk2.slice(sponsoredIdx + "Sponsored".length).trim();
         afterSponsored = afterSponsored.replace(/\s*\d*:?\d+\s*\/\s*\d*:?\d+.*$/, "");
         if (domain) {
           const di = afterSponsored.indexOf(domain);
@@ -8898,18 +9849,18 @@ var FacebookAdExtractor = class {
       }
       const ctaRe2 = new RegExp("\\b(" + CTA_LABELS.map((l) => l.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|") + ")\\b", "i");
       if (domain) {
-        const di = chunk.indexOf(domain);
+        const di = chunk2.indexOf(domain);
         if (di >= 0) {
-          let afterDomain = chunk.slice(di + domain.length).trim();
+          let afterDomain = chunk2.slice(di + domain.length).trim();
           const ctaMatch = afterDomain.search(ctaRe2);
           if (ctaMatch >= 0) afterDomain = afterDomain.slice(0, ctaMatch).trim();
           if (afterDomain.length > 3 && afterDomain.length < 120) headline = afterDomain;
         }
       }
       if (!headline) {
-        const ctaIdx = chunk.search(ctaRe2);
+        const ctaIdx = chunk2.search(ctaRe2);
         if (ctaIdx > 0) {
-          const beforeCta = chunk.slice(0, ctaIdx).trimEnd();
+          const beforeCta = chunk2.slice(0, ctaIdx).trimEnd();
           const sentenceEnd = Math.max(beforeCta.lastIndexOf(". "), beforeCta.lastIndexOf("! "), beforeCta.lastIndexOf("? "));
           if (sentenceEnd >= 0) {
             const candidate = beforeCta.slice(sentenceEnd + 2).trim();
@@ -8921,12 +9872,12 @@ var FacebookAdExtractor = class {
       }
       let cta = null;
       for (const lbl of CTA_LABELS) {
-        if (chunk.toLowerCase().includes(lbl.toLowerCase())) {
+        if (chunk2.toLowerCase().includes(lbl.toLowerCase())) {
           cta = lbl;
           break;
         }
       }
-      const multiM = chunk.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
+      const multiM = chunk2.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
       const clusterCount = multiM ? parseInt(multiM[1]) : null;
       return {
         libraryId,
@@ -8949,8 +9900,8 @@ var FacebookAdExtractor = class {
     const unknownCreativeCount = ads.filter((a) => a.creativeType === "unknown").length;
     const advertiserPageId = new URL(page.url()).searchParams.get("view_all_page_id") ?? null;
     const nameFreq = /* @__PURE__ */ new Map();
-    for (const chunk of adChunks) {
-      const detailsParts = chunk.split("See ad details ");
+    for (const chunk2 of adChunks) {
+      const detailsParts = chunk2.split("See ad details ");
       if (detailsParts.length < 2) continue;
       const sponsoredIdx = detailsParts[1].indexOf(" Sponsored");
       if (sponsoredIdx < 0) continue;
@@ -9690,11 +10641,11 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
     }
     if (last < bodyText.length) adChunks.push(bodyText.slice(last));
     const advertiserMap = /* @__PURE__ */ new Map();
-    for (const chunk of adChunks) {
-      const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
+    for (const chunk2 of adChunks) {
+      const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
       if (!lidM) continue;
       const libraryId = lidM[1];
-      const detailsParts = chunk.split("See ad details ");
+      const detailsParts = chunk2.split("See ad details ");
       if (detailsParts.length < 2) continue;
       const afterDetails = detailsParts[1];
       const sponsoredIdx = afterDetails.indexOf(" Sponsored");
@@ -9774,9 +10725,9 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
 // src/api/instagram-routes.ts
 import { Hono as Hono5 } from "hono";
 import { z as z14 } from "zod";
-import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync2, statSync, writeFileSync } from "fs";
-import { homedir as homedir2 } from "os";
-import { join as join4 } from "path";
+import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync3, statSync, writeFileSync as writeFileSync2 } from "fs";
+import { homedir as homedir3 } from "os";
+import { join as join5 } from "path";
 import { Readable as Readable2 } from "stream";
 import { pipeline as pipeline2 } from "stream/promises";
 import { spawn } from "child_process";
@@ -10199,7 +11150,7 @@ async function resolveInstagramLaunch(body) {
   };
 }
 function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join5(homedir3(), "Downloads", "mcp-scraper");
 }
 function safeFilePart(input) {
   return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
@@ -10207,8 +11158,8 @@ function safeFilePart(input) {
 function mediaOutputDir(shortcode, sourceUrl) {
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
-  const dir = join4(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
-  mkdirSync2(dir, { recursive: true });
+  const dir = join5(outputBaseDir2(), "instagram", `${stamp}-${slug}`);
+  mkdirSync3(dir, { recursive: true });
   return dir;
 }
 async function downloadToFile(url, destPath, referer) {
@@ -10246,8 +11197,8 @@ function runFfmpegMux(videoPath, audioPath, outPath) {
       stdio: ["ignore", "ignore", "pipe"]
     });
     let stderr = "";
-    child.stderr.on("data", (chunk) => {
-      stderr += String(chunk);
+    child.stderr.on("data", (chunk2) => {
+      stderr += String(chunk2);
     });
     child.on("error", (err) => resolve({ ok: false, error: err.message }));
     child.on("close", (code) => resolve({ ok: code === 0, error: code === 0 ? null : stderr.trim() || `ffmpeg exited ${code}` }));
@@ -10348,8 +11299,8 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     let outputDir = null;
     if (body.downloadMedia) {
       outputDir = mediaOutputDir(extraction.shortcode, sourceUrl.href);
-      const textPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
-      writeFileSync(textPath, [
+      const textPath = join5(outputDir, `${extraction.shortcode ?? "instagram"}-text.txt`);
+      writeFileSync2(textPath, [
         `URL: ${extraction.pageUrl}`,
         extraction.caption ? `Caption: ${extraction.caption}` : "",
         "",
@@ -10357,7 +11308,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
       ].filter(Boolean).join("\n"), "utf8");
       downloads.push({ kind: "text", url: null, savedPath: textPath, sizeBytes: statSync(textPath).size, mimeType: "text/plain", error: null });
       if (mediaTypes.has("image") && extraction.imageUrl) {
-        const imagePathBase = join4(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
+        const imagePathBase = join5(outputDir, `${extraction.shortcode ?? "instagram"}-image`);
         try {
           const downloaded = await downloadToFile(extraction.imageUrl, imagePathBase, extraction.pageUrl);
           const ext = extFromMime(downloaded.mimeType, "jpg");
@@ -10377,7 +11328,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         if (track.streamType === "video" && !mediaTypes.has("video")) continue;
         if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
         const kind = track.streamType === "audio" ? "audio" : "video";
-        const target = join4(outputDir, trackFilename(track, index, extraction.shortcode));
+        const target = join5(outputDir, trackFilename(track, index, extraction.shortcode));
         try {
           const downloaded = await downloadToFile(track.url, target, extraction.pageUrl);
           downloads.push({ kind, url: track.url, savedPath: downloaded.savedPath, sizeBytes: downloaded.sizeBytes, mimeType: downloaded.mimeType, error: null });
@@ -10387,7 +11338,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         }
       }
       if (body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack && savedByUrl.has(extraction.selectedVideoTrack.url) && savedByUrl.has(extraction.selectedAudioTrack.url)) {
-        const outPath = join4(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
+        const outPath = join5(outputDir, `${extraction.shortcode ?? "instagram"}-muxed.mp4`);
         const muxed = await runFfmpegMux(savedByUrl.get(extraction.selectedVideoTrack.url), savedByUrl.get(extraction.selectedAudioTrack.url), outPath);
         if (muxed.ok) {
           downloads.push({ kind: "muxed_video", url: null, savedPath: outPath, sizeBytes: statSync(outPath).size, mimeType: "video/mp4", error: null });
@@ -11372,7 +12323,7 @@ import { Hono as Hono7 } from "hono";
 
 // src/directory/directory-workflow.ts
 import { mkdir as mkdir2, writeFile } from "fs/promises";
-import { join as join5 } from "path";
+import { join as join6 } from "path";
 import { z as z16 } from "zod";
 
 // src/directory/location-db.ts
@@ -11708,11 +12659,11 @@ function csvRowsFor(result) {
   return rows;
 }
 async function saveDirectoryCsv(result) {
-  const outDir = join5(outputBaseDir(), "directory-workflows");
+  const outDir = join6(outputBaseDir(), "directory-workflows");
   await mkdir2(outDir, { recursive: true });
   const stamp = result.extractedAt.replace(/[:.]/g, "-");
   const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
-  const path5 = join5(outDir, `${stamp}-${slug}-directory-workflow.csv`);
+  const path5 = join6(outDir, `${stamp}-${slug}-directory-workflow.csv`);
   const headers = [
     "source_query",
     "source_location",
@@ -12380,14 +13331,14 @@ workflowApp.post("/cron/dispatch", async (c) => {
 import { Hono as Hono9 } from "hono";
 
 // src/serp-intelligence/page-snapshot-extractor.ts
-import { createHash } from "crypto";
+import { createHash as createHash2 } from "crypto";
 import pLimit3 from "p-limit";
 var DEFAULT_TIMEOUT_MS = 15e3;
 var DEFAULT_MAX_CONCURRENCY = 2;
 var DEFAULT_MAX_CONTENT_CHARS = 25e4;
 function sha256(value) {
   if (!value) return null;
-  return createHash("sha256").update(value).digest("hex");
+  return createHash2("sha256").update(value).digest("hex");
 }
 function countWords(markdown) {
   const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
@@ -12436,7 +13387,7 @@ function countLinks(html, baseUrl) {
     totalLinkCount: internalLinkCount + externalLinkCount
   };
 }
-function withTimeout(operation, timeoutMs) {
+function withTimeout2(operation, timeoutMs) {
   return new Promise((resolve, reject) => {
     const timer = setTimeout(() => reject(new Error(`Page snapshot timed out after ${timeoutMs}ms`)), timeoutMs);
     operation.then(
@@ -12553,7 +13504,7 @@ async function capturePageSnapshot(target, options = {}) {
     });
   }
   try {
-    const extraction = await withTimeout(
+    const extraction = await withTimeout2(
       extractKpo({ url: checked.parsed.href, kernelApiKey }),
       timeoutMs
     );
@@ -13486,7 +14437,7 @@ async function listReplayRows(sessionId) {
 }
 
 // src/services/browser-agent/browser-agent-service.ts
-import Kernel3 from "@onkernel/sdk";
+import Kernel4 from "@onkernel/sdk";
 import { chromium as playwrightChromium } from "playwright";
 
 // src/services/fanout/cdp-capture.ts
@@ -14054,7 +15005,7 @@ function buildFanoutResult(ctx) {
       interceptorReady: ready,
       rawBytes,
       unmappedKeys: parsed.unmappedKeys,
-      note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. Open a direct/no-proxy hosted browser session, navigate to chatgpt.com or claude.ai after opening, run a new prompt in that session, then call browser_capture_fanout again."
+      note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call query_fanout_workflow again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. Open a direct/no-proxy hosted browser session, navigate to chatgpt.com or claude.ai after opening, run a new prompt in that session, then call query_fanout_workflow again."
     };
   }
   return result;
@@ -14169,7 +15120,7 @@ async function runFanoutCapture(page, input) {
     }
     const adapter = adapterForHost(hostname);
     if (!adapter) {
-      throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
+      throw new Error(`query_fanout_workflow supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
     }
     const text = cap.bestAnswerText();
     const parsed = adapter.parse(text, input.prompt || "");
@@ -14189,7 +15140,7 @@ var DEFAULT_TIMEOUT_SECONDS = 600;
 function client() {
   const apiKey = browserServiceApiKey();
   if (!apiKey) throw new Error("Browser backend API key is required");
-  return new Kernel3({ apiKey });
+  return new Kernel4({ apiKey });
 }
 function isProfileConflict(err) {
   const message = err instanceof Error ? err.message : String(err);
@@ -15721,6 +16672,10 @@ app.get("/stats/spots", async (c) => {
   const taken = await countActiveUsers();
   return c.json({ taken, open: true });
 });
+app.get("/catalog", (c) => {
+  c.header("Cache-Control", "public, max-age=60");
+  return c.json(CATALOG);
+});
 app.get("/me", async (c) => {
   const token = getCookie(c, "session");
   if (!token) return c.json({ authenticated: false });
@@ -15994,7 +16949,7 @@ app.post("/extract-url", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = ExtractUrlBodySchema.safeParse(raw);
   if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
-  const { url, screenshot: screenshot2, screenshotDevice, extractBranding, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
+  const { url, screenshot: screenshot2, screenshotDevice, extractBranding: extractBranding2, downloadMedia, mediaTypes, allowLocal } = bodyResult.data;
   if (!allowLocal) {
     const checked = await validatePublicHttpUrl(url, { field: "URL" });
     if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
@@ -16027,7 +16982,7 @@ app.post("/extract-url", auth2, async (c) => {
     const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
     const [result, pageData] = await Promise.all([
       extractKpo({ url: canonicalUrl, kernelApiKey }),
-      screenshot2 || extractBranding ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding }).catch((err) => {
+      screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding2 }).catch((err) => {
         console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
         return null;
       }) : null
@@ -16071,7 +17026,7 @@ app.post("/map-urls", auth2, async (c) => {
     debited = true;
     const result = await spiderSite({
       startUrl: parsed.href,
-      maxUrls: Math.min(2e3, Math.max(1, body.maxUrls ?? 500)),
+      maxUrls: Math.min(1e4, Math.max(1, body.maxUrls ?? 500)),
       concurrency: Math.min(20, Math.max(1, body.concurrency ?? 12)),
       kernelApiKey: body.browserFallback ?? body.kernelFallback ? browserServiceApiKey() : void 0
     });
@@ -16108,7 +17063,43 @@ app.post("/extract-site", auth2, async (c) => {
   if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
   const parsed = checked.parsed;
   const user = c.get("user");
-  const siteHoldMc = MC_COSTS.page_scrape * 10;
+  if (body.background === true) {
+    const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
+    if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
+    const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
+    const heldMc = maxPages * MC_COSTS.page_scrape;
+    const { ok: holdOk, balance_mc: holdBal } = await debitMc(user.id, heldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
+    if (!holdOk) return c.json(insufficientBalanceResponse(holdBal, heldMc), 402);
+    const jobId = `ext_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
+    let jobCreated = false;
+    try {
+      await createExtractJob(jobId, user.id, parsed.href, {
+        maxPages,
+        concurrency: concurrencyLimitForUser(user),
+        urlsPerBrowser: body.rotateProxyEvery ?? 10,
+        formats: body.formats,
+        heldMc
+      });
+      jobCreated = true;
+      await inngest.send({ name: "mcp-scraper/extract.requested", data: { jobId } });
+    } catch (err) {
+      if (jobCreated) {
+        await settleExtractJob(jobId, user.id, heldMc, 0, "enqueue failed").catch(() => {
+        });
+      } else {
+        await creditMc(user.id, heldMc, LedgerOperation.EXTRACT_SITE_REFUND, "enqueue failed").catch(() => {
+        });
+      }
+      await failExtractJob(jobId, err instanceof Error ? err.message : "enqueue failed").catch(() => {
+      });
+      return c.json({ error: "Failed to enqueue background crawl" }, 503);
+    }
+    return c.json({ jobId, status: "pending", statusUrl: `/extract-site/status/${jobId}` });
+  }
+  const affordablePagesSync = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
+  if (affordablePagesSync < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
+  const siteMaxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 100)), affordablePagesSync);
+  const siteHoldMc = siteMaxPages * MC_COSTS.page_scrape;
   const gate = await acquireConcurrencyGate(user, "extract_site", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
     metadata: { url: parsed.href }
@@ -16119,10 +17110,17 @@ app.post("/extract-site", auth2, async (c) => {
     const { ok: siteOk, balance_mc: siteBal } = await debitMc(user.id, siteHoldMc, LedgerOperation.EXTRACT_SITE_HOLD, parsed.hostname);
     if (!siteOk) return c.json(insufficientBalanceResponse(siteBal, siteHoldMc), 402);
     debited = true;
+    const rotateProxies = body.rotateProxies === true;
+    const wantsBranding = body.formats?.includes("branding") === true;
     const result = await extractSite({
       startUrl: parsed.href,
-      maxPages: Math.min(200, Math.max(1, body.maxPages ?? 100)),
-      kernelApiKey: body.browserFallback ?? body.kernelFallback ? browserServiceApiKey() : void 0
+      maxPages: siteMaxPages,
+      kernelApiKey: rotateProxies || wantsBranding || body.browserFallback || body.kernelFallback ? browserServiceApiKey() : void 0,
+      formats: body.formats,
+      ...rotateProxies ? {
+        rotateProxyEvery: body.rotateProxyEvery ?? 30,
+        parallelism: concurrencyLimitForUser(user)
+      } : {}
     });
     const pageCount = result.pages?.length ?? 1;
     const actualSiteMc = pageCount * MC_COSTS.page_scrape;
@@ -16153,6 +17151,21 @@ app.post("/extract-site", auth2, async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+app.get("/extract-site/status/:id", auth2, async (c) => {
+  const user = c.get("user");
+  const job = await getExtractJob(c.req.param("id"));
+  if (!job || job.userId !== user.id) return c.json({ error: "Job not found" }, 404);
+  return c.json({
+    jobId: job.id,
+    status: job.status,
+    startUrl: job.startUrl,
+    totalUrls: job.totalUrls,
+    doneUrls: job.doneUrls,
+    artifacts: job.artifacts ?? [],
+    error: job.error,
+    updatedAt: job.updatedAt
+  });
+});
 app.post("/billing/checkout", requireAllowedOrigin, sessionAuth, async (c) => {
   try {
     const user = c.get("sessionUser");
@@ -16332,7 +17345,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-FG7ZWEGA.js");
+  const { drainQueue } = await import("./worker-2WVKKCC7.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult] = await Promise.all([
@@ -16341,7 +17354,7 @@ app.get("/cron/tick", async (c) => {
   ]);
   return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
 });
-app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn] }));
+app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
 app.route("/api/internal/site-architecture-auditor", siteAuditApp);
 app.route("/youtube", youtubeApp);
 app.route("/screenshot", screenshotApp);
@@ -16462,4 +17475,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-LX65PB7D.js.map
+//# sourceMappingURL=server-UOQ2JIUQ.js.map