mcp-scraper 0.7.0 → 0.9.0

package/dist/bin/api-server.cjs

@@ -6717,7 +6717,8 @@ async function spiderSite(opts) {
     totalFound: results.length,
     durationMs: Date.now() - startMs,
     truncated: queue.length > 0 || results.length >= maxUrls,
-    browserRetries
+    browserRetries,
+    sitemapUrls: sitemapUrls.slice(0, maxUrls)
   };
 }
 var SKIP_EXTENSIONS, SKIP_PATH_SEGMENTS, SKIP_PATH_PREFIXES, SKIP_QUERY_PARAMS, KERNEL_RETRY_LIMIT, UA;
@@ -6944,6 +6945,465 @@ var init_rotating_proxy_crawl = __esm({
   }
 });
 
+// src/api/image-audit.ts
+function formatBytes(n) {
+  if (n == null) return null;
+  if (n < 1024) return `${n} B`;
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
+}
+function collectUrls(pages) {
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of pages) for (const raw of p.imageLinks || []) {
+    const u = decodeEntities(raw);
+    if (!/^https?:\/\//i.test(u)) continue;
+    if (/[{}]|%7[bd]/i.test(u)) continue;
+    const k = dedupKey(u);
+    const prev = byKey.get(k);
+    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
+  }
+  return [...byKey.values()];
+}
+async function sizeAndType(url, timeoutMs) {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  const read = (res) => ({
+    len: res.headers.get("content-length"),
+    cr: res.headers.get("content-range"),
+    ct: res.headers.get("content-type"),
+    status: res.status
+  });
+  try {
+    let bytes = null;
+    let ct = null;
+    let status = null;
+    try {
+      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
+      status = h.status;
+      ct = h.ct;
+      if (h.len) bytes = Number(h.len);
+    } catch {
+    }
+    if (bytes == null) {
+      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
+      const r = read(g);
+      status = status ?? r.status;
+      ct = ct ?? r.ct;
+      if (r.cr && r.cr.includes("/")) {
+        const tail = r.cr.split("/")[1];
+        if (tail && tail !== "*") bytes = Number(tail);
+      } else if (r.len) bytes = Number(r.len);
+      try {
+        await g.body?.cancel();
+      } catch {
+      }
+    }
+    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
+  } catch (e) {
+    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
+  } finally {
+    clearTimeout(t);
+  }
+}
+async function pool(items, n, fn) {
+  const out = new Array(items.length);
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) {
+      const idx = i++;
+      out[idx] = await fn(items[idx]);
+    }
+  }));
+  return out;
+}
+async function auditImages(pages, opts = {}) {
+  const concurrency = opts.concurrency ?? 12;
+  const timeoutMs = opts.timeoutMs ?? 12e3;
+  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
+  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
+  const rows = heads.map((r) => {
+    const format = formatOf(r.contentType, r.url);
+    return {
+      ...r,
+      size: formatBytes(r.bytes),
+      format,
+      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
+      legacyFormat: format !== "unknown" && !MODERN.has(format)
+    };
+  });
+  const sized = rows.filter((r) => r.bytes != null);
+  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
+  const formatCounts = {};
+  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
+  return {
+    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
+    summary: {
+      unique: rows.length,
+      sized: sized.length,
+      totalBytes,
+      totalSize: formatBytes(totalBytes) ?? "0 B",
+      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
+      over100kb: rows.filter((r) => r.over100kb).length,
+      legacyFormat: rows.filter((r) => r.legacyFormat).length,
+      formatCounts
+    }
+  };
+}
+function renderImageSection(audit) {
+  const s = audit.summary;
+  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
+  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
+  const lines = [
+    `## Images`,
+    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
+    `Formats: ${fmt}`
+  ];
+  if (heaviest.length) {
+    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
+    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
+  }
+  return lines.join("\n");
+}
+var OVER_BYTES, MODERN, decodeEntities, dedupKey, formatOf;
+var init_image_audit = __esm({
+  "src/api/image-audit.ts"() {
+    "use strict";
+    OVER_BYTES = 100 * 1024;
+    MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
+    decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
+    dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
+    formatOf = (ct, url) => {
+      if (ct) {
+        const m = ct.split(";")[0].trim().toLowerCase();
+        if (m.startsWith("image/")) return m.replace("image/", "");
+      }
+      return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
+    };
+  }
+});
+
+// src/api/seo-link-graph.ts
+function normalize(u) {
+  return u.split("#")[0].replace(/\/$/, "");
+}
+function buildLinkGraph(pages, startUrl) {
+  const statusByUrl = /* @__PURE__ */ new Map();
+  for (const p of pages) statusByUrl.set(normalize(p.url), p.status);
+  const edges = [];
+  for (const p of pages) {
+    for (const l of p.outlinks ?? []) {
+      const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
+      edges.push({
+        from: p.url,
+        to: l.href,
+        anchor: l.anchor,
+        rel: l.rel,
+        internal: l.internal,
+        nofollow,
+        targetStatus: statusByUrl.get(normalize(l.href)) ?? null
+      });
+    }
+  }
+  const inboundFrom = /* @__PURE__ */ new Map();
+  const inboundAnchors = /* @__PURE__ */ new Map();
+  const adjacency = /* @__PURE__ */ new Map();
+  for (const e of edges) {
+    if (!e.internal) continue;
+    const to = normalize(e.to);
+    const from = normalize(e.from);
+    if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
+    inboundFrom.get(to).add(from);
+    if (e.anchor) {
+      if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
+      inboundAnchors.get(to).push(e.anchor);
+    }
+    if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
+      if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
+      adjacency.get(from).add(to);
+    }
+  }
+  const depth = /* @__PURE__ */ new Map();
+  const start = normalize(startUrl);
+  const queue = [start];
+  depth.set(start, 0);
+  while (queue.length > 0) {
+    const cur = queue.shift();
+    const d = depth.get(cur);
+    for (const next of adjacency.get(cur) ?? []) {
+      if (!depth.has(next)) {
+        depth.set(next, d + 1);
+        queue.push(next);
+      }
+    }
+  }
+  const topAnchorsFor = (url) => {
+    const counts = /* @__PURE__ */ new Map();
+    for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
+    return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
+  };
+  const metrics = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const key = normalize(p.url);
+    const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
+    metrics.set(p.url, {
+      url: p.url,
+      inlinks: inbound.size,
+      uniqueInlinks: inbound.size,
+      outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
+      outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
+      crawlDepth: depth.has(key) ? depth.get(key) : null,
+      orphan: inbound.size === 0 && key !== start,
+      topAnchors: topAnchorsFor(key)
+    });
+  }
+  return { edges, metrics };
+}
+var init_seo_link_graph = __esm({
+  "src/api/seo-link-graph.ts"() {
+    "use strict";
+  }
+});
+
+// src/api/seo-link-report.ts
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function buildLinkReport(edges, metrics, siteUrl) {
+  let siteReg = "";
+  try {
+    siteReg = registrableDomain(new URL(siteUrl).hostname);
+  } catch {
+    siteReg = "";
+  }
+  const internalEdges = edges.filter((e) => e.internal);
+  const pages = metrics.length;
+  const orphans = metrics.filter((m) => m.orphan).length;
+  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
+  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
+  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
+  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+  for (const m of metrics) {
+    if (m.inlinks === 0) distribution.zero++;
+    else if (m.inlinks <= 2) distribution.oneToTwo++;
+    else if (m.inlinks <= 10) distribution.threeToTen++;
+    else distribution.elevenPlus++;
+  }
+  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
+  const domMap = /* @__PURE__ */ new Map();
+  let externalTotal = 0;
+  for (const e of edges) {
+    let domain;
+    try {
+      domain = registrableDomain(new URL(e.to).hostname);
+    } catch {
+      continue;
+    }
+    if (!domain || domain === siteReg) continue;
+    externalTotal++;
+    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+    d.links++;
+    if (e.nofollow) d.nofollow++;
+    d.pages.add(e.from);
+    domMap.set(domain, d);
+  }
+  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+  const round = (n) => Math.round(n * 10) / 10;
+  return {
+    summary: {
+      internal: {
+        totalLinks: internalEdges.length,
+        pages,
+        orphans,
+        brokenInternal,
+        avgInlinks: pages ? round(sumInlinks / pages) : 0,
+        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
+        distribution,
+        topByInlinks
+      },
+      external: {
+        totalLinks: externalTotal,
+        uniqueDomains: externalDomains.length,
+        topDomains: externalDomains.slice(0, 20)
+      }
+    },
+    externalDomains
+  };
+}
+function renderLinkReport(r) {
+  const s = r.summary;
+  const lines = [
+    `## Link analysis`,
+    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
+    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
+    ``,
+    `### Inlink distribution`,
+    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
+    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
+    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
+    `- 11+: ${s.internal.distribution.elevenPlus}`,
+    ``,
+    `### Top 20 internal pages by inlinks`,
+    `| inlinks | out (int/ext) | URL |`,
+    `|---|---|---|`,
+    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
+    ``,
+    `### Top 20 external domains by links`,
+    `| links | nofollow | from pages | domain |`,
+    `|---|---|---|---|`,
+    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
+  ];
+  return lines.join("\n");
+}
+var init_seo_link_report = __esm({
+  "src/api/seo-link-report.ts"() {
+    "use strict";
+  }
+});
+
+// src/api/seo-issues.ts
+function dupes(pages, key) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    const k = key(p);
+    if (k === null || k === void 0 || k === "") continue;
+    const ks = String(k);
+    if (!groups.has(ks)) groups.set(ks, []);
+    groups.get(ks).push(p.url);
+  }
+  return [...groups.values()].filter((g) => g.length > 1).flat();
+}
+function computeIssues(pages, metrics, precomputedBrokenLinkPages) {
+  const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
+  const where = (fn) => pages.filter(fn).map((p) => p.url);
+  const ok = (p) => p.status === 200;
+  const pathname = (u) => {
+    try {
+      return new URL(u).pathname;
+    } catch {
+      return "";
+    }
+  };
+  const report = {
+    "title.missing": group(where((p) => ok(p) && !p.title)),
+    "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
+    "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
+    "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
+    "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
+    "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
+    "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
+    "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
+    "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
+    "h1.missing": group(where((p) => ok(p) && !p.h1)),
+    "h1.multiple": group(where((p) => !!p.h1_2)),
+    "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
+    "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
+    "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
+    "indexability.nonIndexable": group(where((p) => !p.indexable)),
+    "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
+    "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
+    "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
+    "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
+    "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
+    "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
+    "response.noResponse": group(where((p) => p.status === null)),
+    "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
+    "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
+    "images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
+    "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
+    "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
+    "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
+    "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
+    "links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
+  };
+  let brokenLinkPages = precomputedBrokenLinkPages;
+  if (!brokenLinkPages) {
+    const normUrl2 = (u) => {
+      try {
+        const x = new URL(u);
+        x.hash = "";
+        return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
+      } catch {
+        return u.replace(/#.*$/, "").replace(/\/+$/, "");
+      }
+    };
+    const statusByUrl = new Map(pages.map((p) => [normUrl2(p.url), p.status]));
+    brokenLinkPages = /* @__PURE__ */ new Set();
+    for (const p of pages) {
+      for (const l of p.outlinks ?? []) {
+        if (!l.internal) continue;
+        const st = statusByUrl.get(normUrl2(l.href));
+        if (st != null && st >= 400) brokenLinkPages.add(p.url);
+      }
+    }
+  }
+  report["links.brokenInternal"] = group([...brokenLinkPages]);
+  return report;
+}
+function renderIssueReport(siteUrl, pages, report, metrics) {
+  const total = pages.length;
+  const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
+  const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
+  const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
+  const maxDepth = depths.length ? Math.max(...depths) : 0;
+  const orphans = [...metrics.values()].filter((m) => m.orphan).length;
+  const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
+  return [
+    `# SEO Crawl Report: ${siteUrl}`,
+    `**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
+    `
+## Issues
+| | Issue | Count | Examples |
+|---|-------|-------|----------|
+${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
+    `
+## Most-linked pages
+${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
+    `
+_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
+  ].join("\n");
+}
+var THIN_WORDS, TITLE_MAX, TITLE_PX_MAX, TITLE_MIN, META_MAX, META_MIN, H1_MAX, URL_MAX;
+var init_seo_issues = __esm({
+  "src/api/seo-issues.ts"() {
+    "use strict";
+    THIN_WORDS = 200;
+    TITLE_MAX = 60;
+    TITLE_PX_MAX = 561;
+    TITLE_MIN = 30;
+    META_MAX = 155;
+    META_MIN = 70;
+    H1_MAX = 70;
+    URL_MAX = 115;
+  }
+});
+
+// src/api/seo-audit.ts
+function buildSeoAudit(pages, startUrl) {
+  const { edges, metrics } = buildLinkGraph(pages, startUrl);
+  const issues = computeIssues(pages, metrics);
+  const { summary } = buildLinkReport(edges, [...metrics.values()], startUrl);
+  return {
+    issues,
+    linkSummary: summary,
+    pageMetrics: [...metrics.values()].map((m) => ({
+      url: m.url,
+      inboundInternal: m.inlinks,
+      outboundInternal: m.outlinksInternal,
+      orphan: m.orphan
+    }))
+  };
+}
+var init_seo_audit = __esm({
+  "src/api/seo-audit.ts"() {
+    "use strict";
+    init_seo_link_graph();
+    init_seo_link_report();
+    init_seo_issues();
+  }
+});
+
 // src/api/site-extractor.ts
 function estimatePixels(s) {
   let px = 0;
@@ -6986,6 +7446,7 @@ function emptyPageData(url, status, via) {
     imageLinks: [],
     indexable: status === 200,
     indexabilityReason: status === 200 ? null : "non-200",
+    inSitemap: null,
     schemaTypes: [],
     canonicalUrl: null,
     internalLinks: 0,
@@ -7002,7 +7463,8 @@ function makeSeedSpider(startUrl, urls) {
     totalFound: urls.length,
     durationMs: 0,
     truncated: false,
-    browserRetries: 0
+    browserRetries: 0,
+    sitemapUrls: []
   };
 }
 function parsePageData(url, html, status, via, resp) {
@@ -7169,6 +7631,7 @@ function parsePageData(url, html, status, via, resp) {
     imageLinks,
     indexable,
     indexabilityReason,
+    inSitemap: null,
     schemaTypes,
     canonicalUrl,
     internalLinks,
@@ -7182,11 +7645,11 @@ async function extractPagesRotating(urls, opts) {
   const fetched = await crawlWithRotation(urls, { apiKey: opts.kernelApiKey, concurrency: opts.concurrency, urlsPerBrowser: opts.urlsPerBrowser });
   return fetched.map((r) => r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser"));
 }
-async function discoverUrls(startUrl, maxPages, kernelApiKey) {
+async function discoverUrlsWithSitemap(startUrl, maxPages, kernelApiKey) {
   const sitemapUrls = await discoverSitemapUrls2(startUrl, maxPages);
-  if (sitemapUrls.length > 0) return sitemapUrls;
+  if (sitemapUrls.length > 0) return { urls: sitemapUrls, sitemapUrls };
   const spider = await spiderSite({ startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey });
-  return spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+  return { urls: spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url), sitemapUrls: spider.sitemapUrls };
 }
 async function fetchAndParse(url, kernelApiKey) {
   const startedAt = Date.now();
@@ -7245,6 +7708,7 @@ async function extractSite(opts) {
   let browserRetries = 0;
   let truncated = false;
   let spider;
+  let sitemapUrls = [];
   if (rotating) {
     const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
     const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
@@ -7262,7 +7726,8 @@ async function extractSite(opts) {
       for (const u of opts.seedUrls) enqueue(u);
     } else {
       enqueue(opts.startUrl);
-      for (const u of await discoverSitemapUrls2(opts.startUrl, maxPages)) enqueue(u);
+      sitemapUrls = await discoverSitemapUrls2(opts.startUrl, maxPages);
+      for (const u of sitemapUrls) enqueue(u);
     }
     const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
     let done = 0;
@@ -7291,6 +7756,7 @@ async function extractSite(opts) {
     spider = makeSeedSpider(opts.startUrl, [...seen]);
   } else {
     spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
+    sitemapUrls = spider.sitemapUrls;
     const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
     browserRetries = spider.browserRetries;
     await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
@@ -7300,10 +7766,17 @@ async function extractSite(opts) {
     });
     truncated = spider.truncated || pages.length >= maxPages;
   }
+  const sitemapSet = sitemapUrls.length > 0 ? new Set(sitemapUrls.map((u) => normalizeUrl(u, opts.startUrl) ?? u)) : null;
+  for (const p of pages) p.inSitemap = sitemapSet ? sitemapSet.has(normalizeUrl(p.url, opts.startUrl) ?? p.url) : null;
   let branding = null;
   if (opts.formats?.includes("branding") && opts.kernelApiKey) {
     branding = await extractBranding(opts.startUrl, opts.kernelApiKey).catch(() => null);
   }
+  const seoAudit = opts.formats?.includes("issues") ? buildSeoAudit(pages, opts.startUrl) : null;
+  let imageAudit = null;
+  if (opts.formats?.includes("images")) {
+    imageAudit = await auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+  }
   return {
     startUrl: opts.startUrl,
     pages,
@@ -7312,7 +7785,9 @@ async function extractSite(opts) {
     durationMs: Date.now() - startMs,
     truncated,
     browserRetries,
-    branding
+    branding,
+    seoAudit,
+    imageAudit
   };
 }
 var import_turndown2, import_node_crypto3, PIXEL_WIDTHS, UA2, EXTRACT_CONCURRENCY, MAX_PAGE_MARKDOWN, turndown;
@@ -7325,6 +7800,8 @@ var init_site_extractor = __esm({
     init_kernel_fetch();
     init_rotating_proxy_crawl();
     init_screenshot();
+    init_image_audit();
+    init_seo_audit();
     PIXEL_WIDTHS = { i: 4, l: 4, j: 4, ".": 4, ",": 4, "'": 4, t: 6, f: 6, r: 6, " ": 4, m: 14, w: 13, W: 16, M: 16 };
     UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
     EXTRACT_CONCURRENCY = 6;
@@ -10048,440 +10525,6 @@ var init_site_audit = __esm({
   }
 });
 
-// src/api/seo-link-graph.ts
-function normalize(u) {
-  return u.split("#")[0].replace(/\/$/, "");
-}
-function buildLinkGraph(pages, startUrl) {
-  const statusByUrl = /* @__PURE__ */ new Map();
-  for (const p of pages) statusByUrl.set(normalize(p.url), p.status);
-  const edges = [];
-  for (const p of pages) {
-    for (const l of p.outlinks ?? []) {
-      const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
-      edges.push({
-        from: p.url,
-        to: l.href,
-        anchor: l.anchor,
-        rel: l.rel,
-        internal: l.internal,
-        nofollow,
-        targetStatus: statusByUrl.get(normalize(l.href)) ?? null
-      });
-    }
-  }
-  const inboundFrom = /* @__PURE__ */ new Map();
-  const inboundAnchors = /* @__PURE__ */ new Map();
-  const adjacency = /* @__PURE__ */ new Map();
-  for (const e of edges) {
-    if (!e.internal) continue;
-    const to = normalize(e.to);
-    const from = normalize(e.from);
-    if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
-    inboundFrom.get(to).add(from);
-    if (e.anchor) {
-      if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
-      inboundAnchors.get(to).push(e.anchor);
-    }
-    if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
-      if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
-      adjacency.get(from).add(to);
-    }
-  }
-  const depth = /* @__PURE__ */ new Map();
-  const start = normalize(startUrl);
-  const queue = [start];
-  depth.set(start, 0);
-  while (queue.length > 0) {
-    const cur = queue.shift();
-    const d = depth.get(cur);
-    for (const next of adjacency.get(cur) ?? []) {
-      if (!depth.has(next)) {
-        depth.set(next, d + 1);
-        queue.push(next);
-      }
-    }
-  }
-  const topAnchorsFor = (url) => {
-    const counts = /* @__PURE__ */ new Map();
-    for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
-    return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
-  };
-  const metrics = /* @__PURE__ */ new Map();
-  for (const p of pages) {
-    const key = normalize(p.url);
-    const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
-    metrics.set(p.url, {
-      url: p.url,
-      inlinks: inbound.size,
-      uniqueInlinks: inbound.size,
-      outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
-      outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
-      crawlDepth: depth.has(key) ? depth.get(key) : null,
-      orphan: inbound.size === 0 && key !== start,
-      topAnchors: topAnchorsFor(key)
-    });
-  }
-  return { edges, metrics };
-}
-var init_seo_link_graph = __esm({
-  "src/api/seo-link-graph.ts"() {
-    "use strict";
-  }
-});
-
-// src/api/seo-link-report.ts
-function registrableDomain(host) {
-  const h = host.replace(/^www\./, "").toLowerCase();
-  const parts = h.split(".");
-  return parts.length <= 2 ? h : parts.slice(-2).join(".");
-}
-function buildLinkReport(edges, metrics, siteUrl) {
-  let siteReg = "";
-  try {
-    siteReg = registrableDomain(new URL(siteUrl).hostname);
-  } catch {
-    siteReg = "";
-  }
-  const internalEdges = edges.filter((e) => e.internal);
-  const pages = metrics.length;
-  const orphans = metrics.filter((m) => m.orphan).length;
-  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
-  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
-  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
-  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
-  for (const m of metrics) {
-    if (m.inlinks === 0) distribution.zero++;
-    else if (m.inlinks <= 2) distribution.oneToTwo++;
-    else if (m.inlinks <= 10) distribution.threeToTen++;
-    else distribution.elevenPlus++;
-  }
-  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
-  const domMap = /* @__PURE__ */ new Map();
-  let externalTotal = 0;
-  for (const e of edges) {
-    let domain;
-    try {
-      domain = registrableDomain(new URL(e.to).hostname);
-    } catch {
-      continue;
-    }
-    if (!domain || domain === siteReg) continue;
-    externalTotal++;
-    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
-    d.links++;
-    if (e.nofollow) d.nofollow++;
-    d.pages.add(e.from);
-    domMap.set(domain, d);
-  }
-  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
-  const round = (n) => Math.round(n * 10) / 10;
-  return {
-    summary: {
-      internal: {
-        totalLinks: internalEdges.length,
-        pages,
-        orphans,
-        brokenInternal,
-        avgInlinks: pages ? round(sumInlinks / pages) : 0,
-        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
-        distribution,
-        topByInlinks
-      },
-      external: {
-        totalLinks: externalTotal,
-        uniqueDomains: externalDomains.length,
-        topDomains: externalDomains.slice(0, 20)
-      }
-    },
-    externalDomains
-  };
-}
-function renderLinkReport(r) {
-  const s = r.summary;
-  const lines = [
-    `## Link analysis`,
-    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
-    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
-    ``,
-    `### Inlink distribution`,
-    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
-    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
-    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
-    `- 11+: ${s.internal.distribution.elevenPlus}`,
-    ``,
-    `### Top 20 internal pages by inlinks`,
-    `| inlinks | out (int/ext) | URL |`,
-    `|---|---|---|`,
-    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
-    ``,
-    `### Top 20 external domains by links`,
-    `| links | nofollow | from pages | domain |`,
-    `|---|---|---|---|`,
-    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
-  ];
-  return lines.join("\n");
-}
-var init_seo_link_report = __esm({
-  "src/api/seo-link-report.ts"() {
-    "use strict";
-  }
-});
-
-// src/api/seo-issues.ts
-function dupes(pages, key) {
-  const groups = /* @__PURE__ */ new Map();
-  for (const p of pages) {
-    const k = key(p);
-    if (k === null || k === void 0 || k === "") continue;
-    const ks = String(k);
-    if (!groups.has(ks)) groups.set(ks, []);
-    groups.get(ks).push(p.url);
-  }
-  return [...groups.values()].filter((g) => g.length > 1).flat();
-}
-function computeIssues(pages, metrics, precomputedBrokenLinkPages) {
-  const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
-  const where = (fn) => pages.filter(fn).map((p) => p.url);
-  const ok = (p) => p.status === 200;
-  const pathname = (u) => {
-    try {
-      return new URL(u).pathname;
-    } catch {
-      return "";
-    }
-  };
-  const report = {
-    "title.missing": group(where((p) => ok(p) && !p.title)),
-    "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
-    "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
-    "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
-    "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
-    "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
-    "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
-    "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
-    "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
-    "h1.missing": group(where((p) => ok(p) && !p.h1)),
-    "h1.multiple": group(where((p) => !!p.h1_2)),
-    "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
-    "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
-    "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
-    "indexability.nonIndexable": group(where((p) => !p.indexable)),
-    "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
-    "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
-    "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
-    "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
-    "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
-    "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
-    "response.noResponse": group(where((p) => p.status === null)),
-    "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
-    "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
-    "images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
-    "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
-    "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
-    "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
-    "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
-    "links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
-  };
-  let brokenLinkPages = precomputedBrokenLinkPages;
-  if (!brokenLinkPages) {
-    const normUrl2 = (u) => {
-      try {
-        const x = new URL(u);
-        x.hash = "";
-        return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
-      } catch {
-        return u.replace(/#.*$/, "").replace(/\/+$/, "");
-      }
-    };
-    const statusByUrl = new Map(pages.map((p) => [normUrl2(p.url), p.status]));
-    brokenLinkPages = /* @__PURE__ */ new Set();
-    for (const p of pages) {
-      for (const l of p.outlinks ?? []) {
-        if (!l.internal) continue;
-        const st = statusByUrl.get(normUrl2(l.href));
-        if (st != null && st >= 400) brokenLinkPages.add(p.url);
-      }
-    }
-  }
-  report["links.brokenInternal"] = group([...brokenLinkPages]);
-  return report;
-}
-function renderIssueReport(siteUrl, pages, report, metrics) {
-  const total = pages.length;
-  const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
-  const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
-  const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
-  const maxDepth = depths.length ? Math.max(...depths) : 0;
-  const orphans = [...metrics.values()].filter((m) => m.orphan).length;
-  const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
-  return [
-    `# SEO Crawl Report: ${siteUrl}`,
-    `**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
-    `
-## Issues
-| | Issue | Count | Examples |
-|---|-------|-------|----------|
-${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
-    `
-## Most-linked pages
-${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
-    `
-_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
-  ].join("\n");
-}
-var THIN_WORDS, TITLE_MAX, TITLE_PX_MAX, TITLE_MIN, META_MAX, META_MIN, H1_MAX, URL_MAX;
-var init_seo_issues = __esm({
-  "src/api/seo-issues.ts"() {
-    "use strict";
-    THIN_WORDS = 200;
-    TITLE_MAX = 60;
-    TITLE_PX_MAX = 561;
-    TITLE_MIN = 30;
-    META_MAX = 155;
-    META_MIN = 70;
-    H1_MAX = 70;
-    URL_MAX = 115;
-  }
-});
-
-// src/api/image-audit.ts
-function formatBytes(n) {
-  if (n == null) return null;
-  if (n < 1024) return `${n} B`;
-  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
-  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
-}
-function collectUrls(pages) {
-  const byKey = /* @__PURE__ */ new Map();
-  for (const p of pages) for (const raw of p.imageLinks || []) {
-    const u = decodeEntities(raw);
-    if (!/^https?:\/\//i.test(u)) continue;
-    if (/[{}]|%7[bd]/i.test(u)) continue;
-    const k = dedupKey(u);
-    const prev = byKey.get(k);
-    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
-  }
-  return [...byKey.values()];
-}
-async function sizeAndType(url, timeoutMs) {
-  const ctrl = new AbortController();
-  const t = setTimeout(() => ctrl.abort(), timeoutMs);
-  const read = (res) => ({
-    len: res.headers.get("content-length"),
-    cr: res.headers.get("content-range"),
-    ct: res.headers.get("content-type"),
-    status: res.status
-  });
-  try {
-    let bytes = null;
-    let ct = null;
-    let status = null;
-    try {
-      const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
-      status = h.status;
-      ct = h.ct;
-      if (h.len) bytes = Number(h.len);
-    } catch {
-    }
-    if (bytes == null) {
-      const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
-      const r = read(g);
-      status = status ?? r.status;
-      ct = ct ?? r.ct;
-      if (r.cr && r.cr.includes("/")) {
-        const tail = r.cr.split("/")[1];
-        if (tail && tail !== "*") bytes = Number(tail);
-      } else if (r.len) bytes = Number(r.len);
-      try {
-        await g.body?.cancel();
-      } catch {
-      }
-    }
-    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
-  } catch (e) {
-    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
-  } finally {
-    clearTimeout(t);
-  }
-}
-async function pool(items, n, fn) {
-  const out = new Array(items.length);
-  let i = 0;
-  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
-    while (i < items.length) {
-      const idx = i++;
-      out[idx] = await fn(items[idx]);
-    }
-  }));
-  return out;
-}
-async function auditImages(pages, opts = {}) {
-  const concurrency = opts.concurrency ?? 12;
-  const timeoutMs = opts.timeoutMs ?? 12e3;
-  const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
-  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
-  const rows = heads.map((r) => {
-    const format = formatOf(r.contentType, r.url);
-    return {
-      ...r,
-      size: formatBytes(r.bytes),
-      format,
-      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
-      legacyFormat: format !== "unknown" && !MODERN.has(format)
-    };
-  });
-  const sized = rows.filter((r) => r.bytes != null);
-  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
-  const formatCounts = {};
-  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
-  return {
-    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
-    summary: {
-      unique: rows.length,
-      sized: sized.length,
-      totalBytes,
-      totalSize: formatBytes(totalBytes) ?? "0 B",
-      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
-      over100kb: rows.filter((r) => r.over100kb).length,
-      legacyFormat: rows.filter((r) => r.legacyFormat).length,
-      formatCounts
-    }
-  };
-}
-function renderImageSection(audit) {
-  const s = audit.summary;
-  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
-  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
-  const lines = [
-    `## Images`,
-    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
-    `Formats: ${fmt}`
-  ];
-  if (heaviest.length) {
-    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
-    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
-  }
-  return lines.join("\n");
-}
-var OVER_BYTES, MODERN, decodeEntities, dedupKey, formatOf;
-var init_image_audit = __esm({
-  "src/api/image-audit.ts"() {
-    "use strict";
-    OVER_BYTES = 100 * 1024;
-    MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
-    decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
-    dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
-    formatOf = (ct, url) => {
-      if (ct) {
-        const m = ct.split(";")[0].trim().toLowerCase();
-        if (m.startsWith("image/")) return m.replace("image/", "");
-      }
-      return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
-    };
-  }
-});
-
 // src/api/site-extract-repository.ts
 var site_extract_repository_exports = {};
 __export(site_extract_repository_exports, {
@@ -10912,6 +10955,7 @@ ${renderImageSection(extras.imageAudit)}`;
       (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
       (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
     }
+    if (extras.seoAudit) (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
     if (extras.branding) (0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
     const artifactFiles = [
       { name: "report.md", type: "text/markdown" },
@@ -10927,6 +10971,7 @@ ${renderImageSection(extras.imageAudit)}`;
       artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
       artifactFiles.push({ name: "images-summary.json", type: "application/json" });
     }
+    if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
     if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
     const zip = new import_yazl.ZipFile();
     const zipOut = (0, import_node_fs3.createWriteStream)((0, import_node_path3.join)(dir, "bundle.zip"));
@@ -10974,6 +11019,7 @@ var init_site_extract = __esm({
     "use strict";
     init_client();
     init_site_extractor();
+    init_seo_audit();
     init_seo_link_graph();
     init_seo_link_report();
     init_seo_issues();
@@ -11012,12 +11058,13 @@ var init_site_extract = __esm({
         const maxPages = Number(job.options.maxPages ?? 1e4);
         const concurrency = Number(job.options.concurrency ?? 1);
         const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
-        const seed = await step.run("discover", async () => {
-          const discovered = await discoverUrls(job.startUrl, maxPages, key);
-          const out = discovered.length ? discovered : [job.startUrl];
-          await setExtractJobTotal(jobId, out.length);
-          return out;
+        const seedRaw = await step.run("discover", async () => {
+          const discovered = await discoverUrlsWithSitemap(job.startUrl, maxPages, key);
+          const urls = discovered.urls.length ? discovered.urls : [job.startUrl];
+          await setExtractJobTotal(jobId, urls.length);
+          return { urls, sitemapUrls: discovered.sitemapUrls };
         });
+        const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [] } : seedRaw;
         const siteHost = (() => {
           try {
             return new URL(job.startUrl).hostname.replace(/^www\./, "");
@@ -11041,8 +11088,9 @@ var init_site_extract = __esm({
             return false;
           }
         };
-        const queue = [...seed];
-        const seen = new Set(seed.map(norm));
+        const sitemapSet = seed.sitemapUrls.length > 0 ? new Set(seed.sitemapUrls.map(norm)) : null;
+        const queue = [...seed.urls];
+        const seen = new Set(seed.urls.map(norm));
         let crawled = 0;
         const MAX_BATCHES = Math.ceil(maxPages / BATCH) + 30;
         for (let i = 0; i < MAX_BATCHES && crawled < maxPages && crawled < queue.length; i++) {
@@ -11050,7 +11098,7 @@ var init_site_extract = __esm({
           if (!batch.length) break;
           const newLinks = await step.run(`crawl-batch-${i}`, async () => {
             const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
-            await saveExtractPages(jobId, pages);
+            await saveExtractPages(jobId, pages.map((p) => ({ ...p, inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null })));
             const found = [];
             for (const p of pages) for (const l of p.outlinks ?? []) {
               if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
@@ -11074,9 +11122,13 @@ var init_site_extract = __esm({
           const pages = await getExtractedPages(jobId);
           return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
         }) : null;
+        const seoAudit = Array.isArray(job.options.formats) && job.options.formats.includes("issues") ? await step.run("seo-audit", async () => {
+          const pages = await getExtractedPages(jobId);
+          return buildSeoAudit(pages, job.startUrl);
+        }) : null;
         const artifacts = await step.run("finalize", async () => {
           const { assembleExtractArtifacts: assembleExtractArtifacts2 } = await Promise.resolve().then(() => (init_extract_bundle(), extract_bundle_exports));
-          const stored = await assembleExtractArtifacts2(job, { branding, imageAudit });
+          const stored = await assembleExtractArtifacts2(job, { branding, imageAudit, seoAudit });
           await completeExtractJob(jobId, stored);
           return stored;
         });
@@ -13963,7 +14015,7 @@ var init_server_schemas = __esm({
       rotateProxies: import_zod13.z.boolean().optional(),
       rotateProxyEvery: import_zod13.z.number().int().min(1).max(100).optional(),
       background: import_zod13.z.boolean().optional(),
-      formats: import_zod13.z.array(import_zod13.z.enum(["markdown", "links", "json", "images", "branding"])).optional()
+      formats: import_zod13.z.array(import_zod13.z.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
     });
     YoutubeHarvestBodySchema = import_zod13.z.object({
       mode: import_zod13.z.enum(["search", "channel"]),
@@ -27400,7 +27452,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.7.0";
+    PACKAGE_VERSION = "0.9.0";
   }
 });
 
@@ -27573,7 +27625,7 @@ var init_output_schema_registry = __esm({
 });
 
 // src/mcp/mcp-tool-schemas.ts
-var import_zod33, HarvestPaaInputSchema, ExtractUrlInputSchema, DiffPageInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema;
+var import_zod33, HarvestPaaInputSchema, ExtractUrlInputSchema, DiffPageInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, AuditSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, RedditThreadInputSchema, VideoFrameAnalysisInputSchema, VideoFrameAnalysisStatusInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, GoogleAdsSearchInputSchema, GoogleAdsPageIntelInputSchema, GoogleAdsTranscribeInputSchema, InstagramProfileContentInputSchema, InstagramMediaDownloadInputSchema, MapsPlaceIntelInputSchema, TrustpilotReviewsInputSchema, G2ReviewsInputSchema, ReviewCardSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, ArtifactPointerOutputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, DiffPageOutputSchema, ExtractSiteOutputSchema, AuditSiteOutputSchema, MapsPlaceIntelOutputSchema, TrustpilotReviewsOutputSchema, G2ReviewsOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, VideoFrameAnalysisOutputSchema, VideoFrameAnalysisStatusOutputSchema, RedditThreadOutputSchema, FacebookPageIntelOutputSchema, GoogleAdsSearchOutputSchema, GoogleAdsPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, InstagramBrowserOutput, InstagramPaginationOutput, InstagramProfileContentOutputSchema, InstagramMediaTrackOutput, InstagramDownloadOutput, InstagramMediaDownloadOutputSchema, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, GoogleAdsTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema, ReportArtifactReadInputSchema, ReportArtifactReadOutputSchema, ListServiceConnectionsInputSchema, ListServiceConnectionsOutputSchema, SlackSendMessageInputSchema, SlackSendMessageOutputSchema, GmailSendMessageInputSchema, GmailSendMessageOutputSchema, GoogleCalendarCreateEventInputSchema, GoogleCalendarCreateEventOutputSchema;
 var init_mcp_tool_schemas = __esm({
   "src/mcp/mcp-tool-schemas.ts"() {
     "use strict";
@@ -28598,6 +28650,52 @@ var init_mcp_tool_schemas = __esm({
       totalBytes: import_zod33.z.number().int().min(0),
       nextOffset: import_zod33.z.number().int().min(0).nullable()
     };
+    ListServiceConnectionsInputSchema = {};
+    ListServiceConnectionsOutputSchema = {
+      connections: import_zod33.z.array(import_zod33.z.object({
+        connectionId: import_zod33.z.string(),
+        providerConfigKey: import_zod33.z.string(),
+        label: import_zod33.z.string(),
+        status: import_zod33.z.string(),
+        actionsEnabled: import_zod33.z.boolean()
+      }))
+    };
+    SlackSendMessageInputSchema = {
+      connectionId: import_zod33.z.string().min(1).describe("A Slack connectionId from list_service_connections, with actionsEnabled true."),
+      channel: import_zod33.z.string().min(1).describe(`Slack channel ID to send to, e.g. "C1234567890". Get this from the connection's own read tools, not guessed.`),
+      text: import_zod33.z.string().min(1).max(4e3).describe("Message text to send.")
+    };
+    SlackSendMessageOutputSchema = {
+      ok: import_zod33.z.boolean(),
+      result: import_zod33.z.unknown().optional(),
+      error: NullableString
+    };
+    GmailSendMessageInputSchema = {
+      connectionId: import_zod33.z.string().min(1).describe("A Gmail connectionId from list_service_connections, with actionsEnabled true."),
+      to: import_zod33.z.string().email().describe("Recipient email address."),
+      subject: import_zod33.z.string().min(1).max(500).describe("Email subject line."),
+      body: import_zod33.z.string().min(1).max(5e4).describe("Plain-text email body.")
+    };
+    GmailSendMessageOutputSchema = {
+      ok: import_zod33.z.boolean(),
+      result: import_zod33.z.unknown().optional(),
+      error: NullableString
+    };
+    GoogleCalendarCreateEventInputSchema = {
+      connectionId: import_zod33.z.string().min(1).describe("A Google Calendar connectionId from list_service_connections, with actionsEnabled true."),
+      calendarId: import_zod33.z.string().min(1).default("primary").describe('Calendar to create the event in. Default "primary".'),
+      summary: import_zod33.z.string().min(1).max(500).describe("Event title."),
+      description: import_zod33.z.string().max(5e3).optional().describe("Event description."),
+      location: import_zod33.z.string().max(500).optional().describe("Event location."),
+      startDateTime: import_zod33.z.string().min(1).describe('Start time, ISO 8601, e.g. "2026-07-15T09:00:00-06:00".'),
+      endDateTime: import_zod33.z.string().min(1).describe('End time, ISO 8601, e.g. "2026-07-15T10:00:00-06:00".'),
+      timeZone: import_zod33.z.string().max(100).optional().describe('IANA timezone, e.g. "America/Denver". Applies to both start and end.')
+    };
+    GoogleCalendarCreateEventOutputSchema = {
+      ok: import_zod33.z.boolean(),
+      result: import_zod33.z.unknown().optional(),
+      error: NullableString
+    };
   }
 });
 
@@ -29277,6 +29375,34 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
       openWorldHint: false
     }
   }, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
+  server.registerTool("list_service_connections", {
+    title: "List Connected Services",
+    description: "List the third-party service connections (Slack, Gmail, Google Calendar) the account has authorized, with each connection's id and whether it is enabled for live actions. Get a connectionId from here before calling slack_send_message, gmail_send_message, or google_calendar_create_event.",
+    inputSchema: ListServiceConnectionsInputSchema,
+    outputSchema: recordOutputSchema("list_service_connections", ListServiceConnectionsOutputSchema),
+    annotations: { title: "List Connected Services", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
+  }, async (input) => executor.listServiceConnections(input));
+  server.registerTool("slack_send_message", {
+    title: "Send Slack Message",
+    description: "Send a message to a Slack channel through a connected, action-enabled Slack connection. Requires a connectionId from list_service_connections with actionsEnabled true; the person must have explicitly turned actions on for that connection.",
+    inputSchema: SlackSendMessageInputSchema,
+    outputSchema: recordOutputSchema("slack_send_message", SlackSendMessageOutputSchema),
+    annotations: { title: "Send Slack Message", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
+  }, async (input) => executor.slackSendMessage(input));
+  server.registerTool("gmail_send_message", {
+    title: "Send Gmail Message",
+    description: "Send an email through a connected, action-enabled Gmail connection. Requires a connectionId from list_service_connections with actionsEnabled true; the person must have explicitly turned actions on for that connection.",
+    inputSchema: GmailSendMessageInputSchema,
+    outputSchema: recordOutputSchema("gmail_send_message", GmailSendMessageOutputSchema),
+    annotations: { title: "Send Gmail Message", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
+  }, async (input) => executor.gmailSendMessage(input));
+  server.registerTool("google_calendar_create_event", {
+    title: "Create Calendar Event",
+    description: "Create an event on a connected, action-enabled Google Calendar connection. Requires a connectionId from list_service_connections with actionsEnabled true; the person must have explicitly turned actions on for that connection.",
+    inputSchema: GoogleCalendarCreateEventInputSchema,
+    outputSchema: recordOutputSchema("google_calendar_create_event", GoogleCalendarCreateEventOutputSchema),
+    annotations: { title: "Create Calendar Event", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
+  }, async (input) => executor.googleCalendarCreateEvent(input));
 }
 var import_mcp, import_node_fs9, import_node_path12, import_node_crypto10;
 var init_paa_mcp_server = __esm({
@@ -29585,6 +29711,18 @@ var init_http_mcp_tool_executor = __esm({
       creditsInfo(input) {
         return this.call("/billing/credits", input);
       }
+      listServiceConnections(input) {
+        return this.getJson("/schedule-connections");
+      }
+      slackSendMessage(input) {
+        return this.call("/schedule-connections/actions/slack/send-message", input);
+      }
+      gmailSendMessage(input) {
+        return this.call("/schedule-connections/actions/gmail/send-message", input);
+      }
+      googleCalendarCreateEvent(input) {
+        return this.call("/schedule-connections/actions/google-calendar/create-event", input);
+      }
       captureSerpSnapshot(input) {
         return this.call("/serp-intelligence/capture", input, this.serpIntelligenceTimeoutMs);
       }
@@ -38653,6 +38791,312 @@ var init_memory_db = __esm({
   }
 });
 
+// src/api/nango-control.ts
+function controlBaseUrl() {
+  return (process.env.NANGO_CONTROL_URL?.trim() || DEFAULT_NANGO_CONTROL_URL).replace(/\/$/, "");
+}
+function controlSecret() {
+  const secret2 = process.env.SCHEDULE_INTEGRATIONS_SECRET?.trim();
+  if (!secret2) throw new NangoControlError("Scheduled service connections are not configured.", 503);
+  return secret2;
+}
+function isRecord(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function unwrapData(value) {
+  if (isRecord(value) && isRecord(value.data)) return value.data;
+  return value;
+}
+function cleanString(value, max = 300) {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed ? trimmed.slice(0, max) : null;
+}
+function firstString(record, keys, max = 300) {
+  for (const key of keys) {
+    const value = cleanString(record[key], max);
+    if (value) return value;
+  }
+  return null;
+}
+function cleanTools(value) {
+  if (!Array.isArray(value)) return [];
+  const tools = value.map((tool) => cleanString(tool, 200)).filter((tool) => !!tool);
+  return [...new Set(tools)].slice(0, 200);
+}
+function cleanStringArray(value, maxItems = 20, maxLength = 100) {
+  if (!Array.isArray(value)) return [];
+  const items = value.map((item) => cleanString(item, maxLength)).filter((item) => !!item);
+  return [...new Set(items)].slice(0, maxItems);
+}
+function cleanHttpsUrl(value) {
+  const candidate = cleanString(value, 2e3);
+  if (!candidate) return null;
+  try {
+    const url = new URL(candidate);
+    return url.protocol === "https:" ? url.toString() : null;
+  } catch {
+    return null;
+  }
+}
+function arrayFromPayload(value, keys) {
+  const unwrapped = unwrapData(value);
+  if (Array.isArray(unwrapped)) return unwrapped;
+  if (!isRecord(unwrapped)) return [];
+  for (const key of keys) {
+    if (Array.isArray(unwrapped[key])) return unwrapped[key];
+  }
+  return [];
+}
+async function controlRequest(path6, init) {
+  const response = await fetch(`${controlBaseUrl()}${path6}`, {
+    ...init,
+    headers: {
+      accept: "application/json",
+      authorization: `Bearer ${controlSecret()}`,
+      ...init?.body ? { "content-type": "application/json" } : {},
+      ...init?.headers ?? {}
+    },
+    signal: AbortSignal.timeout(2e4)
+  }).catch((err) => {
+    throw new NangoControlError(`Scheduled service connection control is unavailable: ${err instanceof Error ? err.message : "network error"}`);
+  });
+  const text = await response.text();
+  let body = {};
+  try {
+    body = text ? JSON.parse(text) : {};
+  } catch {
+    body = {};
+  }
+  if (!response.ok) {
+    const upstream = isRecord(body) ? cleanString(body.error, 300) : null;
+    throw new NangoControlError(upstream || `Scheduled service connection control failed (${response.status}).`);
+  }
+  return body;
+}
+function sanitizeScheduleConnectionSelections(value) {
+  if (value == null) return [];
+  if (!Array.isArray(value)) throw new ScheduleConnectionValidationError("connections must be an array.");
+  if (value.length > 20) throw new ScheduleConnectionValidationError("A scheduled action can use at most 20 service connections.");
+  const selections = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const item of value) {
+    if (!isRecord(item)) throw new ScheduleConnectionValidationError("Each service connection must be an object.");
+    const connectionId = firstString(item, ["connectionId", "connection_id"]);
+    const providerConfigKey = firstString(item, ["providerConfigKey", "provider_config_key", "integrationId", "integration_id"]);
+    const allowedTools = cleanTools(item.allowedTools ?? item.allowed_tools ?? item.allowedCapabilities ?? item.allowed_capabilities);
+    if (!connectionId || !providerConfigKey) throw new ScheduleConnectionValidationError("Each service connection needs a connectionId and providerConfigKey.");
+    if (allowedTools.length === 0) throw new ScheduleConnectionValidationError("Each selected service needs at least one allowed tool.");
+    const key = `${providerConfigKey}:${connectionId}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    selections.push({ connectionId, providerConfigKey, allowedTools });
+  }
+  return selections;
+}
+async function getNangoCatalog() {
+  const body = await controlRequest("/api/internal/nango/catalog");
+  const rows = arrayFromPayload(body, ["providers", "catalog", "integrations", "services"]);
+  const result = [];
+  for (const row of rows) {
+    if (!isRecord(row)) continue;
+    const providerConfigKey = firstString(row, ["providerConfigKey", "provider_config_key", "integrationId", "integration_id", "id"]);
+    if (!providerConfigKey) continue;
+    const provider = firstString(row, ["provider"]);
+    const label = firstString(row, ["label", "displayName", "display_name", "name"]) || providerConfigKey;
+    const description = firstString(row, ["description"], 500);
+    const logoUrl = cleanHttpsUrl(row.logoUrl ?? row.logo_url ?? row.logo);
+    const docsUrl = cleanHttpsUrl(row.docsUrl ?? row.docs_url ?? row.docs);
+    const authMode = firstString(row, ["authMode", "auth_mode"], 100);
+    const categories = cleanStringArray(row.categories);
+    const disabledTools = DISABLED_NANGO_TOOLS[providerConfigKey];
+    const safeDefaultAllowedTools = cleanTools(
+      row.safeDefaultAllowedTools ?? row.safe_default_allowed_tools ?? row.defaultAllowedTools ?? row.default_allowed_tools ?? row.allowedTools ?? row.allowed_tools
+    ).filter((tool) => !disabledTools?.has(tool));
+    result.push({
+      providerConfigKey,
+      provider,
+      label,
+      description,
+      logoUrl,
+      docsUrl,
+      authMode,
+      categories,
+      safeDefaultAllowedTools
+    });
+  }
+  return result;
+}
+async function getNangoConnections(identity) {
+  const query = new URLSearchParams({ identity }).toString();
+  const body = await controlRequest(`/api/internal/nango/connections?${query}`);
+  const rows = arrayFromPayload(body, ["connections"]);
+  const result = [];
+  for (const row of rows) {
+    if (!isRecord(row)) continue;
+    const connectionId = firstString(row, ["connectionId", "connection_id", "id"]);
+    const providerConfigKey = firstString(row, ["providerConfigKey", "provider_config_key", "integrationId", "integration_id", "provider"]);
+    if (!connectionId || !providerConfigKey) continue;
+    const provider = firstString(row, ["provider"]);
+    const label = firstString(row, ["label", "accountLabel", "account_label", "displayName", "display_name", "endUserEmail", "end_user_email"]) || providerConfigKey;
+    const rawStatus = firstString(row, ["status"], 64) || "connected";
+    const reconnectRequired = row.reconnectRequired === true || row.reconnect_required === true || rawStatus === "needs_reauth" || rawStatus === "reauth_required" || rawStatus === "invalid" || rawStatus === "error" || rawStatus === "revoked" || rawStatus === "disabled";
+    result.push({
+      connectionId,
+      providerConfigKey,
+      provider,
+      label,
+      status: reconnectRequired ? "needs_reauth" : "connected",
+      reconnectRequired,
+      actionsEnabled: row.actionsEnabled === true || row.actions_enabled === true,
+      createdAt: firstString(row, ["createdAt", "created_at"], 100),
+      updatedAt: firstString(row, ["updatedAt", "updated_at"], 100)
+    });
+  }
+  return result;
+}
+function sanitizeConnectSession(body) {
+  const data = unwrapData(body);
+  if (!isRecord(data)) throw new NangoControlError("The connection service returned an invalid connect session.");
+  const connectLink = firstString(data, ["connectLink", "connect_link"], 2e3);
+  if (!connectLink) throw new NangoControlError("The connection service did not return a connect link.");
+  return {
+    connectLink,
+    sessionToken: firstString(data, ["sessionToken", "session_token", "token"], 2e3),
+    expiresAt: firstString(data, ["expiresAt", "expires_at"], 100)
+  };
+}
+async function createNangoConnectSession(identity, providerConfigKey) {
+  const body = await controlRequest("/api/internal/nango/connect-session", {
+    method: "POST",
+    body: JSON.stringify({ identity, email: identity, provider: providerConfigKey })
+  });
+  return sanitizeConnectSession(body);
+}
+async function createNangoReconnectSession(identity, connectionId) {
+  const body = await controlRequest("/api/internal/nango/reconnect-session", {
+    method: "POST",
+    body: JSON.stringify({ identity, connectionId, email: identity })
+  });
+  return sanitizeConnectSession(body);
+}
+function sanitizeBindingRows(body, defaultScheduleActionId) {
+  const data = unwrapData(body);
+  const flattened = [];
+  if (isRecord(data) && isRecord(data.bindings) && !Array.isArray(data.bindings)) {
+    for (const [scheduleActionId, value] of Object.entries(data.bindings)) {
+      if (Array.isArray(value)) value.forEach((row) => flattened.push({ row, scheduleActionId }));
+    }
+  } else {
+    const rows = arrayFromPayload(data, ["bindings", "connections"]);
+    for (const row of rows) {
+      if (isRecord(row) && Array.isArray(row.connections)) {
+        const scheduleActionId = firstString(row, ["scheduleActionId", "schedule_action_id", "scheduleId", "schedule_id"]) || defaultScheduleActionId;
+        row.connections.forEach((connection) => flattened.push({ row: connection, scheduleActionId }));
+      } else {
+        flattened.push({ row, scheduleActionId: defaultScheduleActionId });
+      }
+    }
+  }
+  const result = [];
+  for (const item of flattened) {
+    if (!isRecord(item.row)) continue;
+    const connectionId = firstString(item.row, ["connectionId", "connection_id"]);
+    const providerConfigKey = firstString(item.row, ["providerConfigKey", "provider_config_key", "integrationId", "integration_id", "provider"]);
+    if (!connectionId || !providerConfigKey) continue;
+    result.push({
+      scheduleActionId: firstString(item.row, ["scheduleActionId", "schedule_action_id", "scheduleId", "schedule_id"]) || item.scheduleActionId,
+      connectionId,
+      providerConfigKey,
+      allowedTools: cleanTools(item.row.allowedTools ?? item.row.allowed_tools ?? item.row.allowedCapabilities ?? item.row.allowed_capabilities),
+      label: firstString(item.row, ["label", "accountLabel", "account_label", "displayName", "display_name"])
+    });
+  }
+  return result;
+}
+async function getScheduleConnectionBindings(identity, scheduleActionId) {
+  const query = new URLSearchParams({ identity, scheduleId: scheduleActionId });
+  const body = await controlRequest(`/api/internal/nango/bindings?${query.toString()}`);
+  return sanitizeBindingRows(body, scheduleActionId);
+}
+async function putScheduleConnectionBindings(identity, scheduleActionId, connections) {
+  const body = await controlRequest("/api/internal/nango/bindings", {
+    method: "PUT",
+    body: JSON.stringify({
+      identity,
+      scheduleId: scheduleActionId,
+      connections: connections.map((connection) => ({
+        connectionId: connection.connectionId,
+        allowedTools: connection.allowedTools
+      }))
+    })
+  });
+  const sanitized = sanitizeBindingRows(body, scheduleActionId);
+  const expectedByConnectionId = new Map(connections.map((connection) => [
+    connection.connectionId,
+    {
+      providerConfigKey: connection.providerConfigKey,
+      allowedTools: [...connection.allowedTools].sort()
+    }
+  ]));
+  const responseMatchesRequest = sanitized.length === connections.length && sanitized.every((binding) => {
+    const expected = expectedByConnectionId.get(binding.connectionId);
+    return !!expected && binding.scheduleActionId === scheduleActionId && binding.providerConfigKey === expected.providerConfigKey && [...binding.allowedTools].sort().join("\0") === expected.allowedTools.join("\0");
+  });
+  if (!responseMatchesRequest) {
+    throw new NangoControlError("Scheduled service connection control returned an invalid binding response.");
+  }
+  return sanitized;
+}
+async function deleteScheduleConnectionBindings(identity, scheduleActionId) {
+  await controlRequest("/api/internal/nango/bindings", {
+    method: "DELETE",
+    body: JSON.stringify({ identity, scheduleId: scheduleActionId })
+  });
+}
+async function setScheduleConnectionActionsEnabled(identity, connectionId, enabled) {
+  const body = await controlRequest("/api/internal/nango/connections/actions/enable", {
+    method: "POST",
+    body: JSON.stringify({ identity, connectionId, enabled })
+  });
+  const data = unwrapData(body);
+  if (!isRecord(data) || !isRecord(data.connection)) return enabled;
+  return data.connection.actionsEnabled === true;
+}
+async function callScheduleConnectionAction(identity, connectionId, input) {
+  const body = await controlRequest("/api/internal/nango/connections/actions/call", {
+    method: "POST",
+    body: JSON.stringify({ identity, connectionId, input })
+  });
+  const data = unwrapData(body);
+  return isRecord(data) ? data.result ?? data : data;
+}
+var DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, NangoControlError, ScheduleConnectionValidationError;
+var init_nango_control = __esm({
+  "src/api/nango-control.ts"() {
+    "use strict";
+    DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
+    DISABLED_NANGO_TOOLS = {
+      "google-mail": /* @__PURE__ */ new Set(["list-filters"]),
+      slack: /* @__PURE__ */ new Set(["search-files", "search-messages"])
+    };
+    NangoControlError = class extends Error {
+      status;
+      constructor(message, status = 502) {
+        super(message);
+        this.name = "NangoControlError";
+        this.status = status;
+      }
+    };
+    ScheduleConnectionValidationError = class extends Error {
+      constructor(message) {
+        super(message);
+        this.name = "ScheduleConnectionValidationError";
+      }
+    };
+  }
+});
+
 // src/api/webhook.ts
 async function deliverWebhook(url, payload, retries = 3) {
   for (let attempt = 1; attempt <= retries; attempt++) {
@@ -38839,6 +39283,7 @@ async function createSignupStripeCustomer(email) {
 }
 function opForCostPath(p) {
   if (p === "/harvest" || p === "/harvest/sync") return "harvest";
+  if (p === "/diff-page") return "diff_page";
   if (p === "/extract-url") return "page_scrape";
   if (p === "/extract-site") return "extract_site";
   if (p === "/map-urls") return "url_map";
@@ -38877,6 +39322,20 @@ function maybeProvisionInBackground(user, vaultCount) {
     }
   })();
 }
+function scheduleConnectionError(c, err, fallback) {
+  if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
+  if (err instanceof NangoControlError) return c.json({ ok: false, error: err.message }, err.status);
+  console.error("[schedule-connections]", err instanceof Error ? err.message : String(err));
+  return c.json({ ok: false, error: fallback }, 502);
+}
+function providerConfigKeyFrom(value) {
+  if (typeof value !== "string") return null;
+  const key = value.trim();
+  return key && key.length <= 200 ? key : null;
+}
+function bindingsForSchedule(bindings, scheduleActionId) {
+  return bindings.filter((binding) => binding.scheduleActionId === scheduleActionId);
+}
 function combineAbortSignals(signals) {
   const controller = new AbortController();
   const abortFrom = (signal) => {
@@ -38984,6 +39443,7 @@ var init_server = __esm({
     init_session();
     init_memory();
     init_memory_db();
+    init_nango_control();
     init_concurrency_gates();
     secureCookies2 = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
     isProduction2 = secureCookies2;
@@ -39182,6 +39642,13 @@ var init_server = __esm({
       c.header("Cache-Control", "public, max-age=60");
       return c.json(CATALOG);
     });
+    app.get("/rates", (c) => {
+      const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => ({
+        ...cost,
+        ...cost.notes ? { notes: cost.notes.replace(/ via fal\.ai/g, "") } : {}
+      }));
+      return c.json({ creditsPerDollarStarter: Math.round(1 / 15e-5), costs });
+    });
     app.get("/me", async (c) => {
       const token = (0, import_cookie2.getCookie)(c, "session");
       if (!token) return c.json({ authenticated: false });
@@ -39509,6 +39976,103 @@ var init_server = __esm({
         return c.json({ error: message }, 500);
       }
     });
+    app.get("/schedule-connection-catalog", auth2, async (c) => {
+      try {
+        return c.json({ ok: true, catalog: await getNangoCatalog() });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to load the service connection catalog.");
+      }
+    });
+    app.get("/schedule-connections", auth2, async (c) => {
+      try {
+        const user = c.get("user");
+        return c.json({ ok: true, connections: await getNangoConnections(user.email) });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to load service connections.");
+      }
+    });
+    app.post("/schedule-connections/session", auth2, async (c) => {
+      const user = c.get("user");
+      const body = await c.req.json().catch(() => ({}));
+      const providerConfigKey = providerConfigKeyFrom(body.providerConfigKey);
+      if (!providerConfigKey) return c.json({ ok: false, error: "providerConfigKey is required." }, 400);
+      try {
+        const session = await createNangoConnectSession(user.email, providerConfigKey);
+        return c.json({ ok: true, ...session });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to start the service connection flow.");
+      }
+    });
+    app.post("/schedule-connections/:id/reconnect-session", auth2, async (c) => {
+      const user = c.get("user");
+      try {
+        const session = await createNangoReconnectSession(user.email, c.req.param("id"));
+        return c.json({ ok: true, ...session });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to start the service reconnection flow.");
+      }
+    });
+    app.post("/schedule-connections/:id/enable-actions", auth2, async (c) => {
+      const user = c.get("user");
+      const body = await c.req.json().catch(() => ({}));
+      if (typeof body.enabled !== "boolean") return c.json({ ok: false, error: "enabled must be a boolean." }, 400);
+      try {
+        const actionsEnabled = await setScheduleConnectionActionsEnabled(user.email, c.req.param("id"), body.enabled);
+        return c.json({ ok: true, actionsEnabled });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to update this connection's action setting.");
+      }
+    });
+    app.post("/schedule-connections/actions/slack/send-message", auth2, async (c) => {
+      const user = c.get("user");
+      const body = await c.req.json().catch(() => ({}));
+      const connectionId = providerConfigKeyFrom(body.connectionId);
+      if (!connectionId || typeof body.channel !== "string" || typeof body.text !== "string") {
+        return c.json({ ok: false, error: "connectionId, channel, and text are required." }, 400);
+      }
+      try {
+        const result = await callScheduleConnectionAction(user.email, connectionId, { channel: body.channel, text: body.text });
+        return c.json({ ok: true, result });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to send the Slack message.");
+      }
+    });
+    app.post("/schedule-connections/actions/gmail/send-message", auth2, async (c) => {
+      const user = c.get("user");
+      const body = await c.req.json().catch(() => ({}));
+      const connectionId = providerConfigKeyFrom(body.connectionId);
+      if (!connectionId || typeof body.to !== "string" || typeof body.subject !== "string" || typeof body.body !== "string") {
+        return c.json({ ok: false, error: "connectionId, to, subject, and body are required." }, 400);
+      }
+      try {
+        const result = await callScheduleConnectionAction(user.email, connectionId, { to: body.to, subject: body.subject, body: body.body });
+        return c.json({ ok: true, result });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to send the email.");
+      }
+    });
+    app.post("/schedule-connections/actions/google-calendar/create-event", auth2, async (c) => {
+      const user = c.get("user");
+      const body = await c.req.json().catch(() => ({}));
+      const connectionId = providerConfigKeyFrom(body.connectionId);
+      if (!connectionId || typeof body.summary !== "string" || typeof body.startDateTime !== "string" || typeof body.endDateTime !== "string") {
+        return c.json({ ok: false, error: "connectionId, summary, startDateTime, and endDateTime are required." }, 400);
+      }
+      const timeZone = typeof body.timeZone === "string" ? body.timeZone : void 0;
+      try {
+        const result = await callScheduleConnectionAction(user.email, connectionId, {
+          calendarId: typeof body.calendarId === "string" ? body.calendarId : "primary",
+          summary: body.summary,
+          description: typeof body.description === "string" ? body.description : void 0,
+          location: typeof body.location === "string" ? body.location : void 0,
+          start: { dateTime: body.startDateTime, timeZone },
+          end: { dateTime: body.endDateTime, timeZone }
+        });
+        return c.json({ ok: true, result });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to create the calendar event.");
+      }
+    });
     app.post("/schedule-link", auth2, async (c) => {
       try {
         const user = c.get("user");
@@ -39527,15 +40091,58 @@ var init_server = __esm({
       const { key, error } = await getOrCreateUserMemoryKey(user);
       if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
       const r = await memoryCall("listScheduledActionsTool", {}, key);
-      return c.json(r);
+      if (!r.ok || !Array.isArray(r.actions)) return c.json(r);
+      try {
+        const actions = await Promise.all(r.actions.map(async (action) => {
+          const id = typeof action.id === "string" ? action.id : "";
+          if (!id) return { ...action, connections: [] };
+          const bindings = await getScheduleConnectionBindings(user.email, id);
+          return { ...action, connections: bindingsForSchedule(bindings, id) };
+        }));
+        return c.json({
+          ...r,
+          actions
+        });
+      } catch (err) {
+        console.warn("[schedule-actions] connection bindings unavailable:", err instanceof Error ? err.message : String(err));
+        return c.json({
+          ...r,
+          actions: r.actions.map((action) => ({ ...action, connections: [] })),
+          connectionBindingsUnavailable: true
+        });
+      }
     });
     app.post("/schedule-actions", auth2, async (c) => {
       const user = c.get("user");
       const { key, error } = await getOrCreateUserMemoryKey(user);
       if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
       const body = await c.req.json().catch(() => ({}));
-      const r = await memoryCall("createScheduledActionTool", body, key);
-      return c.json(r);
+      const hasConnections = Object.prototype.hasOwnProperty.call(body, "connections");
+      let connections;
+      try {
+        connections = sanitizeScheduleConnectionSelections(body.connections);
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Invalid service connections.");
+      }
+      const actionBody = { ...body };
+      delete actionBody.connections;
+      const r = await memoryCall("createScheduledActionTool", actionBody, key);
+      if (!r.ok || !hasConnections) return c.json(r);
+      if (connections.length === 0) return c.json({ ...r, connections: [] });
+      if (!r.id) {
+        console.error("[schedule-actions] create returned no id while service connections were requested");
+        return c.json({ ok: false, error: "The scheduled action was created without a usable id; service connections were not attached." }, 502);
+      }
+      try {
+        const bound = await putScheduleConnectionBindings(user.email, r.id, connections);
+        return c.json({ ...r, connections: bound });
+      } catch (err) {
+        const rollback = await memoryCall("deleteScheduledActionTool", { id: r.id }, key).catch(() => ({ ok: false }));
+        if (!rollback.ok) console.error("[schedule-actions] failed to compensate action after connection binding failure", r.id);
+        const response = scheduleConnectionError(c, err, "The scheduled action was not created because its service connections could not be attached.");
+        response.headers.set("x-schedule-create-rolled-back", rollback.ok ? "true" : "false");
+        return response;
+      }
     });
     app.post("/schedule-actions/propose", auth2, async (c) => {
       const user = c.get("user");
@@ -39545,6 +40152,33 @@ var init_server = __esm({
       const r = await memoryCall("proposeScheduledActionTool", body, key);
       return c.json(r);
     });
+    app.get("/schedule-actions/:id/connections", auth2, async (c) => {
+      try {
+        const user = c.get("user");
+        const scheduleActionId = c.req.param("id");
+        const bindings = await getScheduleConnectionBindings(user.email, scheduleActionId);
+        return c.json({ ok: true, connections: bindingsForSchedule(bindings, scheduleActionId) });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to load scheduled action service connections.");
+      }
+    });
+    app.put("/schedule-actions/:id/connections", auth2, async (c) => {
+      const user = c.get("user");
+      const body = await c.req.json().catch(() => ({}));
+      let connections;
+      try {
+        connections = sanitizeScheduleConnectionSelections(body.connections);
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Invalid service connections.");
+      }
+      try {
+        const scheduleActionId = c.req.param("id");
+        const bindings = await putScheduleConnectionBindings(user.email, scheduleActionId, connections);
+        return c.json({ ok: true, connections: bindings });
+      } catch (err) {
+        return scheduleConnectionError(c, err, "Unable to update scheduled action service connections.");
+      }
+    });
     app.post("/schedule-actions/:id/pause", auth2, async (c) => {
       const user = c.get("user");
       const { key, error } = await getOrCreateUserMemoryKey(user);
@@ -39563,8 +40197,16 @@ var init_server = __esm({
       const user = c.get("user");
       const { key, error } = await getOrCreateUserMemoryKey(user);
       if (!key) return c.json({ ok: false, error: error ?? "memory unavailable" }, 503);
-      const r = await memoryCall("deleteScheduledActionTool", { id: c.req.param("id") }, key);
-      return c.json(r);
+      const scheduleActionId = c.req.param("id");
+      const r = await memoryCall("deleteScheduledActionTool", { id: scheduleActionId }, key);
+      if (!r.ok) return c.json(r);
+      try {
+        await deleteScheduleConnectionBindings(user.email, scheduleActionId);
+        return c.json(r);
+      } catch (err) {
+        console.error("[schedule-actions] action deleted but connection binding cleanup failed", scheduleActionId, err instanceof Error ? err.message : String(err));
+        return c.json({ ...r, connectionBindingCleanupPending: true });
+      }
     });
     app.post("/schedule-link/revoke", auth2, async (c) => {
       try {