mcp-scraper 0.3.16 → 0.3.18

package/dist/bin/api-server.cjs

@@ -117,18 +117,26 @@ var init_errors = __esm({
 });
 
 // src/api/outbound-sanitize.ts
+function redactVendorUrls(value) {
+  return value.replace(VENDOR_URL_RE, "[browser-service]");
+}
 function sanitizeOutboundDiagnostics(value, parentKey = "") {
   if (typeof value === "string") {
-    if (SANITIZED_VALUE_KEYS.test(parentKey) && /kernel/i.test(value)) {
-      return sanitizeVendorName(value);
+    let out = redactVendorUrls(value);
+    if (SANITIZED_VALUE_KEYS.test(parentKey) && /kernel/i.test(out)) {
+      out = sanitizeVendorName(out);
     }
-    return value;
+    return out;
   }
   if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
   if (value !== null && typeof value === "object") {
     const out = {};
     for (const [key, val] of Object.entries(value)) {
       const renamed = KEY_RENAMES[key] ?? key;
+      if (VENDOR_URL_KEYS.has(key)) {
+        out[renamed] = null;
+        continue;
+      }
       out[renamed] = sanitizeOutboundDiagnostics(val, key);
     }
     return out;
@@ -149,7 +157,7 @@ function sanitizeHarvestResult(result) {
     }
   };
 }
-var KEY_RENAMES, SANITIZED_VALUE_KEYS;
+var KEY_RENAMES, SANITIZED_VALUE_KEYS, VENDOR_URL_KEYS, VENDOR_URL_RE;
 var init_outbound_sanitize = __esm({
   "src/api/outbound-sanitize.ts"() {
     "use strict";
@@ -167,6 +175,16 @@ var init_outbound_sanitize = __esm({
       kernelProxyId: "proxyId"
     };
     SANITIZED_VALUE_KEYS = /error|message/i;
+    VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
+      "hosted_url",
+      "hostedUrl",
+      "live_view_url",
+      "liveViewUrl",
+      "cdp_ws_url",
+      "cdpWsUrl",
+      "browser_live_view_url"
+    ]);
+    VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
   }
 });
 
@@ -9054,6 +9072,104 @@ var init_seo_link_graph = __esm({
   }
 });
 
+// src/api/seo-link-report.ts
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function buildLinkReport(edges, metrics, siteUrl) {
+  let siteReg = "";
+  try {
+    siteReg = registrableDomain(new URL(siteUrl).hostname);
+  } catch {
+    siteReg = "";
+  }
+  const internalEdges = edges.filter((e) => e.internal);
+  const pages = metrics.length;
+  const orphans = metrics.filter((m) => m.orphan).length;
+  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
+  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
+  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
+  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+  for (const m of metrics) {
+    if (m.inlinks === 0) distribution.zero++;
+    else if (m.inlinks <= 2) distribution.oneToTwo++;
+    else if (m.inlinks <= 10) distribution.threeToTen++;
+    else distribution.elevenPlus++;
+  }
+  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
+  const domMap = /* @__PURE__ */ new Map();
+  let externalTotal = 0;
+  for (const e of edges) {
+    let domain;
+    try {
+      domain = registrableDomain(new URL(e.to).hostname);
+    } catch {
+      continue;
+    }
+    if (!domain || domain === siteReg) continue;
+    externalTotal++;
+    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+    d.links++;
+    if (e.nofollow) d.nofollow++;
+    d.pages.add(e.from);
+    domMap.set(domain, d);
+  }
+  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+  const round = (n) => Math.round(n * 10) / 10;
+  return {
+    summary: {
+      internal: {
+        totalLinks: internalEdges.length,
+        pages,
+        orphans,
+        brokenInternal,
+        avgInlinks: pages ? round(sumInlinks / pages) : 0,
+        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
+        distribution,
+        topByInlinks
+      },
+      external: {
+        totalLinks: externalTotal,
+        uniqueDomains: externalDomains.length,
+        topDomains: externalDomains.slice(0, 20)
+      }
+    },
+    externalDomains
+  };
+}
+function renderLinkReport(r) {
+  const s = r.summary;
+  const lines = [
+    `## Link analysis`,
+    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
+    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
+    ``,
+    `### Inlink distribution`,
+    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
+    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
+    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
+    `- 11+: ${s.internal.distribution.elevenPlus}`,
+    ``,
+    `### Top 20 internal pages by inlinks`,
+    `| inlinks | out (int/ext) | URL |`,
+    `|---|---|---|`,
+    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
+    ``,
+    `### Top 20 external domains by links`,
+    `| links | nofollow | from pages | domain |`,
+    `|---|---|---|---|`,
+    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
+  ];
+  return lines.join("\n");
+}
+var init_seo_link_report = __esm({
+  "src/api/seo-link-report.ts"() {
+    "use strict";
+  }
+});
+
 // src/api/seo-issues.ts
 function dupes(pages, key) {
   const groups = /* @__PURE__ */ new Map();
@@ -9714,14 +9830,10 @@ var init_site_extract_repository = __esm({
 });
 
 // src/inngest/functions/site-extract.ts
-function chunk(items, size) {
-  const out = [];
-  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
-  return out;
-}
 function buildArtifacts(job, pages, branding, imageAudit) {
   const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
   const issues = computeIssues(pages, metrics);
+  const linkReport = buildLinkReport(edges, [...metrics.values()], job.startUrl);
   let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
   if (imageAudit) reportMd += `
 
@@ -9737,7 +9849,10 @@ ${renderImageSection(imageAudit)}`;
     { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
     { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
     { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
-    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
+    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" },
+    { name: "link-report.md", body: renderLinkReport(linkReport), type: "text/markdown" },
+    { name: "links-summary.json", body: JSON.stringify(linkReport.summary, null, 2), type: "application/json" },
+    { name: "external-domains.json", body: JSON.stringify(linkReport.externalDomains, null, 2), type: "application/json" }
   ];
   if (imageAudit) {
     files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
@@ -9753,6 +9868,7 @@ var init_site_extract = __esm({
     init_client();
     init_site_extractor();
     init_seo_link_graph();
+    init_seo_link_report();
     init_seo_issues();
     init_image_audit();
     init_blob_store();
@@ -9788,18 +9904,62 @@ var init_site_extract = __esm({
         const maxPages = Number(job.options.maxPages ?? 1e4);
         const concurrency = Number(job.options.concurrency ?? 1);
         const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
-        const urls = await step.run("discover", async () => {
+        const seed = await step.run("discover", async () => {
           const discovered = await discoverUrls(job.startUrl, maxPages, key);
-          await setExtractJobTotal(jobId, discovered.length);
-          return discovered;
+          const out = discovered.length ? discovered : [job.startUrl];
+          await setExtractJobTotal(jobId, out.length);
+          return out;
         });
-        const batches = chunk(urls, BATCH);
-        for (let i = 0; i < batches.length; i++) {
-          await step.run(`crawl-batch-${i}`, async () => {
-            const pages = await extractPagesRotating(batches[i], { kernelApiKey: key, concurrency, urlsPerBrowser });
+        const siteHost = (() => {
+          try {
+            return new URL(job.startUrl).hostname.replace(/^www\./, "");
+          } catch {
+            return "";
+          }
+        })();
+        const norm = (u) => {
+          try {
+            const x = new URL(u);
+            x.hash = "";
+            return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
+          } catch {
+            return u;
+          }
+        };
+        const sameSite = (u) => {
+          try {
+            return new URL(u).hostname.replace(/^www\./, "") === siteHost;
+          } catch {
+            return false;
+          }
+        };
+        const queue = [...seed];
+        const seen = new Set(seed.map(norm));
+        let crawled = 0;
+        const MAX_BATCHES = Math.ceil(maxPages / BATCH) + 30;
+        for (let i = 0; i < MAX_BATCHES && crawled < maxPages && crawled < queue.length; i++) {
+          const batch = queue.slice(crawled, Math.min(crawled + BATCH, maxPages));
+          if (!batch.length) break;
+          const newLinks = await step.run(`crawl-batch-${i}`, async () => {
+            const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
             await saveExtractPages(jobId, pages);
-            return pages.length;
+            const found = [];
+            for (const p of pages) for (const l of p.outlinks ?? []) {
+              if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
+            }
+            return [...new Set(found)];
           });
+          crawled += batch.length;
+          let added = false;
+          for (const href of newLinks) {
+            const n = norm(href);
+            if (!seen.has(n) && seen.size < maxPages) {
+              seen.add(n);
+              queue.push(href);
+              added = true;
+            }
+          }
+          if (added) await setExtractJobTotal(jobId, Math.min(seen.size, maxPages));
         }
         const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
         const imageAudit = Array.isArray(job.options.formats) && job.options.formats.includes("images") ? await step.run("image-audit", async () => {
@@ -12144,10 +12304,10 @@ function buildTranscriptMarkdown(result) {
   if (result.chunks?.length) {
     lines.push("## Timestamped Segments");
     lines.push("");
-    for (const chunk2 of result.chunks) {
-      const start = fmtTs(chunk2.timestamp[0]);
-      const end = fmtTs(chunk2.timestamp[1]);
-      lines.push(`**[${start} \u2192 ${end}]** ${chunk2.text.trim()}`);
+    for (const chunk of result.chunks) {
+      const start = fmtTs(chunk.timestamp[0]);
+      const end = fmtTs(chunk.timestamp[1]);
+      lines.push(`**[${start} \u2192 ${end}]** ${chunk.text.trim()}`);
       lines.push("");
     }
   }
@@ -12785,20 +12945,20 @@ var init_FacebookAdExtractor = __esm({
           if (splitRe.lastIndex === m.index) splitRe.lastIndex++;
         }
         if (last < bodyText.length) adChunks.push(bodyText.slice(last));
-        const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk2) => {
-          const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
+        const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk) => {
+          const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
           const libraryId = lidM ? lidM[1] : null;
-          const statusM = chunk2.match(/^(Active|Inactive)/i);
+          const statusM = chunk.match(/^(Active|Inactive)/i);
           const status = statusM ? statusM[1] : null;
-          const startM = chunk2.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
+          const startM = chunk.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
           const started = startM ? startM[1].replace(/\s+/g, " ").trim() : null;
           const visual = adVisuals[libraryId ?? ""] ?? { imageSrc: null, videoSrc: null, videoPoster: null };
-          const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk2);
+          const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk);
           const creativeType = visual.videoSrc || hasVideoText ? "video" : visual.imageSrc ? "image" : "unknown";
           let domain = null;
           const domainRe = /\b([A-Z0-9]{2,}(?:\.[A-Z]{2,})+)\b/g;
           let dm;
-          while ((dm = domainRe.exec(chunk2)) !== null) {
+          while ((dm = domainRe.exec(chunk)) !== null) {
             const c = dm[1];
             if (c.length >= 6 && c !== "LIBRARY") {
               domain = c;
@@ -12807,9 +12967,9 @@ var init_FacebookAdExtractor = __esm({
           }
           let primaryText = null;
           let headline = null;
-          const sponsoredIdx = chunk2.search(/\bSponsored\b/i);
+          const sponsoredIdx = chunk.search(/\bSponsored\b/i);
           if (sponsoredIdx >= 0) {
-            let afterSponsored = chunk2.slice(sponsoredIdx + "Sponsored".length).trim();
+            let afterSponsored = chunk.slice(sponsoredIdx + "Sponsored".length).trim();
             afterSponsored = afterSponsored.replace(/\s*\d*:?\d+\s*\/\s*\d*:?\d+.*$/, "");
             if (domain) {
               const di = afterSponsored.indexOf(domain);
@@ -12820,18 +12980,18 @@ var init_FacebookAdExtractor = __esm({
           }
           const ctaRe2 = new RegExp("\\b(" + CTA_LABELS.map((l) => l.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|") + ")\\b", "i");
           if (domain) {
-            const di = chunk2.indexOf(domain);
+            const di = chunk.indexOf(domain);
             if (di >= 0) {
-              let afterDomain = chunk2.slice(di + domain.length).trim();
+              let afterDomain = chunk.slice(di + domain.length).trim();
               const ctaMatch = afterDomain.search(ctaRe2);
               if (ctaMatch >= 0) afterDomain = afterDomain.slice(0, ctaMatch).trim();
               if (afterDomain.length > 3 && afterDomain.length < 120) headline = afterDomain;
             }
           }
           if (!headline) {
-            const ctaIdx = chunk2.search(ctaRe2);
+            const ctaIdx = chunk.search(ctaRe2);
             if (ctaIdx > 0) {
-              const beforeCta = chunk2.slice(0, ctaIdx).trimEnd();
+              const beforeCta = chunk.slice(0, ctaIdx).trimEnd();
               const sentenceEnd = Math.max(beforeCta.lastIndexOf(". "), beforeCta.lastIndexOf("! "), beforeCta.lastIndexOf("? "));
               if (sentenceEnd >= 0) {
                 const candidate = beforeCta.slice(sentenceEnd + 2).trim();
@@ -12843,12 +13003,12 @@ var init_FacebookAdExtractor = __esm({
           }
           let cta = null;
           for (const lbl of CTA_LABELS) {
-            if (chunk2.toLowerCase().includes(lbl.toLowerCase())) {
+            if (chunk.toLowerCase().includes(lbl.toLowerCase())) {
               cta = lbl;
               break;
             }
           }
-          const multiM = chunk2.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
+          const multiM = chunk.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
           const clusterCount = multiM ? parseInt(multiM[1]) : null;
           return {
             libraryId,
@@ -12871,8 +13031,8 @@ var init_FacebookAdExtractor = __esm({
         const unknownCreativeCount = ads.filter((a) => a.creativeType === "unknown").length;
         const advertiserPageId = new URL(page.url()).searchParams.get("view_all_page_id") ?? null;
         const nameFreq = /* @__PURE__ */ new Map();
-        for (const chunk2 of adChunks) {
-          const detailsParts = chunk2.split("See ad details ");
+        for (const chunk of adChunks) {
+          const detailsParts = chunk.split("See ad details ");
           if (detailsParts.length < 2) continue;
           const sponsoredIdx = detailsParts[1].indexOf(" Sponsored");
           if (sponsoredIdx < 0) continue;
@@ -14291,11 +14451,11 @@ var init_facebook_ad_routes = __esm({
         }
         if (last < bodyText.length) adChunks.push(bodyText.slice(last));
         const advertiserMap = /* @__PURE__ */ new Map();
-        for (const chunk2 of adChunks) {
-          const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
+        for (const chunk of adChunks) {
+          const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
           if (!lidM) continue;
           const libraryId = lidM[1];
-          const detailsParts = chunk2.split("See ad details ");
+          const detailsParts = chunk.split("See ad details ");
           if (detailsParts.length < 2) continue;
           const afterDetails = detailsParts[1];
           const sponsoredIdx = afterDetails.indexOf(" Sponsored");
@@ -14819,8 +14979,8 @@ function runFfmpegMux(videoPath, audioPath, outPath) {
       stdio: ["ignore", "ignore", "pipe"]
     });
     let stderr = "";
-    child.stderr.on("data", (chunk2) => {
-      stderr += String(chunk2);
+    child.stderr.on("data", (chunk) => {
+      stderr += String(chunk);
     });
     child.on("error", (err) => resolve({ ok: false, error: err.message }));
     child.on("close", (code) => resolve({ ok: code === 0, error: code === 0 ? null : stderr.trim() || `ffmpeg exited ${code}` }));
@@ -16231,6 +16391,9 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
       `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
       `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
       `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      `- \`link-report.md\` \u2014 link analysis: top internal pages by inlinks, top external domains, distribution`,
+      `- \`links-summary.json\` \u2014 link totals + top-20 internal pages + top-20 external domains`,
+      `- \`external-domains.json\` \u2014 every external domain linked to, ordered by link count`,
       ...imageAudit ? [
         `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
         `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
@@ -16266,7 +16429,13 @@ ${indexRows.join("\n")}`
       (0, import_node_fs5.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
 
 ${renderImageSection(imageAudit)}` : ""), "utf8");
-      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      const linkReportFile = (0, import_node_path7.join)(dir, "link-report.md");
+      const linksSummaryFile = (0, import_node_path7.join)(dir, "links-summary.json");
+      const externalDomainsFile = (0, import_node_path7.join)(dir, "external-domains.json");
+      (0, import_node_fs5.writeFileSync)(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
+      (0, import_node_fs5.writeFileSync)(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
+      (0, import_node_fs5.writeFileSync)(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
       if (imageAudit) {
         const imagesJsonl = (0, import_node_path7.join)(dir, "images.jsonl");
         const imagesSummary = (0, import_node_path7.join)(dir, "images-summary.json");
@@ -16718,6 +16887,7 @@ function buildSeoExport(siteUrl, pages, branding) {
     metrics: [...metrics.values()],
     issues,
     reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    linkReport: buildLinkReport(edges, [...metrics.values()], siteUrl),
     branding: branding ?? void 0
   };
 }
@@ -16826,6 +16996,9 @@ async function formatAuditSite(raw, input) {
   const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
   const img = imageAudit.summary;
   const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const lr = seo.linkReport.summary;
+  const topExt = lr.external.topDomains.slice(0, 3).map((dm) => `${dm.domain} (${dm.links})`).join(", ") || "\u2014";
+  const linkLine = `${lr.internal.totalLinks} internal / ${lr.external.totalLinks} external links \xB7 ${lr.internal.orphans} orphans \xB7 ${lr.internal.brokenInternal} broken \xB7 avg ${lr.internal.avgInlinks} inlinks/page \xB7 top external: ${topExt}`;
   const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
   const structuredContent = {
     url: input.url,
@@ -16833,7 +17006,8 @@ async function formatAuditSite(raw, input) {
     durationMs: d.durationMs ?? 0,
     bulkFolder: bulk?.dir ?? null,
     issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
-    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
+    links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
   };
   const location2 = bulk ? `
 ## \u{1F4C1} Technical audit saved
@@ -16852,6 +17026,9 @@ Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS
 ## Top issues
 ${topIssues || "_none found_"}`,
     `
+## Links
+${linkLine}`,
+    `
 ## Images
 ${imgLine}`
   ].join("\n");
@@ -18041,6 +18218,7 @@ var init_mcp_response_formatter = __esm({
     init_errors();
     init_workflow_catalog();
     init_seo_link_graph();
+    init_seo_link_report();
     init_seo_issues();
     init_image_audit();
     reportSavingEnabled = true;
@@ -23577,7 +23755,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.16";
+    PACKAGE_VERSION = "0.3.18";
   }
 });
 
@@ -23601,7 +23779,11 @@ ROUTING
 - Workflows (multi-step): local directory build -> directory_workflow; rank blueprint -> rank_tracker_workflow;
   AI-answer citation fan-out (AEO) -> query_fanout_workflow; deep research -> deep_research_workflow.
 - Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) -> the browser_* agent
-  (browser_open, then navigate/read); save/reuse logins via browser_profile_*.
+  (browser_open, then navigate/read).
+- Logged-in sites (ChatGPT, Claude, Reddit, any account): save the login first -> browser_profile_connect
+  (returns an mcpscraper.dev sign-in link for the user; one profile holds MANY logins, call it again with
+  the same profile + a new domain to add accounts) -> poll browser_profile_list until AUTHENTICATED ->
+  browser_open with that profile. browser_profile_list also shows what a profile is connected to.
 
 NOTES
 - Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
@@ -23969,6 +24151,13 @@ var init_mcp_tool_schemas = __esm({
         totalBytes: import_zod27.z.number().min(0),
         over100kb: import_zod27.z.number().int().min(0),
         legacyFormat: import_zod27.z.number().int().min(0)
+      }),
+      links: import_zod27.z.object({
+        internal: import_zod27.z.number().int().min(0),
+        external: import_zod27.z.number().int().min(0),
+        orphans: import_zod27.z.number().int().min(0),
+        brokenInternal: import_zod27.z.number().int().min(0),
+        externalDomains: import_zod27.z.number().int().min(0)
       })
     };
     MapsPlaceIntelOutputSchema = {
@@ -24828,7 +25017,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
   server.registerTool("audit_site", {
     title: "Technical SEO Audit",
-    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph (inlinks/orphans/crawl-depth), a link analysis (top internal pages by inlinks, top external domains by link count, broken/orphan counts), an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, link-report.md, links-summary.json, external-domains.json, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
     inputSchema: AuditSiteInputSchema,
     outputSchema: AuditSiteOutputSchema,
     annotations: liveWebToolAnnotations("Technical SEO Audit")
@@ -25390,8 +25579,64 @@ async function migrateBrowserAgent() {
     )
   `);
   await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_replays_session ON browser_agent_replays(session_id)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS browser_auth_connections (
+      connection_id TEXT PRIMARY KEY,
+      hosted_url    TEXT,
+      domain        TEXT,
+      profile       TEXT,
+      account_email TEXT,
+      note          TEXT,
+      created_at    TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at    TEXT
+    )
+  `);
+  try {
+    await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN account_email TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN note TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN updated_at TEXT`);
+  } catch {
+  }
+  await db.execute(`CREATE INDEX IF NOT EXISTS browser_auth_connections_profile ON browser_auth_connections(profile)`);
   _ready = true;
 }
+async function saveAuthConnection(input) {
+  if (!input.connectionId) return;
+  const db = getDb();
+  await db.execute({
+    sql: `INSERT INTO browser_auth_connections (connection_id, hosted_url, domain, profile, account_email, note, updated_at)
+          VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
+          ON CONFLICT(connection_id) DO UPDATE SET
+            hosted_url    = COALESCE(excluded.hosted_url, browser_auth_connections.hosted_url),
+            domain        = COALESCE(excluded.domain, browser_auth_connections.domain),
+            profile       = COALESCE(excluded.profile, browser_auth_connections.profile),
+            account_email = COALESCE(excluded.account_email, browser_auth_connections.account_email),
+            note          = COALESCE(excluded.note, browser_auth_connections.note),
+            updated_at    = datetime('now')`,
+    args: [input.connectionId, input.hostedUrl, input.domain, input.profile, input.accountEmail ?? null, input.note ?? null]
+  });
+}
+async function getAuthConnectionRow(connectionId) {
+  if (!connectionId) return null;
+  const db = getDb();
+  const res = await db.execute({ sql: `SELECT * FROM browser_auth_connections WHERE connection_id = ?`, args: [connectionId] });
+  return res.rows[0] ?? null;
+}
+async function listAuthConnectionsByProfile(profile) {
+  if (!profile) return [];
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT * FROM browser_auth_connections WHERE profile = ? ORDER BY COALESCE(updated_at, created_at) DESC`,
+    args: [profile]
+  });
+  return res.rows;
+}
 async function createSessionRow(input) {
   const db = getDb();
   const id = `bas_${(0, import_node_crypto6.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
@@ -26419,6 +26664,17 @@ async function findProfileConnection(k, profileName, domain) {
   }
   return null;
 }
+async function listProfileConnections(profileName) {
+  if (!profileName) return [];
+  const k = client();
+  const out = [];
+  for await (const connection of k.auth.connections.list({ profile_name: profileName })) {
+    const data = connection;
+    if (data.profile_name && data.profile_name !== profileName) continue;
+    out.push(profileOnboardResult(connection));
+  }
+  return out;
+}
 async function findOrCreateProfileConnection(k, opts) {
   try {
     return await k.auth.connections.create({
@@ -26821,7 +27077,7 @@ function publicSession(row) {
     session_id: row.id,
     status: row.status,
     label: row.label,
-    live_view_url: row.live_view_url,
+    watch_url: `/console/${row.id}`,
     created_at: row.created_at,
     last_action_at: row.last_action_at,
     closed_at: row.closed_at,
@@ -26877,10 +27133,23 @@ function buildBrowserAgentRoutes() {
     const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
     const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : "chatgpt.com";
     const loginUrl = typeof body.login_url === "string" ? body.login_url.trim() : void 0;
+    const accountEmail = typeof body.account_email === "string" ? body.account_email.trim() || null : null;
+    const note = typeof body.note === "string" ? body.note.trim() || null : null;
     if (!profileName) return c.json({ error: "profile is required" }, 400);
     if (!domain) return c.json({ error: "domain is required" }, 400);
     try {
       const result = await startProfileOnboarding({ profileName, domain, loginUrl });
+      try {
+        await saveAuthConnection({
+          connectionId: result.connectionId,
+          hostedUrl: result.hostedUrl,
+          domain: result.domain,
+          profile: result.profileName,
+          accountEmail,
+          note
+        });
+      } catch {
+      }
       return c.json({
         connection_id: result.connectionId,
         profile: result.profileName,
@@ -26889,8 +27158,6 @@ function buildBrowserAgentRoutes() {
         flow_status: result.flowStatus,
         flow_step: result.flowStep,
         flow_expires_at: result.flowExpiresAt,
-        hosted_url: result.hostedUrl,
-        live_view_url: result.liveViewUrl,
         browser_session_id: result.browserSessionId,
         post_login_url: result.postLoginUrl
       });
@@ -26905,6 +27172,17 @@ function buildBrowserAgentRoutes() {
     const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
     try {
       const result = await getProfileOnboardingStatus({ connectionId, profileName, domain });
+      if (result.hostedUrl) {
+        try {
+          await saveAuthConnection({
+            connectionId: result.connectionId,
+            hostedUrl: result.hostedUrl,
+            domain: result.domain,
+            profile: result.profileName
+          });
+        } catch {
+        }
+      }
       return c.json({
         connection_id: result.connectionId,
         profile: result.profileName,
@@ -26913,8 +27191,6 @@ function buildBrowserAgentRoutes() {
         flow_status: result.flowStatus,
         flow_step: result.flowStep,
         flow_expires_at: result.flowExpiresAt,
-        hosted_url: result.hostedUrl,
-        live_view_url: result.liveViewUrl,
         browser_session_id: result.browserSessionId,
         post_login_url: result.postLoginUrl
       });
@@ -26922,6 +27198,37 @@ function buildBrowserAgentRoutes() {
       return c.json(failure(err), 404);
     }
   });
+  app2.post("/profiles/list", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
+    if (!profileName) return c.json({ error: "profile is required" }, 400);
+    const domainFilter = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
+    const connectionIdFilter = typeof body.connection_id === "string" ? body.connection_id.trim() : void 0;
+    try {
+      const [live, stored] = await Promise.all([
+        listProfileConnections(profileName),
+        listAuthConnectionsByProfile(profileName).catch(() => [])
+      ]);
+      const notesByDomain = new Map(stored.map((row) => [row.domain ?? "", row]));
+      const notesByConnection = new Map(stored.map((row) => [row.connection_id, row]));
+      let connections = live.map((conn) => {
+        const record = notesByConnection.get(conn.connectionId) ?? notesByDomain.get(conn.domain);
+        return {
+          connection_id: conn.connectionId,
+          domain: conn.domain,
+          status: conn.status,
+          account_email: record?.account_email ?? null,
+          note: record?.note ?? null,
+          last_connected_at: record?.updated_at ?? record?.created_at ?? null
+        };
+      });
+      if (connectionIdFilter) connections = connections.filter((c2) => c2.connection_id === connectionIdFilter);
+      if (domainFilter) connections = connections.filter((c2) => c2.domain === domainFilter);
+      return c.json({ profile: profileName, connections });
+    } catch (err) {
+      return c.json(failure(err), 502);
+    }
+  });
   app2.post("/sessions", async (c) => {
     const user = c.get("user");
     if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
@@ -27294,6 +27601,50 @@ var init_browser_agent_routes = __esm({
 });
 
 // src/api/browser-agent-console.ts
+function renderAuthConsoleHtml(connectionId, hostedUrl) {
+  const safeId = String(connectionId).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]);
+  const goUrl = `/console/auth/${encodeURIComponent(connectionId)}/go`;
+  const body = hostedUrl ? `<iframe class="stage" allow="clipboard-read; clipboard-write" src="${goUrl}"></iframe>
+       <div class="fallback">
+         <span>Sign-in window not showing?</span>
+         <a class="btn" href="${goUrl}" target="_blank" rel="noopener">Open sign-in in a new tab</a>
+       </div>` : `<div class="empty">
+         <div>This sign-in link is no longer active.</div>
+         <div class="muted">Ask the assistant to reconnect the profile, then open the new link.</div>
+       </div>`;
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+<title>MCP Scraper \u2014 Secure Sign-in</title>
+<style>
+  :root { color-scheme: dark; }
+  :where(*) { box-sizing: border-box; }
+  body { margin: 0; height: 100vh; display: flex; flex-direction: column; font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, sans-serif; background: #0b0e14; color: #d7dce5; }
+  header { display: flex; align-items: center; gap: 10px; padding: 12px 18px; border-bottom: 1px solid #1c2230; background: #0f131c; }
+  header h1 { font-size: 14px; margin: 0; font-weight: 600; color: #fff; }
+  header .muted { color: #6b7689; font-size: 12px; }
+  .spacer { flex: 1; }
+  .stage { flex: 1; width: 100%; border: 0; display: block; background: #05070b; }
+  .fallback { display: flex; align-items: center; gap: 12px; padding: 10px 18px; border-top: 1px solid #1c2230; background: #0f131c; font-size: 13px; color: #8893a7; }
+  .btn { background: #2b6cff; color: #fff; border-radius: 7px; padding: 7px 12px; text-decoration: none; font-weight: 500; }
+  .empty { flex: 1; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 8px; color: #8893a7; }
+  .muted { color: #6b7689; font-size: 12px; }
+</style>
+${ATTRIBUTION_PIXEL_SCRIPT}
+</head>
+<body>
+<header>
+  <h1>MCP Scraper</h1>
+  <span class="muted">Secure sign-in \xB7 ${safeId}</span>
+  <span class="spacer"></span>
+  <span class="muted">Complete the login below \u2014 your session is saved to your profile.</span>
+</header>
+${body}
+</body>
+</html>`;
+}
 function renderConsoleHtml(initialSessionId) {
   const initial = JSON.stringify(initialSessionId ?? "");
   return `<!DOCTYPE html>
@@ -28079,6 +28430,7 @@ var init_server = __esm({
     init_mcp_routes();
     init_browser_agent_routes();
     init_browser_agent_console();
+    init_browser_agent_db();
     init_stripe_routes();
     init_site_audit_worker();
     import_factory6 = require("hono/factory");
@@ -28907,6 +29259,19 @@ var init_server = __esm({
     app.route("/mcp", mcpApp);
     app.route("/agent", buildBrowserAgentRoutes());
     app.get("/console", (c) => c.html(renderConsoleHtml()));
+    app.get("/console/auth/:id", async (c) => {
+      const id = c.req.param("id");
+      await migrateBrowserAgent();
+      const row = await getAuthConnectionRow(id);
+      return c.html(renderAuthConsoleHtml(id, row?.hosted_url ?? null));
+    });
+    app.get("/console/auth/:id/go", async (c) => {
+      const id = c.req.param("id");
+      await migrateBrowserAgent();
+      const row = await getAuthConnectionRow(id);
+      if (!row?.hosted_url) return c.text("This sign-in link is no longer active.", 404);
+      return c.redirect(row.hosted_url, 302);
+    });
     app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
     app.route("/stripe", stripeApp);
     if (!process.env.INNGEST_EVENT_KEY) {