mcp-scraper 0.3.16 → 0.3.17

package/dist/bin/api-server.cjs

@@ -9054,6 +9054,104 @@ var init_seo_link_graph = __esm({
   }
 });
 
+// src/api/seo-link-report.ts
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function buildLinkReport(edges, metrics, siteUrl) {
+  let siteReg = "";
+  try {
+    siteReg = registrableDomain(new URL(siteUrl).hostname);
+  } catch {
+    siteReg = "";
+  }
+  const internalEdges = edges.filter((e) => e.internal);
+  const pages = metrics.length;
+  const orphans = metrics.filter((m) => m.orphan).length;
+  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
+  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
+  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
+  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+  for (const m of metrics) {
+    if (m.inlinks === 0) distribution.zero++;
+    else if (m.inlinks <= 2) distribution.oneToTwo++;
+    else if (m.inlinks <= 10) distribution.threeToTen++;
+    else distribution.elevenPlus++;
+  }
+  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
+  const domMap = /* @__PURE__ */ new Map();
+  let externalTotal = 0;
+  for (const e of edges) {
+    let domain;
+    try {
+      domain = registrableDomain(new URL(e.to).hostname);
+    } catch {
+      continue;
+    }
+    if (!domain || domain === siteReg) continue;
+    externalTotal++;
+    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+    d.links++;
+    if (e.nofollow) d.nofollow++;
+    d.pages.add(e.from);
+    domMap.set(domain, d);
+  }
+  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+  const round = (n) => Math.round(n * 10) / 10;
+  return {
+    summary: {
+      internal: {
+        totalLinks: internalEdges.length,
+        pages,
+        orphans,
+        brokenInternal,
+        avgInlinks: pages ? round(sumInlinks / pages) : 0,
+        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
+        distribution,
+        topByInlinks
+      },
+      external: {
+        totalLinks: externalTotal,
+        uniqueDomains: externalDomains.length,
+        topDomains: externalDomains.slice(0, 20)
+      }
+    },
+    externalDomains
+  };
+}
+function renderLinkReport(r) {
+  const s = r.summary;
+  const lines = [
+    `## Link analysis`,
+    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
+    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
+    ``,
+    `### Inlink distribution`,
+    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
+    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
+    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
+    `- 11+: ${s.internal.distribution.elevenPlus}`,
+    ``,
+    `### Top 20 internal pages by inlinks`,
+    `| inlinks | out (int/ext) | URL |`,
+    `|---|---|---|`,
+    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
+    ``,
+    `### Top 20 external domains by links`,
+    `| links | nofollow | from pages | domain |`,
+    `|---|---|---|---|`,
+    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
+  ];
+  return lines.join("\n");
+}
+var init_seo_link_report = __esm({
+  "src/api/seo-link-report.ts"() {
+    "use strict";
+  }
+});
+
 // src/api/seo-issues.ts
 function dupes(pages, key) {
   const groups = /* @__PURE__ */ new Map();
@@ -9722,6 +9820,7 @@ function chunk(items, size) {
 function buildArtifacts(job, pages, branding, imageAudit) {
   const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
   const issues = computeIssues(pages, metrics);
+  const linkReport = buildLinkReport(edges, [...metrics.values()], job.startUrl);
   let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
   if (imageAudit) reportMd += `
 
@@ -9737,7 +9836,10 @@ ${renderImageSection(imageAudit)}`;
     { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
     { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
     { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
-    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
+    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" },
+    { name: "link-report.md", body: renderLinkReport(linkReport), type: "text/markdown" },
+    { name: "links-summary.json", body: JSON.stringify(linkReport.summary, null, 2), type: "application/json" },
+    { name: "external-domains.json", body: JSON.stringify(linkReport.externalDomains, null, 2), type: "application/json" }
   ];
   if (imageAudit) {
     files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
@@ -9753,6 +9855,7 @@ var init_site_extract = __esm({
     init_client();
     init_site_extractor();
     init_seo_link_graph();
+    init_seo_link_report();
     init_seo_issues();
     init_image_audit();
     init_blob_store();
@@ -16231,6 +16334,9 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
       `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
       `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
       `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
+      `- \`link-report.md\` \u2014 link analysis: top internal pages by inlinks, top external domains, distribution`,
+      `- \`links-summary.json\` \u2014 link totals + top-20 internal pages + top-20 external domains`,
+      `- \`external-domains.json\` \u2014 every external domain linked to, ordered by link count`,
       ...imageAudit ? [
         `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
         `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
@@ -16266,7 +16372,13 @@ ${indexRows.join("\n")}`
       (0, import_node_fs5.writeFileSync)(reportFile, seo.reportMd + (imageAudit ? `
 
 ${renderImageSection(imageAudit)}` : ""), "utf8");
-      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
+      const linkReportFile = (0, import_node_path7.join)(dir, "link-report.md");
+      const linksSummaryFile = (0, import_node_path7.join)(dir, "links-summary.json");
+      const externalDomainsFile = (0, import_node_path7.join)(dir, "external-domains.json");
+      (0, import_node_fs5.writeFileSync)(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
+      (0, import_node_fs5.writeFileSync)(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
+      (0, import_node_fs5.writeFileSync)(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
+      seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
       if (imageAudit) {
         const imagesJsonl = (0, import_node_path7.join)(dir, "images.jsonl");
         const imagesSummary = (0, import_node_path7.join)(dir, "images-summary.json");
@@ -16718,6 +16830,7 @@ function buildSeoExport(siteUrl, pages, branding) {
     metrics: [...metrics.values()],
     issues,
     reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
+    linkReport: buildLinkReport(edges, [...metrics.values()], siteUrl),
     branding: branding ?? void 0
   };
 }
@@ -16826,6 +16939,9 @@ async function formatAuditSite(raw, input) {
   const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
   const img = imageAudit.summary;
   const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
+  const lr = seo.linkReport.summary;
+  const topExt = lr.external.topDomains.slice(0, 3).map((dm) => `${dm.domain} (${dm.links})`).join(", ") || "\u2014";
+  const linkLine = `${lr.internal.totalLinks} internal / ${lr.external.totalLinks} external links \xB7 ${lr.internal.orphans} orphans \xB7 ${lr.internal.brokenInternal} broken \xB7 avg ${lr.internal.avgInlinks} inlinks/page \xB7 top external: ${topExt}`;
   const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
   const structuredContent = {
     url: input.url,
@@ -16833,7 +16949,8 @@ async function formatAuditSite(raw, input) {
     durationMs: d.durationMs ?? 0,
     bulkFolder: bulk?.dir ?? null,
     issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
-    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
+    images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
+    links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
   };
   const location2 = bulk ? `
 ## \u{1F4C1} Technical audit saved
@@ -16852,6 +16969,9 @@ Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS
 ## Top issues
 ${topIssues || "_none found_"}`,
     `
+## Links
+${linkLine}`,
+    `
 ## Images
 ${imgLine}`
   ].join("\n");
@@ -18041,6 +18161,7 @@ var init_mcp_response_formatter = __esm({
     init_errors();
     init_workflow_catalog();
     init_seo_link_graph();
+    init_seo_link_report();
     init_seo_issues();
     init_image_audit();
     reportSavingEnabled = true;
@@ -23577,7 +23698,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.16";
+    PACKAGE_VERSION = "0.3.17";
   }
 });
 
@@ -23969,6 +24090,13 @@ var init_mcp_tool_schemas = __esm({
         totalBytes: import_zod27.z.number().min(0),
         over100kb: import_zod27.z.number().int().min(0),
         legacyFormat: import_zod27.z.number().int().min(0)
+      }),
+      links: import_zod27.z.object({
+        internal: import_zod27.z.number().int().min(0),
+        external: import_zod27.z.number().int().min(0),
+        orphans: import_zod27.z.number().int().min(0),
+        brokenInternal: import_zod27.z.number().int().min(0),
+        externalDomains: import_zod27.z.number().int().min(0)
       })
     };
     MapsPlaceIntelOutputSchema = {
@@ -24828,7 +24956,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => formatExtractSite(await executor.extractSite(input), input));
   server.registerTool("audit_site", {
     title: "Technical SEO Audit",
-    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
+    description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph (inlinks/orphans/crawl-depth), a link analysis (top internal pages by inlinks, top external domains by link count, broken/orphan counts), an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, link-report.md, links-summary.json, external-domains.json, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
     inputSchema: AuditSiteInputSchema,
     outputSchema: AuditSiteOutputSchema,
     annotations: liveWebToolAnnotations("Technical SEO Audit")