mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-PGJQDMC2.js

@@ -1,383 +0,0 @@
-import {
-  validatePublicHttpUrl
-} from "./chunk-GGZEC22A.js";
-
-// src/api/image-audit.ts
-var OVER_BYTES = 100 * 1024;
-var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
-function formatBytes(n) {
-  if (n == null) return null;
-  if (n < 1024) return `${n} B`;
-  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
-  return `${(n / (1024 * 1024)).toFixed(2)} MB`;
-}
-var decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
-var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
-var formatOf = (ct, url) => {
-  if (ct) {
-    const m = ct.split(";")[0].trim().toLowerCase();
-    if (m.startsWith("image/")) return m.replace("image/", "");
-  }
-  return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
-};
-function collectUrls(pages) {
-  const byKey = /* @__PURE__ */ new Map();
-  for (const p of pages) for (const raw of p.imageLinks || []) {
-    const u = decodeEntities(raw);
-    if (!/^https?:\/\//i.test(u)) continue;
-    if (/[{}]|%7[bd]/i.test(u)) continue;
-    const k = dedupKey(u);
-    const prev = byKey.get(k);
-    if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
-  }
-  return [...byKey.values()];
-}
-async function sizeAndType(url, timeoutMs) {
-  const ctrl = new AbortController();
-  const t = setTimeout(() => ctrl.abort(), timeoutMs);
-  const read = (res) => ({
-    len: res.headers.get("content-length"),
-    cr: res.headers.get("content-range"),
-    ct: res.headers.get("content-type"),
-    status: res.status
-  });
-  const safeFetch = async (method) => {
-    let target = url;
-    for (let redirects = 0; redirects <= 5; redirects++) {
-      const checked = await validatePublicHttpUrl(target, { field: "image URL" });
-      if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Image URL was rejected");
-      const response = await fetch(checked.parsed.href, {
-        method,
-        ...method === "GET" ? { headers: { Range: "bytes=0-0" } } : {},
-        redirect: "manual",
-        signal: ctrl.signal
-      });
-      if (response.status >= 300 && response.status < 400) {
-        const location = response.headers.get("location");
-        if (!location) throw new Error(`HTTP ${response.status} redirect did not include Location`);
-        target = new URL(location, checked.parsed.href).href;
-        continue;
-      }
-      return response;
-    }
-    throw new Error("Image request exceeded five redirects");
-  };
-  try {
-    let bytes = null;
-    let ct = null;
-    let status = null;
-    try {
-      const h = read(await safeFetch("HEAD"));
-      status = h.status;
-      ct = h.ct;
-      if (h.len) bytes = Number(h.len);
-    } catch {
-    }
-    if (bytes == null) {
-      const g = await safeFetch("GET");
-      const r = read(g);
-      status = status ?? r.status;
-      ct = ct ?? r.ct;
-      if (r.cr && r.cr.includes("/")) {
-        const tail = r.cr.split("/")[1];
-        if (tail && tail !== "*") bytes = Number(tail);
-      } else if (r.len) bytes = Number(r.len);
-      try {
-        await g.body?.cancel();
-      } catch {
-      }
-    }
-    return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
-  } catch (e) {
-    return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
-  } finally {
-    clearTimeout(t);
-  }
-}
-async function pool(items, n, fn) {
-  const out = new Array(items.length);
-  let i = 0;
-  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
-    while (i < items.length) {
-      const idx = i++;
-      out[idx] = await fn(items[idx]);
-    }
-  }));
-  return out;
-}
-async function auditImages(pages, opts = {}) {
-  return auditImageUrls(collectUrls(pages), opts);
-}
-async function auditImageUrls(inputUrls, opts = {}) {
-  const concurrency = opts.concurrency ?? 12;
-  const timeoutMs = opts.timeoutMs ?? 12e3;
-  const max = opts.max ?? 5e3;
-  const urls = collectUrls([{ imageLinks: inputUrls }]).slice(0, max);
-  const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
-  const rows = heads.map((r) => {
-    const format = formatOf(r.contentType, r.url);
-    return {
-      ...r,
-      size: formatBytes(r.bytes),
-      format,
-      over100kb: r.bytes != null && r.bytes > OVER_BYTES,
-      legacyFormat: format !== "unknown" && !MODERN.has(format)
-    };
-  });
-  const sized = rows.filter((r) => r.bytes != null);
-  const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
-  const formatCounts = {};
-  for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
-  return {
-    rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
-    summary: {
-      unique: rows.length,
-      sized: sized.length,
-      totalBytes,
-      totalSize: formatBytes(totalBytes) ?? "0 B",
-      avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
-      over100kb: rows.filter((r) => r.over100kb).length,
-      legacyFormat: rows.filter((r) => r.legacyFormat).length,
-      formatCounts,
-      ...opts.sampleTruncated ? { sampleTruncated: true, sampleCap: max } : {}
-    }
-  };
-}
-function renderImageSection(audit) {
-  const s = audit.summary;
-  const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join("  ");
-  const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
-  const lines = [
-    `## Images`,
-    `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
-    `Formats: ${fmt}`
-  ];
-  if (heaviest.length) {
-    lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
-    for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
-  }
-  return lines.join("\n");
-}
-
-// src/api/seo-link-report.ts
-function registrableDomain(host) {
-  const h = host.replace(/^www\./, "").toLowerCase();
-  const parts = h.split(".");
-  return parts.length <= 2 ? h : parts.slice(-2).join(".");
-}
-function buildLinkReport(edges, metrics, siteUrl) {
-  let siteReg = "";
-  try {
-    siteReg = registrableDomain(new URL(siteUrl).hostname);
-  } catch {
-    siteReg = "";
-  }
-  const internalEdges = edges.filter((e) => e.internal);
-  const pages = metrics.length;
-  const orphans = metrics.filter((m) => m.orphan).length;
-  const brokenInternal = internalEdges.filter((e) => e.targetStatus != null && e.targetStatus >= 400).length;
-  const sumInlinks = metrics.reduce((a, m) => a + m.inlinks, 0);
-  const sumOutlinks = metrics.reduce((a, m) => a + m.outlinksInternal + m.outlinksExternal, 0);
-  const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
-  for (const m of metrics) {
-    if (m.inlinks === 0) distribution.zero++;
-    else if (m.inlinks <= 2) distribution.oneToTwo++;
-    else if (m.inlinks <= 10) distribution.threeToTen++;
-    else distribution.elevenPlus++;
-  }
-  const topByInlinks = [...metrics].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }));
-  const domMap = /* @__PURE__ */ new Map();
-  let externalTotal = 0;
-  for (const e of edges) {
-    let domain;
-    try {
-      domain = registrableDomain(new URL(e.to).hostname);
-    } catch {
-      continue;
-    }
-    if (!domain || domain === siteReg) continue;
-    externalTotal++;
-    const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
-    d.links++;
-    if (e.nofollow) d.nofollow++;
-    d.pages.add(e.from);
-    domMap.set(domain, d);
-  }
-  const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
-  const round = (n) => Math.round(n * 10) / 10;
-  return {
-    summary: {
-      internal: {
-        totalLinks: internalEdges.length,
-        pages,
-        orphans,
-        brokenInternal,
-        avgInlinks: pages ? round(sumInlinks / pages) : 0,
-        avgOutlinks: pages ? round(sumOutlinks / pages) : 0,
-        distribution,
-        topByInlinks
-      },
-      external: {
-        totalLinks: externalTotal,
-        uniqueDomains: externalDomains.length,
-        topDomains: externalDomains.slice(0, 20)
-      }
-    },
-    externalDomains
-  };
-}
-function renderLinkReport(r) {
-  const s = r.summary;
-  const lines = [
-    `## Link analysis`,
-    `**Internal:** ${s.internal.totalLinks} links \xB7 ${s.internal.pages} pages \xB7 avg ${s.internal.avgInlinks} inlinks/page \xB7 ${s.internal.orphans} orphans \xB7 ${s.internal.brokenInternal} broken`,
-    `**External:** ${s.external.totalLinks} links to ${s.external.uniqueDomains} domains`,
-    ``,
-    `### Inlink distribution`,
-    `- 0 inlinks (orphans): ${s.internal.distribution.zero}`,
-    `- 1\u20132: ${s.internal.distribution.oneToTwo}`,
-    `- 3\u201310: ${s.internal.distribution.threeToTen}`,
-    `- 11+: ${s.internal.distribution.elevenPlus}`,
-    ``,
-    `### Top 20 internal pages by inlinks`,
-    `| inlinks | out (int/ext) | URL |`,
-    `|---|---|---|`,
-    ...s.internal.topByInlinks.map((p) => `| ${p.inlinks} | ${p.outlinksInternal}/${p.outlinksExternal} | ${p.url} |`),
-    ``,
-    `### Top 20 external domains by links`,
-    `| links | nofollow | from pages | domain |`,
-    `|---|---|---|---|`,
-    ...s.external.topDomains.map((d) => `| ${d.links} | ${d.nofollow} | ${d.pages} | ${d.domain} |`)
-  ];
-  return lines.join("\n");
-}
-
-// src/api/seo-issues.ts
-var THIN_WORDS = 200;
-var TITLE_MAX = 60;
-var TITLE_PX_MAX = 561;
-var TITLE_MIN = 30;
-var META_MAX = 155;
-var META_MIN = 70;
-var H1_MAX = 70;
-var URL_MAX = 115;
-function dupes(pages, key) {
-  const groups = /* @__PURE__ */ new Map();
-  for (const p of pages) {
-    const k = key(p);
-    if (k === null || k === void 0 || k === "") continue;
-    const ks = String(k);
-    if (!groups.has(ks)) groups.set(ks, []);
-    groups.get(ks).push(p.url);
-  }
-  return [...groups.values()].filter((g) => g.length > 1).flat();
-}
-function computeIssues(pages, metrics, precomputedBrokenLinkPages) {
-  const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
-  const where = (fn) => pages.filter(fn).map((p) => p.url);
-  const extracted = (p) => p.extractionStatus !== "failed";
-  const ok = (p) => extracted(p) && p.status === 200;
-  const extractedUrls = new Set(pages.filter(extracted).map((page) => page.url));
-  const pathname = (u) => {
-    try {
-      return new URL(u).pathname;
-    } catch {
-      return "";
-    }
-  };
-  const report = {
-    "title.missing": group(where((p) => ok(p) && !p.title)),
-    "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
-    "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
-    "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
-    "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
-    "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
-    "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
-    "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
-    "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
-    "h1.missing": group(where((p) => ok(p) && !p.h1)),
-    "h1.multiple": group(where((p) => !!p.h1_2)),
-    "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
-    "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
-    "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
-    // A transport/extraction failure is not an SEO conclusion about the URL.
-    // Keep it in a dedicated crawl bucket so empty records cannot become a
-    // wall of false missing-title/H1/non-indexable findings.
-    "crawl.extractionFailed": group(where((p) => !extracted(p))),
-    "indexability.nonIndexable": group(where((p) => extracted(p) && !p.indexable)),
-    "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
-    "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
-    "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
-    "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
-    "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
-    "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
-    "response.noResponse": group(where((p) => p.status === null)),
-    "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
-    "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
-    "images.missingAlt": group(where((p) => extracted(p) && p.imagesMissingAlt > 0)),
-    "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
-    "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
-    "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
-    "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
-    "links.orphan": group([...metrics.values()].filter((m) => m.orphan && extractedUrls.has(m.url)).map((m) => m.url))
-  };
-  let brokenLinkPages = precomputedBrokenLinkPages;
-  if (!brokenLinkPages) {
-    const normUrl = (u) => {
-      try {
-        const x = new URL(u);
-        x.hash = "";
-        return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
-      } catch {
-        return u.replace(/#.*$/, "").replace(/\/+$/, "");
-      }
-    };
-    const statusByUrl = new Map(pages.map((p) => [normUrl(p.url), p.status]));
-    brokenLinkPages = /* @__PURE__ */ new Set();
-    for (const p of pages) {
-      for (const l of p.outlinks ?? []) {
-        if (!l.internal) continue;
-        const st = statusByUrl.get(normUrl(l.href));
-        if (st != null && st >= 400) brokenLinkPages.add(p.url);
-      }
-    }
-  }
-  report["links.brokenInternal"] = group([...brokenLinkPages]);
-  return report;
-}
-function renderIssueReport(siteUrl, pages, report, metrics) {
-  const total = pages.length;
-  const successful = pages.filter((page) => page.extractionStatus !== "failed").length;
-  const failed = total - successful;
-  const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
-  const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
-  const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
-  const maxDepth = depths.length ? Math.max(...depths) : 0;
-  const extractedUrls = new Set(pages.filter((page) => page.extractionStatus !== "failed").map((page) => page.url));
-  const orphans = [...metrics.values()].filter((m) => m.orphan && extractedUrls.has(m.url)).length;
-  const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
-  return [
-    `# SEO Crawl Report: ${siteUrl}`,
-    `**${total} pages attempted** \xB7 ${successful} extracted \xB7 ${failed} failed \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
-    `
-## Issues
-| | Issue | Count | Examples |
-|---|-------|-------|----------|
-${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
-    `
-## Most-linked pages
-${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
-    `
-_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
-  ].join("\n");
-}
-
-export {
-  auditImages,
-  auditImageUrls,
-  renderImageSection,
-  buildLinkReport,
-  renderLinkReport,
-  computeIssues,
-  renderIssueReport
-};