katman-mcp 0.1.0

package/dist/tools/crawl/audit/bots.js

@@ -0,0 +1,114 @@
+// AI-bot access test: fetch the homepage with each crawler's user agent and compare with a browser.
+// Spoofed user agents only exercise UA-based rules; IP-verified bot rules (e.g. Cloudflare's verified-bot
+// categories, "fake Googlebot" rules) can behave differently for the real crawler.
+import { BROWSER_UA, CRAWLERS } from "../../../crawlers.js";
+import { mapLimit } from "../../../net.js";
+export function detectCdn(h) {
+    const server = (h["server"] ?? "").toLowerCase();
+    const via = (h["via"] ?? "").toLowerCase();
+    if (h["cf-ray"] || server.includes("cloudflare"))
+        return "cloudflare";
+    if (h["x-vercel-id"] || server.includes("vercel"))
+        return "vercel";
+    if (h["x-nf-request-id"] || server.includes("netlify"))
+        return "netlify";
+    if (h["x-amz-cf-id"] || via.includes("cloudfront"))
+        return "cloudfront";
+    if (h["x-fastly-request-id"] || server.includes("fastly") || /^cache-[a-z]{3}/i.test(h["x-served-by"] ?? ""))
+        return "fastly";
+    if (server.includes("akamai") || h["akamai-grn"] || h["x-akamai-transformed"])
+        return "akamai";
+    if (h["x-sucuri-id"] || server.includes("sucuri"))
+        return "sucuri";
+    if (h["x-iinfo"] || (h["x-cdn"] ?? "").toLowerCase().includes("incapsula"))
+        return "imperva";
+    if (h["fly-request-id"] || server.startsWith("fly/"))
+        return "fly";
+    if (h["x-render-origin-server"] || server === "render")
+        return "render";
+    if (h["x-github-request-id"] || server.includes("github.com"))
+        return "github-pages";
+    if (h["x-railway-request-id"] || server.includes("railway"))
+        return "railway";
+    if (server.includes("bunnycdn"))
+        return "bunny";
+    if (h["x-firebase-hosting-version"] || server.includes("firebase"))
+        return "firebase";
+    return undefined;
+}
+const MARKERS = [
+    [/<title>\s*just a moment/i, 'Cloudflare challenge page ("Just a moment…")'],
+    [/attention required!\s*\|\s*cloudflare/i, 'Cloudflare block page ("Attention Required")'],
+    [/sorry, you have been blocked/i, "Cloudflare WAF block page"],
+    [/cf-chl-|cf_chl_opt|\/cdn-cgi\/challenge-platform/i, "Cloudflare challenge script"],
+    [/enable javascript and cookies to continue/i, "JavaScript + cookie challenge"],
+    [/checking (if the site connection is secure|your browser)/i, "browser check page"],
+    [/vercel security checkpoint/i, "Vercel Security Checkpoint"],
+    [/captcha-delivery\.com/i, "DataDome captcha"],
+    [/sucuri website firewall/i, "Sucuri firewall block"],
+    [/incapsula incident id|_incapsula_resource/i, "Imperva/Incapsula block"],
+    [/px-captcha|perimeterx/i, "HUMAN/PerimeterX challenge"],
+    [/awswaf|aws-waf-token/i, "AWS WAF challenge"],
+    [/<title>\s*access denied\s*<\/title>/i, "Access Denied page"],
+    [/ddos-guard/i, "DDoS-Guard check"],
+];
+/** Challenge/block signature in headers or the first 40 KB of the body. */
+export function challengeReason(res) {
+    const h = res.headers;
+    if (h["cf-mitigated"])
+        return `cf-mitigated: ${h["cf-mitigated"]}`;
+    if (h["x-vercel-mitigated"])
+        return `x-vercel-mitigated: ${h["x-vercel-mitigated"]}`;
+    if (h["x-amzn-waf-action"])
+        return `x-amzn-waf-action: ${h["x-amzn-waf-action"]}`;
+    const head = res.body.slice(0, 40_000);
+    for (const [re, label] of MARKERS)
+        if (re.test(head))
+            return label;
+    return undefined;
+}
+function kb(n) {
+    return n >= 1024 ? `${(n / 1024).toFixed(1)} KB` : `${n} B`;
+}
+export function classifyBotResponse(bot, browser) {
+    const browserOk = browser ? browser.status > 0 && browser.status < 400 && !challengeReason(browser) : true;
+    const tail = browser && !browserOk ? " (browsers are refused too)" : "";
+    if (bot.status === 0) {
+        return { blocked: browserOk, reason: `no response: ${bot.error ?? "connection error"}${tail}` };
+    }
+    const ch = challengeReason(bot);
+    if ([401, 403, 429, 503].includes(bot.status))
+        return { blocked: true, reason: `HTTP ${bot.status}${ch ? ` · ${ch}` : ""}${tail}` };
+    if (ch)
+        return { blocked: true, reason: `${ch}${tail}` };
+    if (bot.status >= 300 && bot.status < 400) {
+        const loc = bot.location ?? "";
+        if (/challenge|captcha|cdn-cgi|checkpoint/i.test(loc))
+            return { blocked: true, reason: `redirected to a challenge (${loc})` };
+        if (browser && (browser.status !== bot.status || (browser.location ?? "") !== loc)) {
+            return { blocked: false, different: true, reason: `HTTP ${bot.status} → ${loc || "?"} while browsers get HTTP ${browser.status}` };
+        }
+        return { blocked: false };
+    }
+    if (bot.status >= 400)
+        return { blocked: true, reason: `HTTP ${bot.status}${tail}` };
+    if (browser && browser.status === 200 && bot.status === 200 && browser.bytes > 5000 && bot.bytes < browser.bytes * 0.3) {
+        return { blocked: false, different: true, reason: `much smaller page than browsers get (${kb(bot.bytes)} vs ${kb(browser.bytes)})` };
+    }
+    return { blocked: false };
+}
+export async function runBotTest(url, fetcher, crawlers = CRAWLERS.filter((c) => c.testAccess), concurrency = 4) {
+    const headers = { "accept-encoding": "br, gzip" };
+    const browser = await fetcher(url, { ua: BROWSER_UA, redirect: "manual", headers });
+    const probes = await mapLimit(crawlers, Math.min(4, concurrency), async (crawler) => {
+        const res = await fetcher(url, { ua: crawler.ua, redirect: "manual", headers });
+        const v = classifyBotResponse(res, browser);
+        return { crawler, status: res.status, bytes: res.bytes, ms: res.ms, ...v };
+    });
+    const browserBlocked = browser.status === 0
+        ? `no response: ${browser.error ?? "error"}`
+        : [401, 403, 429, 503].includes(browser.status)
+            ? `HTTP ${browser.status}`
+            : challengeReason(browser);
+    return { url, browser, browserBlocked, probes, cdn: detectCdn(browser.headers) };
+}

package/dist/tools/crawl/audit/checks.js

@@ -0,0 +1,126 @@
+// Site-level HTTP checks (network through an injected fetcher): redirects, host canonicalisation, soft 404,
+// text files (llms.txt), preview copies and og:image reachability.
+import { mapLimit } from "../../../net.js";
+import { safeUrl, sameSite } from "../common/urls.js";
+import { headerNoindex } from "./html.js";
+import { looksLikeHtml } from "./robots.js";
+export const PAGE_HEADERS = { "accept-encoding": "br, gzip" };
+export async function followRedirects(url, fetcher, max = 5, opts = {}) {
+    const chain = [];
+    let current = url;
+    for (let i = 0; i <= max; i++) {
+        const res = await fetcher(current, { ...opts, redirect: "manual" });
+        const loc = res.status >= 300 && res.status < 400 && res.location ? safeUrl(res.location, current)?.href : undefined;
+        chain.push({ url: current, status: res.status, location: loc });
+        if (!loc || i === max || chain.some((h) => h.url === loc))
+            return { final: { ...res, finalUrl: current }, chain };
+        current = loc;
+    }
+    throw new Error("unreachable");
+}
+function redirectResult(from, res, expect) {
+    if (res.status === 0)
+        return { from, status: 0, error: res.error };
+    if (res.status >= 300 && res.status < 400) {
+        const to = res.location ? safeUrl(res.location, from) : undefined;
+        return { from, status: res.status, location: to?.href ?? res.location, ok: to ? expect(to) : false, permanent: [301, 308].includes(res.status) };
+    }
+    if (res.status >= 200 && res.status < 300)
+        return { from, status: res.status, ok: false, servesContent: true };
+    return { from, status: res.status, ok: false };
+}
+/** http://host/ must redirect (301/308) to https. */
+export async function checkHttpsRedirect(host, fetcher) {
+    const from = `http://${host}/`;
+    const res = await fetcher(from, { redirect: "manual", timeoutMs: 12000 });
+    return redirectResult(from, res, (to) => to.protocol === "https:");
+}
+/** www twin of an apex host (example.com ↔ www.example.com, example.com.tr ↔ www.example.com.tr); undefined for other subdomains. */
+export function wwwTwin(host) {
+    const h = host.toLowerCase().replace(/:\d+$/, "");
+    if (h.startsWith("www."))
+        return h.slice(4);
+    if (/^\d+\.\d+\.\d+\.\d+$/.test(h) || h === "localhost")
+        return undefined;
+    const labels = h.split(".");
+    const secondLevel = /^(co|com|net|org|gov|edu|ac|gen|web|info|biz|ltd|plc|me|or|ne|go)$/;
+    const apex = labels.length === 2 || (labels.length === 3 && labels[2].length === 2 && secondLevel.test(labels[1]));
+    return apex ? `www.${h}` : undefined;
+}
+/** The www / apex twin must redirect to the main host. */
+export async function checkHostRedirect(mainHost, fetcher) {
+    const altHost = wwwTwin(mainHost) ?? `www.${mainHost}`;
+    const from = `https://${altHost}/`;
+    const res = await fetcher(from, { redirect: "manual", timeoutMs: 12000 });
+    return { ...redirectResult(from, res, (to) => to.hostname.toLowerCase() === mainHost.toLowerCase()), altHost };
+}
+/** An address that cannot exist must answer 404/410. */
+export async function checkSoft404(origin, fetcher, token) {
+    const url = `${origin}/katman-check-404-${token}`;
+    const res = await fetcher(url, { redirect: "manual", headers: PAGE_HEADERS });
+    const noindex = headerNoindex(res.headers) || /<meta[^>]+name=["'](?:robots|googlebot)["'][^>]+content=["'][^"']*noindex/i.test(res.body);
+    return {
+        url,
+        status: res.status,
+        location: res.location ? safeUrl(res.location, url)?.href : undefined,
+        noindex,
+        soft404: res.status >= 200 && res.status < 300,
+        error: res.error,
+    };
+}
+export async function checkTextFile(url, fetcher) {
+    const res = await fetcher(url, { redirect: "follow", maxBytes: 2_000_000, headers: { accept: "text/plain,text/markdown,*/*" } });
+    const isHtml = res.status === 200 && looksLikeHtml(res.body, res.headers["content-type"]);
+    return {
+        url,
+        status: res.status,
+        exists: res.status === 200 && !isHtml && res.body.trim().length > 0,
+        isHtml,
+        bytes: res.bytes,
+        text: isHtml ? "" : res.body,
+        contentType: res.headers["content-type"],
+    };
+}
+/** A preview copy passes when it 301s to the main host, sends noindex, is password-protected, or is gone. */
+export async function checkPreview(url, mainHost, fetcher, mainTitle) {
+    const res = await fetcher(url, { redirect: "manual", headers: PAGE_HEADERS });
+    if (res.status === 0)
+        return { url, status: 0, ok: true, how: `not reachable (${res.error ?? "error"})` };
+    if (res.status >= 300 && res.status < 400) {
+        const to = res.location ? safeUrl(res.location, url) : undefined;
+        const toMain = !!to && sameSite(to.hostname, mainHost) && to.hostname.toLowerCase() === mainHost.toLowerCase();
+        const permanent = [301, 308].includes(res.status);
+        return {
+            url,
+            status: res.status,
+            location: to?.href,
+            ok: toMain && permanent,
+            how: toMain ? (permanent ? `${res.status} → main host` : `${res.status} (temporary) → main host: use 301`) : `redirects to ${to?.href ?? "?"}`,
+        };
+    }
+    if (res.status === 401 || res.status === 403)
+        return { url, status: res.status, ok: true, how: `password/access protected (${res.status})` };
+    if (res.status >= 400)
+        return { url, status: res.status, ok: true, how: `not serving the site (${res.status})` };
+    const title = res.body.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim();
+    const sameTitle = mainTitle && title ? title === mainTitle : undefined;
+    const noindex = headerNoindex(res.headers) || /<meta[^>]+name=["'](?:robots|googlebot)["'][^>]+content=["'][^"']*noindex/i.test(res.body);
+    return {
+        url,
+        status: res.status,
+        ok: noindex,
+        how: noindex ? "200 with noindex" : `200 without noindex${sameTitle ? " (same <title> as the main site: a live copy)" : ""}`,
+        sameTitle,
+    };
+}
+export async function checkImages(urls, fetcher, max = 6) {
+    const out = new Map();
+    const list = [...new Set(urls)].slice(0, max);
+    const res = await mapLimit(list, 4, (u) => fetcher(u, { redirect: "follow", maxBytes: 4096, headers: { accept: "image/*,*/*" } }));
+    list.forEach((u, i) => {
+        const r = res[i];
+        const ct = r.headers["content-type"];
+        out.set(u, { status: r.status, contentType: ct, ok: r.status === 200 && (!ct || /^image\//i.test(ct)) });
+    });
+    return out;
+}

package/dist/tools/crawl/audit/html.js

@@ -0,0 +1,382 @@
+// What a crawler that does not run JavaScript sees in a page's raw HTML. Pure: (html, url) → facts.
+import { parse, HTMLElement } from "node-html-parser";
+import { collapse, countWords, decodeEntities } from "../common/text.js";
+import { safeUrl, sameSite, urlKey } from "../common/urls.js";
+const SKIP_TEXT = new Set(["script", "style", "template", "svg", "noscript", "head", "iframe", "object", "canvas", "math", "title"]);
+const P_SKIP_TAGS = new Set(["nav", "footer", "aside", "form", "dialog", "button", "label", "select", "figcaption"]);
+const P_SKIP_ATTR = /cookie|consent|gdpr|announcement|skip-link|breadcrumb|newsletter|banner-top/i;
+function tagOf(n) {
+    return n instanceof HTMLElement ? (n.rawTagName ?? "").toLowerCase() : "";
+}
+function attr(el, k) {
+    const v = el.getAttribute(k);
+    return v === undefined ? undefined : v;
+}
+function textOf(el) {
+    const out = [];
+    walkText(el, out, undefined);
+    return collapse(decodeEntities(out.join(" ")));
+}
+function walkText(node, out, st) {
+    for (const child of node.childNodes) {
+        if (child.nodeType === 3) {
+            out.push(child.rawText);
+            continue;
+        }
+        if (!(child instanceof HTMLElement))
+            continue;
+        const tag = tagOf(child);
+        if (SKIP_TEXT.has(tag))
+            continue;
+        if (child.getAttribute("hidden") !== undefined)
+            continue;
+        walkText(child, out, st);
+        if (st && !st.h1Seen && tag === "h1") {
+            st.h1Seen = true;
+            st.afterH1Start = out.length;
+        }
+    }
+}
+function parseJsonLd(raw) {
+    const cleaned = raw
+        .trim()
+        .replace(/^<!--/, "")
+        .replace(/-->$/, "")
+        .replace(/^\s*\/\/\s*<!\[CDATA\[/, "")
+        .replace(/\/\/\s*\]\]>\s*$/, "")
+        .trim();
+    return JSON.parse(cleaned);
+}
+function typeName(t) {
+    return t.replace(/^https?:\/\/schema\.org\//i, "").trim();
+}
+function collectTypes(node, out, dates, depth = 0) {
+    if (Array.isArray(node)) {
+        for (const n of node)
+            collectTypes(n, out, dates, depth);
+        return;
+    }
+    if (!node || typeof node !== "object")
+        return;
+    const o = node;
+    const t = o["@type"];
+    if (typeof t === "string")
+        out.add(typeName(t));
+    else if (Array.isArray(t))
+        for (const x of t)
+            if (typeof x === "string")
+                out.add(typeName(x));
+    if (typeof o["dateModified"] === "string")
+        dates.push(o["dateModified"]);
+    if (depth === 0 && Array.isArray(o["@graph"]))
+        for (const g of o["@graph"])
+            collectTypes(g, out, dates, 1);
+}
+const MIXED_SRC = ["img", "script", "iframe", "source", "video", "audio", "embed", "track", "input"];
+export function analyzeHtml(html, pageUrl) {
+    const root = parse(html, {
+        comment: false,
+        lowerCaseTagName: true,
+        blockTextElements: { script: true, noscript: true, style: true },
+    });
+    const page = safeUrl(pageUrl);
+    const host = page?.hostname ?? "";
+    const https = page?.protocol === "https:";
+    // Titles (ignore <title> inside inline SVG)
+    const titles = root.querySelectorAll("title").filter((t) => !t.closest("svg"));
+    const title = titles[0] ? collapse(decodeEntities(titles[0].rawText)) : undefined;
+    // Meta tags
+    let description;
+    let descriptionCount = 0;
+    let metaRobots;
+    let viewport = false;
+    let twitterCard;
+    let generator;
+    const og = {};
+    for (const m of root.querySelectorAll("meta")) {
+        const name = (attr(m, "name") ?? "").toLowerCase().trim();
+        const prop = (attr(m, "property") ?? "").toLowerCase().trim();
+        const content = collapse(decodeEntities(attr(m, "content") ?? ""));
+        if (name === "description") {
+            descriptionCount++;
+            if (description === undefined)
+                description = content;
+        }
+        else if (name === "robots" || name === "googlebot") {
+            metaRobots = metaRobots ? `${metaRobots}, ${content}` : content;
+        }
+        else if (name === "viewport")
+            viewport = true;
+        else if (name === "generator")
+            generator = content;
+        const key = prop || name;
+        if (key === "twitter:card")
+            twitterCard = content;
+        if (key.startsWith("og:")) {
+            if (key === "og:title")
+                og.title ??= content;
+            else if (key === "og:description")
+                og.description ??= content;
+            else if (key === "og:image" || key === "og:image:url" || key === "og:image:secure_url")
+                og.image ??= content;
+            else if (key === "og:image:alt")
+                og.imageAlt ??= content;
+            else if (key === "og:image:width")
+                og.imageWidth ??= content;
+            else if (key === "og:url")
+                og.url ??= content;
+            else if (key === "og:locale")
+                og.locale ??= content;
+        }
+    }
+    const robotsLc = (metaRobots ?? "").toLowerCase();
+    const noindexMeta = /\bnoindex\b|\bnone\b/.test(robotsLc);
+    const nosnippet = /\bnosnippet\b/.test(robotsLc);
+    // Links in <head>: canonical, hreflang
+    let canonicalRaw;
+    let canonicalCount = 0;
+    const hreflang = [];
+    for (const l of root.querySelectorAll("link")) {
+        const rel = (attr(l, "rel") ?? "").toLowerCase().split(/\s+/);
+        const href = attr(l, "href");
+        if (!href)
+            continue;
+        if (rel.includes("canonical")) {
+            canonicalCount++;
+            canonicalRaw ??= href.trim();
+        }
+        const hl = attr(l, "hreflang");
+        if (rel.includes("alternate") && hl) {
+            const abs = safeUrl(href.trim(), pageUrl)?.href ?? href.trim();
+            hreflang.push({ lang: hl.trim(), href: abs });
+        }
+    }
+    const canonical = canonicalRaw ? (safeUrl(canonicalRaw, pageUrl)?.href ?? canonicalRaw) : undefined;
+    // Headings
+    const headings = root.querySelectorAll("h1, h2, h3, h4, h5, h6");
+    const h1s = headings.filter((h) => tagOf(h) === "h1");
+    const h1Text = h1s[0] ? textOf(h1s[0]) || undefined : undefined;
+    let headingSkips = 0;
+    const headingSkipSamples = [];
+    let prev = 0;
+    for (const h of headings) {
+        const lvl = Number(tagOf(h).slice(1));
+        if (prev && lvl > prev + 1) {
+            headingSkips++;
+            if (headingSkipSamples.length < 3)
+                headingSkipSamples.push(`h${prev}→h${lvl} "${textOf(h).slice(0, 40)}"`);
+        }
+        prev = lvl;
+    }
+    // Visible text of <body> (no JS): skip script/style/template/svg/noscript
+    const body = root.querySelector("body") ?? root;
+    const out = [];
+    const st = { h1Seen: false, afterH1Start: -1 };
+    walkText(body, out, st);
+    const bodyText = collapse(decodeEntities(out.join(" ")));
+    const wordsJsOff = countWords(bodyText);
+    const textAfterH1 = st.h1Seen ? collapse(decodeEntities(out.slice(st.afterH1Start).join(" "))).slice(0, 400) : undefined;
+    // First meaningful paragraph (≥40 chars), preferring <main>/<article>, skipping nav/footer/cookie banners
+    const scope = root.querySelector("main") ?? root.querySelector("article") ?? body;
+    let firstParagraph;
+    for (const p of scope.querySelectorAll("p")) {
+        if (excludedParagraph(p))
+            continue;
+        const t = textOf(p);
+        if (t.length >= 40) {
+            firstParagraph = t;
+            break;
+        }
+    }
+    if (!firstParagraph && scope !== body) {
+        for (const p of body.querySelectorAll("p")) {
+            if (excludedParagraph(p))
+                continue;
+            const t = textOf(p);
+            if (t.length >= 40) {
+                firstParagraph = t;
+                break;
+            }
+        }
+    }
+    // Heuristic: an answer sentence (≥30 chars, ending in . ! ?) or the first paragraph starts within ~200 characters
+    // after the H1.
+    let answerNearH1;
+    if (textAfterH1 !== undefined) {
+        const probe = firstParagraph?.slice(0, 40);
+        const at = probe ? textAfterH1.indexOf(probe) : -1;
+        const sentence = textAfterH1.search(/[\p{L}\p{N}][^.!?]{29,}[.!?](\s|$)/u);
+        answerNearH1 = (at >= 0 && at <= 200) || (sentence >= 0 && sentence <= 200);
+    }
+    else if (h1Text)
+        answerNearH1 = false;
+    // JSON-LD
+    let jsonLdBlocks = 0;
+    let jsonLdErrors = 0;
+    const jsonLdErrorSamples = [];
+    const types = new Set();
+    const dates = [];
+    let scriptCount = 0;
+    for (const s of root.querySelectorAll("script"))
+        if (attr(s, "src"))
+            scriptCount++;
+    // JSON-LD straight from the raw HTML: node-html-parser's script rawText can gain a stray "<" when the JSON holds
+    // non-ASCII text, which would report valid JSON-LD as broken.
+    for (const m of html.matchAll(/<script\b([^>]*)>([\s\S]*?)<\/script\s*>/gi)) {
+        const type = (m[1].match(/\btype\s*=\s*["']?([^"'\s>]+)/i)?.[1] ?? "").toLowerCase();
+        if (type !== "application/ld+json")
+            continue;
+        jsonLdBlocks++;
+        try {
+            collectTypes(parseJsonLd(m[2]), types, dates);
+        }
+        catch (e) {
+            jsonLdErrors++;
+            if (jsonLdErrorSamples.length < 2)
+                jsonLdErrorSamples.push(e instanceof Error ? e.message.slice(0, 120) : String(e));
+        }
+    }
+    let dateModified = dates[0];
+    if (!dateModified) {
+        for (const m of root.querySelectorAll("meta")) {
+            const k = (attr(m, "property") ?? attr(m, "name") ?? "").toLowerCase();
+            if (k === "article:modified_time" || k === "og:updated_time") {
+                dateModified = attr(m, "content");
+                break;
+            }
+        }
+    }
+    // Images
+    const imgs = body.querySelectorAll("img");
+    let imagesMissingAlt = 0;
+    let imagesMissingSize = 0;
+    for (const img of imgs) {
+        if (img.getAttribute("alt") === undefined)
+            imagesMissingAlt++;
+        if (img.getAttribute("width") === undefined || img.getAttribute("height") === undefined)
+            imagesMissingSize++;
+    }
+    // Links
+    const links = [];
+    const seenKeys = new Set();
+    let externalLinks = 0;
+    for (const a of root.querySelectorAll("a")) {
+        const raw = (attr(a, "href") ?? "").trim();
+        if (!raw || raw.startsWith("#") || /^(mailto|tel|javascript|data|sms|whatsapp):/i.test(raw))
+            continue;
+        const u = safeUrl(decodeEntities(raw), pageUrl);
+        if (!u || !/^https?:$/.test(u.protocol))
+            continue;
+        if (!sameSite(u.hostname, host)) {
+            externalLinks++;
+            continue;
+        }
+        u.hash = "";
+        const key = urlKey(u);
+        if (seenKeys.has(key))
+            continue;
+        seenKeys.add(key);
+        links.push({
+            href: u.href,
+            key,
+            text: textOf(a).slice(0, 80),
+            nofollow: /\bnofollow\b/i.test(attr(a, "rel") ?? ""),
+            offHost: u.hostname.toLowerCase() !== host.toLowerCase() || (page ? u.protocol !== page.protocol : false),
+        });
+    }
+    // Mixed content (http:// sub-resources on an https page)
+    const mixedContent = [];
+    if (https) {
+        const push = (v) => {
+            if (v && /^http:\/\//i.test(v.trim()) && mixedContent.length < 20)
+                mixedContent.push(v.trim());
+        };
+        for (const tag of MIXED_SRC)
+            for (const el of root.querySelectorAll(tag))
+                push(attr(el, "src"));
+        for (const el of root.querySelectorAll("link")) {
+            const rel = (attr(el, "rel") ?? "").toLowerCase();
+            if (/stylesheet|preload|icon|modulepreload|manifest/.test(rel))
+                push(attr(el, "href"));
+        }
+        for (const el of root.querySelectorAll("source, img")) {
+            for (const part of (attr(el, "srcset") ?? "").split(","))
+                push(part.trim().split(/\s+/)[0]);
+        }
+        for (const el of root.querySelectorAll("object"))
+            push(attr(el, "data"));
+        for (const el of root.querySelectorAll("video"))
+            push(attr(el, "poster"));
+    }
+    // Client-rendered shell: an empty mount point and almost no body text
+    let rootShell = false;
+    for (const sel of ["#root", "#app", "#__next", "#__nuxt", "#svelte", "#q-app", "[data-reactroot]"]) {
+        const el = root.querySelector(sel);
+        if (el && countWords(textOf(el)) <= 5) {
+            rootShell = wordsJsOff < 100;
+            break;
+        }
+    }
+    return {
+        title,
+        titleCount: titles.length,
+        description,
+        descriptionCount,
+        canonical,
+        canonicalRaw,
+        canonicalCount,
+        metaRobots,
+        noindexMeta,
+        nosnippet,
+        h1Count: h1s.length,
+        h1Text,
+        headingSkips,
+        headingSkipSamples,
+        wordsJsOff,
+        firstParagraph,
+        answerNearH1,
+        textAfterH1,
+        jsonLdBlocks,
+        jsonLdTypes: [...types].sort(),
+        jsonLdErrors,
+        jsonLdErrorSamples,
+        dateModified,
+        og,
+        twitterCard,
+        lang: attr(root.querySelector("html") ?? root, "lang")?.trim() || undefined,
+        hreflang,
+        imagesTotal: imgs.length,
+        imagesMissingAlt,
+        imagesMissingSize,
+        links,
+        externalLinks,
+        mixedContent,
+        viewport,
+        charsetOffset: html.search(/<meta[^>]+charset/i),
+        rootShell,
+        scriptCount,
+        generator,
+        cfEmailObfuscation: /__cf_email__|\/cdn-cgi\/l\/email-protection/.test(html),
+    };
+}
+function excludedParagraph(p) {
+    let el = p.parentNode;
+    while (el && el.rawTagName) {
+        const tag = el.rawTagName.toLowerCase();
+        if (P_SKIP_TAGS.has(tag))
+            return true;
+        const role = (el.getAttribute("role") ?? "").toLowerCase();
+        if (role === "navigation" || role === "contentinfo" || role === "dialog")
+            return true;
+        const idc = `${el.getAttribute("id") ?? ""} ${el.getAttribute("class") ?? ""}`;
+        if (P_SKIP_ATTR.test(idc))
+            return true;
+        el = el.parentNode;
+    }
+    return false;
+}
+/** noindex from the X-Robots-Tag header (any bot-specific variant counts). */
+export function headerNoindex(headers) {
+    const v = (headers["x-robots-tag"] ?? "").toLowerCase();
+    return /\bnoindex\b|\bnone\b/.test(v);
+}