@fulldotdev/scan 0.1.0

package/dist/engine/html.js

@@ -0,0 +1,645 @@
+import * as cheerio from "cheerio";
+import { createHash } from "node:crypto";
+import { marked } from "marked";
+import { normalizeUrl } from "./network.js";
+import { httpEvidence } from "./types.js";
+const clean = (value) => (value ?? "").replace(/\s+/g, " ").trim();
+const hash = (value) => createHash("sha256").update(value).digest("hex");
+export function resolveLink(value, base) {
+    try {
+        return value ? normalizeUrl(value, base) : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function markdownLinks(body, base) {
+    const links = [];
+    const tokens = marked.lexer(body);
+    marked.walkTokens(tokens, (token) => {
+        if (token.type === "link" || token.type === "image")
+            links.push({ text: token.text, url: resolveLink(token.href, base) });
+    });
+    return links;
+}
+const nextText = /^(next|volgende|weiter|suivant|siguiente|próxima|proxima|avanti|»|›|>|→|next page|older( posts)?|more)$/i;
+const prevText = /^(previous|prev|vorige|zurück|précédent|anterior|indietro|«|‹|<|←|newer( posts)?)$/i;
+export const loadMoreText = /^(load more|show more|view more|see more|more (products|items|results|posts)|meer laden|toon meer|laad meer|meer (producten|artikelen|resultaten)|mehr laden|mehr anzeigen|voir plus|afficher plus|charger plus|ver más|cargar más|mostra altri|carica altro)$/i;
+const inStockText = /\b(in stock|in voorraad|op voorraad|auf lager|lieferbar|en stock|disponible|available now|ships? (today|within)|leverbaar|direct leverbaar|beschikbaar|verfügbar)\b/i;
+const outOfStockText = /\b(out of stock|sold out|uitverkocht|niet (op|in) voorraad|niet leverbaar|nicht (verfügbar|lieferbar|auf lager)|ausverkauft|épuisé|rupture de stock|agotado|esaurito|currently unavailable|tijdelijk uitverkocht)\b/i;
+const currencySymbols = {
+    "€": "EUR",
+    // A bare dollar sign does not identify a currency.
+    $: "$",
+    "£": "GBP",
+    "¥": "JPY",
+    "₹": "INR",
+    zł: "PLN",
+    Kč: "CZK",
+    kr: "SEK/NOK/DKK",
+    CHF: "CHF",
+    EUR: "EUR",
+    USD: "USD",
+    GBP: "GBP",
+    CAD: "CAD",
+    AUD: "AUD",
+    SEK: "SEK",
+    NOK: "NOK",
+    DKK: "DKK",
+    PLN: "PLN",
+    CZK: "CZK",
+    JPY: "JPY",
+};
+const priceRegex = /(?:(€|\$|£|¥|₹|CHF|EUR|USD|GBP|CAD|AUD|SEK|NOK|DKK|PLN|CZK|JPY)\s?(\d{1,3}(?:[.,\s]\d{3})*(?:[.,]\d{1,2})?|\d+(?:[.,]\d{1,2})?)(?:,-)?)|(?:(\d{1,3}(?:[.,\s]\d{3})*(?:[.,]\d{1,2})?|\d+(?:[.,]\d{1,2})?)(?:,-)?\s?(€|£|EUR|USD|GBP|kr|zł|Kč|CHF|SEK|NOK|DKK|PLN|CZK))/g;
+export function parseAmount(raw) {
+    const text = raw.replace(/\s/g, "").replace(/,-$/, "");
+    const lastComma = text.lastIndexOf(","), lastDot = text.lastIndexOf(".");
+    let normalized;
+    if (lastComma > lastDot) {
+        // 1.234,56 or 12,50
+        normalized =
+            text.length - lastComma - 1 <= 2
+                ? text.replace(/\./g, "").replace(",", ".")
+                : text.replace(/,/g, "");
+    }
+    else if (lastDot > lastComma) {
+        normalized =
+            text.length - lastDot - 1 <= 2
+                ? text.replace(/,/g, "")
+                : text.replace(/\./g, "");
+    }
+    else
+        normalized = text;
+    const value = Number(normalized);
+    return Number.isFinite(value) ? value : null;
+}
+// Visible prices: currency-marked amounts in the main text. Heuristic; a
+// listing page carries many, a product page usually one or a few.
+export function visiblePrices(text) {
+    const prices = [];
+    for (const match of text.matchAll(priceRegex)) {
+        if (prices.length >= 100)
+            break;
+        const symbol = match[1] ?? match[4];
+        const amount = parseAmount(match[2] ?? match[3]);
+        prices.push({
+            raw: match[0].trim(),
+            currency: currencySymbols[symbol] ?? symbol,
+            amount,
+        });
+    }
+    return prices;
+}
+export function extractPage(response, representation = "original-response-html") {
+    const $ = cheerio.load(response.body);
+    const base = resolveLink($("base[href]").first().attr("href"), response.finalUrl) ??
+        response.finalUrl;
+    const meta = $("meta")
+        .toArray()
+        .map((el) => ({
+        name: $(el).attr("name"),
+        property: $(el).attr("property"),
+        httpEquiv: $(el).attr("http-equiv"),
+        content: $(el).attr("content") ?? "",
+    }));
+    const declarations = $("link")
+        .toArray()
+        .map((el) => ({
+        rel: $(el).attr("rel") ?? "",
+        href: resolveLink($(el).attr("href"), base),
+        rawHref: $(el).attr("href"),
+        type: $(el).attr("type"),
+        hreflang: $(el).attr("hreflang"),
+        sizes: $(el).attr("sizes"),
+        fetchpriority: $(el).attr("fetchpriority"),
+    }));
+    const inPagination = (el) => $(el).closest('nav[aria-label*="pagin" i],nav[class*="pagin" i],[class*="pagination" i],[class*="pager" i],[role="navigation"][class*="page" i],.paginate,.page-numbers').length > 0;
+    const links = $("a[href],area[href]")
+        .toArray()
+        .map((el) => {
+        const raw = $(el).attr("href");
+        let fragment = null;
+        try {
+            fragment = new URL(raw, base).hash.slice(1) || null;
+        }
+        catch { }
+        const text = clean($(el).text()).slice(0, 500);
+        const imageAlt = clean($(el)
+            .find("img[alt]")
+            .toArray()
+            .map((img) => $(img).attr("alt"))
+            .join(" "));
+        return {
+            href: resolveLink(raw, base),
+            rawHref: raw,
+            text,
+            // Accessible name fallback order: text, aria-label, image alt, title.
+            name: text ||
+                clean($(el).attr("aria-label")) ||
+                imageAlt ||
+                clean($(el).attr("title")) ||
+                "",
+            rel: $(el).attr("rel") ?? "",
+            target: $(el).attr("target") ?? "",
+            hreflang: $(el).attr("hreflang") || undefined,
+            fragment,
+            location: $(el).closest("header,nav,footer,aside").get(0)?.tagName ?? "content",
+            pagination: inPagination(el) || undefined,
+        };
+    });
+    const images = $("img")
+        .toArray()
+        .map((el) => ({
+        src: resolveLink($(el).attr("src"), base),
+        alt: $(el).attr("alt") ?? null,
+        width: $(el).attr("width"),
+        height: $(el).attr("height"),
+        loading: $(el).attr("loading"),
+        fetchpriority: $(el).attr("fetchpriority"),
+        srcset: $(el).attr("srcset"),
+        sources: $(el)
+            .parent("picture")
+            .children("source")
+            .toArray()
+            .map((source) => ({
+            srcset: $(source).attr("srcset"),
+            sizes: $(source).attr("sizes"),
+            type: $(source).attr("type"),
+            media: $(source).attr("media"),
+        })),
+        sizes: $(el).attr("sizes"),
+    }));
+    const scripts = $("script[src]")
+        .toArray()
+        .map((el) => ({
+        src: resolveLink($(el).attr("src"), base),
+        type: $(el).attr("type"),
+        async: $(el).attr("async") !== undefined,
+        defer: $(el).attr("defer") !== undefined,
+    }));
+    const styles = declarations.filter((l) => l.rel.split(/\s+/).includes("stylesheet"));
+    const forms = $("form")
+        .toArray()
+        .map((el, index) => {
+        const form = $(el);
+        const formId = form.attr("id");
+        const controls = $("input,select,textarea,button")
+            .toArray()
+            .filter((control) => {
+            const owner = $(control).attr("form");
+            return owner
+                ? owner === formId
+                : $(control).closest("form").get(0) === el;
+        })
+            .map((control) => {
+            const field = $(control);
+            const id = field.attr("id");
+            const labelIds = (field.attr("aria-labelledby") ?? "")
+                .split(/\s+/)
+                .filter(Boolean);
+            const labels = $("label")
+                .toArray()
+                .filter((label) => id && $(label).attr("for") === id)
+                .map((label) => clean($(label).text()));
+            const wrapped = clean(field.closest("label").text());
+            if (wrapped)
+                labels.push(wrapped);
+            const referencedLabels = labelIds.map((id) => clean($("[id]")
+                .filter((_, n) => $(n).attr("id") === id)
+                .text()));
+            return {
+                tag: control.tagName,
+                id,
+                name: field.attr("name"),
+                type: field.attr("type") ??
+                    (control.tagName === "button" ? "submit" : "text"),
+                labels,
+                ariaLabel: field.attr("aria-label"),
+                referencedLabels,
+                hasAccessibleName: !!(labels.some(Boolean) ||
+                    field.attr("aria-label") ||
+                    referencedLabels.some(Boolean) ||
+                    (control.tagName === "button" && clean(field.text())) ||
+                    (["submit", "reset", "button"].includes(field.attr("type") ?? "") &&
+                        field.attr("value"))),
+                required: field.attr("required") !== undefined,
+                disabled: field.attr("disabled") !== undefined,
+                autocomplete: field.attr("autocomplete"),
+                pattern: field.attr("pattern"),
+                min: field.attr("min"),
+                max: field.attr("max"),
+                minLength: field.attr("minlength"),
+                maxLength: field.attr("maxlength"),
+                actionOverride: resolveLink(field.attr("formaction"), base),
+                methodOverride: field.attr("formmethod"),
+            };
+        });
+        const action = resolveLink(form.attr("action") || response.finalUrl, base);
+        return {
+            index,
+            id: formId,
+            action,
+            rawAction: form.attr("action") ?? null,
+            method: (form.attr("method") ?? "get").toLowerCase(),
+            noValidate: form.attr("novalidate") !== undefined,
+            enctype: form.attr("enctype") ?? "application/x-www-form-urlencoded",
+            controls,
+            insecureDestination: action?.startsWith("http:") ?? false,
+            verification: "markup-only; no submission performed; JavaScript handlers not verified",
+        };
+    });
+    const structuredData = $('script[type="application/ld+json"]')
+        .toArray()
+        .map((el, index) => {
+        const raw = $(el).text();
+        try {
+            const data = JSON.parse(raw);
+            const errors = [];
+            const check = (node, path) => {
+                if (Array.isArray(node)) {
+                    node.forEach((n, i) => check(n, `${path}[${i}]`));
+                    return;
+                }
+                if (!node || typeof node !== "object") {
+                    errors.push(`${path}: expected object or array`);
+                    return;
+                }
+                if (node["@type"] !== undefined &&
+                    !(typeof node["@type"] === "string" ||
+                        (Array.isArray(node["@type"]) &&
+                            node["@type"].every((t) => typeof t === "string"))))
+                    errors.push(`${path}.@type: expected string or string array`);
+                if (node["@id"] !== undefined && typeof node["@id"] !== "string")
+                    errors.push(`${path}.@id: expected string`);
+                if (node["@graph"])
+                    check(node["@graph"], `${path}.@graph`);
+            };
+            check(data, "$");
+            return {
+                index,
+                raw,
+                data,
+                parseValid: true,
+                structuralErrors: errors,
+                validationScope: "JSON syntax and JSON-LD @type/@id/@graph shapes only; not Google rich-result eligibility",
+            };
+        }
+        catch (error) {
+            return { index, raw, parseValid: false, error: String(error) };
+        }
+    });
+    const microdata = $("[itemscope],[itemprop]")
+        .toArray()
+        .map((el) => ({
+        tag: el.tagName,
+        type: $(el).attr("itemtype"),
+        property: $(el).attr("itemprop"),
+        id: $(el).attr("itemid"),
+        content: $(el).attr("content") ??
+            $(el).attr("href") ??
+            clean($(el).text()).slice(0, 1000),
+    }));
+    const rdfa = $("[typeof],[property]")
+        .not("meta")
+        .toArray()
+        .map((el) => ({
+        tag: el.tagName,
+        type: $(el).attr("typeof"),
+        property: $(el).attr("property"),
+        resource: $(el).attr("resource"),
+        content: $(el).attr("content") ?? clean($(el).text()).slice(0, 1000),
+    }));
+    const headings = $("h1,h2,h3,h4,h5,h6")
+        .toArray()
+        .map((el) => ({
+        level: Number(el.tagName.slice(1)),
+        text: clean($(el).text()),
+    }));
+    const ids = $("[id],a[name]")
+        .toArray()
+        .map((el) => $(el).attr("id") ?? $(el).attr("name"))
+        .filter(Boolean);
+    const body = $("body").clone();
+    body.find("script,style,noscript,template,svg").remove();
+    body.find("p,div,section,article,li,h1,h2,h3,h4,h5,h6,br,td,th").append(" ");
+    const bodyText = clean(body.text());
+    const content = $("main,article").first().clone();
+    content.find("script,style,noscript,template,nav,footer,header,svg").remove();
+    content
+        .find("p,div,section,article,li,h1,h2,h3,h4,h5,h6,br,td,th")
+        .append(" ");
+    const mainText = clean(content.text()) || bodyText;
+    // Plain readability facts, the deterministic subset of what content
+    // plugins check: paragraph and sentence length.
+    const paragraphText = (content.length ? content : body)
+        .find("p")
+        .toArray()
+        .map((el) => clean($(el).text()))
+        .filter(Boolean);
+    const paragraphs = paragraphText
+        .map((text) => text.split(/\s+/).filter(Boolean).length)
+        .filter((n) => n > 0)
+        .slice(0, 500);
+    const sentences = paragraphText
+        .flatMap((text) => text.split(/(?<=[.!?])\s+/))
+        .map((s) => s.split(/\s+/).filter(Boolean).length)
+        .filter((n) => n > 0);
+    const readability = {
+        paragraphs: paragraphs.length,
+        longestParagraphWords: paragraphs.length ? Math.max(...paragraphs) : 0,
+        paragraphsOver150: paragraphs.filter((n) => n > 150).length,
+        sentences: sentences.length,
+        longSentenceShare: sentences.length
+            ? Math.round((sentences.filter((n) => n > 20).length / sentences.length) * 100) / 100
+            : 0,
+    };
+    const metaValues = (name) => meta.filter((m) => m.name?.toLowerCase() === name).map((m) => m.content);
+    // Pagination: rel=next/prev declarations and anchors, numbered links in a
+    // pagination container, and next/previous link text.
+    const relLink = (rel) => declarations.find((l) => l.rel.split(/\s+/).includes(rel))?.href ??
+        links.find((l) => l.rel.split(/\s+/).includes(rel))?.href ??
+        null;
+    const numbered = [
+        ...new Set(links
+            .filter((l) => l.href &&
+            l.pagination &&
+            /^\d{1,4}$/.test(l.text) &&
+            l.href !== response.finalUrl)
+            .map((l) => l.href)),
+    ];
+    const next = relLink("next") ??
+        links.find((l) => l.href && nextText.test(l.text))?.href ??
+        null;
+    const prev = relLink("prev") ??
+        links.find((l) => l.href && prevText.test(l.text))?.href ??
+        null;
+    const pagination = next || prev || numbered.length
+        ? {
+            next,
+            prev,
+            numbered: numbered.slice(0, 50),
+            source: relLink("next")
+                ? "rel-next"
+                : numbered.length
+                    ? "pagination-container"
+                    : "link-text",
+        }
+        : null;
+    const loadMore = $("button,a[href],[role='button']")
+        .toArray()
+        .filter((el) => loadMoreText.test(clean($(el).text())) && !$(el).closest("form").length)
+        .slice(0, 3)
+        .map((el) => ({
+        tag: el.tagName,
+        text: clean($(el).text()),
+        href: resolveLink($(el).attr("href"), base),
+    }));
+    const phones = [
+        ...new Set([
+            ...links
+                .filter((l) => /^tel:/i.test(l.rawHref))
+                .map((l) => l.rawHref.replace(/^tel:/i, "")),
+            // Text numbers must look dialable: an international prefix or a
+            // leading zero, so registration and account numbers are left alone.
+            ...[...bodyText.matchAll(/(?:\+|\b0)\(?\d[\d\s().-]{6,18}\d/g)]
+                .map((m) => m[0].trim())
+                .filter((p) => {
+                const digits = p.replace(/\D/g, "");
+                return /^(\+|00)/.test(p)
+                    ? digits.length >= 10 && digits.length <= 15
+                    : digits.length >= 9 && digits.length <= 11;
+            }),
+        ]),
+    ].slice(0, 20);
+    const emails = [
+        ...new Set([
+            ...links
+                .filter((l) => /^mailto:/i.test(l.rawHref))
+                .map((l) => l.rawHref.replace(/^mailto:/i, "").split("?")[0]),
+            ...[...bodyText.matchAll(/[\w.+-]+@[\w-]+(?:\.[\w-]+)+/g)].map((m) => m[0]),
+        ]),
+    ].slice(0, 20);
+    const prices = visiblePrices(mainText);
+    const availability = {
+        inStock: inStockText.test(mainText),
+        outOfStock: outOfStockText.test(mainText),
+        inStockText: mainText.match(inStockText)?.[0] ?? null,
+        outOfStockText: mainText.match(outOfStockText)?.[0] ?? null,
+    };
+    return {
+        http: httpEvidence(response),
+        base,
+        title: clean($("title").first().text()),
+        titles: $("title")
+            .toArray()
+            .map((n) => clean($(n).text())),
+        descriptions: metaValues("description"),
+        language: $("html").attr("lang") ?? null,
+        meta,
+        declarations,
+        links,
+        images,
+        scripts,
+        styles,
+        forms,
+        structuredData,
+        microdata,
+        rdfa,
+        headings,
+        ids,
+        canonicals: declarations.filter((l) => l.rel.split(/\s+/).includes("canonical")),
+        hreflangs: declarations.filter((l) => l.hreflang),
+        social: meta.filter((m) => m.property?.startsWith("og:") || m.name?.startsWith("twitter:")),
+        bodyText,
+        mainText,
+        wordCount: mainText.split(/\s+/).filter(Boolean).length,
+        readability,
+        textHash: hash(mainText),
+        htmlHash: hash(response.body),
+        robots: meta.filter((m) => /robots|bot/i.test(m.name ?? "")),
+        xRobotsTag: response.headers["x-robots-tag"] ?? null,
+        technologies: detectTechnologies(response.body, response.headers),
+        pagination,
+        loadMore,
+        prices,
+        availability,
+        contacts: { phones, emails },
+        truncated: response.truncated ?? false,
+        representation,
+    };
+}
+const indexDirective = (page) => [
+    page.xRobotsTag,
+    ...page.robots
+        .filter((r) => /^(robots|googlebot)$/i.test(r.name ?? ""))
+        .map((r) => r.content),
+]
+    .filter(Boolean)
+    .join(",");
+// What JavaScript changed: the parts crawlers and agents that do not run
+// scripts would miss or see differently.
+export function compareRendered(original, rendered) {
+    const set = (values) => new Set(values.filter((v) => !!v));
+    const originalLinks = set(original.links.map((l) => l.href)), renderedLinks = set(rendered.links.map((l) => l.href));
+    const originalHeadings = set(original.headings.map((h) => `${h.level}:${h.text}`)), renderedHeadings = set(rendered.headings.map((h) => `${h.level}:${h.text}`));
+    const types = (page) => page.structuredData.flatMap((s) => s.parseValid ? structuredTypes(s.data) : []);
+    const originalTypes = types(original), renderedTypes = types(rendered);
+    const diff = (a, b) => [...b].filter((v) => !a.has(v));
+    return {
+        title: {
+            original: original.title,
+            rendered: rendered.title,
+            same: original.title === rendered.title,
+        },
+        canonical: {
+            original: original.canonicals.map((c) => c.href),
+            rendered: rendered.canonicals.map((c) => c.href),
+            same: original.canonicals.map((c) => c.href).join("|") ===
+                rendered.canonicals.map((c) => c.href).join("|"),
+        },
+        robots: {
+            original: indexDirective(original),
+            rendered: indexDirective(rendered),
+            same: indexDirective(original) === indexDirective(rendered),
+        },
+        language: {
+            original: original.language,
+            rendered: rendered.language,
+            same: original.language === rendered.language,
+        },
+        headings: {
+            original: originalHeadings.size,
+            rendered: renderedHeadings.size,
+            added: diff(originalHeadings, renderedHeadings).slice(0, 50),
+            removed: diff(renderedHeadings, originalHeadings).slice(0, 50),
+        },
+        links: {
+            original: originalLinks.size,
+            rendered: renderedLinks.size,
+            added: diff(originalLinks, renderedLinks),
+            removed: diff(renderedLinks, originalLinks).slice(0, 200),
+        },
+        structuredData: {
+            original: originalTypes,
+            rendered: renderedTypes,
+            addedTypes: renderedTypes.filter((t) => !originalTypes.includes(t)),
+            removedTypes: originalTypes.filter((t) => !renderedTypes.includes(t)),
+        },
+        text: {
+            originalWords: original.wordCount,
+            renderedWords: rendered.wordCount,
+            same: original.textHash === rendered.textHash,
+            // Share of the rendered words already present without JavaScript.
+            ratio: rendered.wordCount
+                ? Math.round((original.wordCount / rendered.wordCount) * 100) / 100
+                : null,
+        },
+        scope: "Original response HTML compared with the DOM after load and network idle in headless Chrome; personalization and consent overlays can add differences",
+    };
+}
+// All @type values in a JSON-LD document, including @graph and nested nodes.
+export function structuredTypes(data, depth = 0) {
+    if (depth > 6 || !data)
+        return [];
+    if (Array.isArray(data))
+        return data.flatMap((d) => structuredTypes(d, depth + 1));
+    if (typeof data !== "object")
+        return [];
+    const own = [data["@type"]].flat().filter((t) => typeof t === "string");
+    return [
+        ...own,
+        ...Object.entries(data)
+            .filter(([k]) => k !== "@type")
+            .flatMap(([, v]) => structuredTypes(v, depth + 1)),
+    ];
+}
+export function detectTechnologies(html, headers) {
+    const rules = [
+        ["WordPress", "cms", /wp-content\/|wp-includes\//i, "html"],
+        ["WooCommerce", "ecommerce", /woocommerce/i, "html"],
+        ["Shopify", "ecommerce", /cdn\.shopify\.com|Shopify\.shop/i, "html"],
+        [
+            "Magento",
+            "ecommerce",
+            /Magento_|mage\/requirejs|\/static\/version\d+\//i,
+            "html",
+        ],
+        ["Lightspeed", "ecommerce", /webshopapp\.com|lightspeed/i, "html"],
+        ["Webflow", "cms", /data-wf-(page|site)=/i, "html"],
+        ["Wix", "cms", /wixstatic\.com|wix-thunderbolt/i, "html"],
+        [
+            "Squarespace",
+            "cms",
+            /static\d?\.squarespace\.com|squarespace-cdn\.com/i,
+            "html",
+        ],
+        ["Next.js", "framework", /_next\/static|__NEXT_DATA__/, "html"],
+        ["Nuxt", "framework", /__NUXT__|\/_nuxt\//, "html"],
+        ["Astro", "framework", /astro-island|data-astro-cid-/, "html"],
+        ["Gatsby", "framework", /___gatsby|gatsby-focus-wrapper/, "html"],
+        ["Cloudflare", "cdn", /cloudflare/i, "server"],
+        ["Netlify", "hosting", /Netlify/i, "server"],
+        ["Vercel", "hosting", /Vercel/i, "server"],
+        [
+            "Google Tag Manager",
+            "analytics",
+            /googletagmanager\.com\/gtm\.js/,
+            "html",
+        ],
+        [
+            "Google Analytics",
+            "analytics",
+            /google-analytics\.com|googletagmanager\.com\/gtag/,
+            "html",
+        ],
+        ["Plausible", "analytics", /plausible\.io\/js\//, "html"],
+        ["PostHog", "analytics", /posthog\.com|i\.posthog\.com/, "html"],
+    ];
+    return rules.flatMap(([name, category, pattern, source]) => {
+        const match = pattern.exec(source === "html" ? html : (headers[source] ?? ""));
+        return match
+            ? [
+                {
+                    name,
+                    category,
+                    source,
+                    evidence: match[0],
+                    confidence: "signature-match",
+                    version: null,
+                },
+            ]
+            : [];
+    });
+}
+export function isHtml(response) {
+    return (/text\/html|application\/xhtml\+xml/i.test(response.headers["content-type"] ?? "") || /^\s*(<!doctype html|<html)/i.test(response.body));
+}
+export function markdownEvidence(body, contentType) {
+    const looksHtml = /text\/html|application\/xhtml/i.test(contentType ?? "") ||
+        /^\s*(<!doctype|<html|<head|<body)/i.test(body);
+    const looksMarkdown = /^#{1,6}\s+\S|\[[^\]]+\]\([^\)]+\)/m.test(body);
+    return {
+        usable: !looksHtml &&
+            body.trim().length > 0 &&
+            (/text\/(markdown|x-markdown)/i.test(contentType ?? "") || looksMarkdown),
+        looksHtml,
+        looksMarkdown,
+    };
+}
+export function textTokens(s) {
+    return new Set(s
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter(Boolean));
+}
+export function textSimilarity(a, b) {
+    const left = textTokens(a), right = textTokens(b);
+    if (!left.size || !right.size)
+        return 0;
+    let intersection = 0;
+    for (const word of left)
+        if (right.has(word))
+            intersection++;
+    return intersection / (left.size + right.size - intersection);
+}

package/dist/engine/language.d.ts

@@ -0,0 +1,13 @@
+export declare function detectLanguage(text: string): string | null;
+export declare function languageOfTag(tag: string | null | undefined): string | null;
+export declare function languageLinks(links: {
+    href: string | null;
+    text: string;
+    hreflang?: string;
+    location: string;
+}[]): {
+    url: string;
+    expected: string;
+    text: string;
+    location: string;
+}[];