@rivi-software/agentic-seo-check 0.0.0-stage → 0.1.1

package/dist/core/html/extract.js

@@ -0,0 +1,207 @@
+import { isTag, isText, hasChildren } from "domhandler";
+import { isSameSite, resolveUrl, hostOf } from "../url.js";
+import { attr, parseDocument, queryAll, queryOne, text } from "./dom.js";
+import { extractStructuredData } from "./structured-data.js";
+import { collapseWhitespace, wordCount } from "./text.js";
+const relList = (rel) => (rel ?? "").toLowerCase().split(/\s+/u).filter(Boolean);
+const attrOrNull = (value) => value?.trim() || null;
+const firstNonEmpty = (values) => {
+    for (const v of values) {
+        if (v) {
+            return v;
+        }
+    }
+    return null;
+};
+const detectCharset = (doc) => {
+    const metaCharset = queryOne(doc, "meta[charset]");
+    const direct = metaCharset ? attr(metaCharset, "charset") : undefined;
+    if (direct) {
+        return direct.trim();
+    }
+    const httpEquiv = queryAll(doc, "meta[http-equiv]").find((el) => (attr(el, "http-equiv") ?? "").toLowerCase() === "content-type");
+    const content = httpEquiv ? attr(httpEquiv, "content") : undefined;
+    const match = content?.match(/charset=(?<charset>[\w-]+)/iu);
+    return match?.groups?.charset ?? null;
+};
+/** Elements whose content is never visible page text. */
+const HIDDEN_ELEMENTS = new Set([
+    "script",
+    "style",
+    "noscript",
+    "template",
+    "svg",
+    "iframe",
+    "head",
+]);
+/** Elements that break words apart; padded with spaces so adjacent words don't fuse. */
+const BLOCK_ELEMENTS = new Set([
+    "p",
+    "div",
+    "li",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "br",
+    "tr",
+    "td",
+    "th",
+    "section",
+    "article",
+    "header",
+    "footer",
+    "nav",
+    "blockquote",
+    "pre",
+    "dd",
+    "dt",
+    "figcaption",
+]);
+const visibleText = (doc) => {
+    const body = queryOne(doc, "body");
+    if (!body) {
+        return "";
+    }
+    const parts = [];
+    const walk = (nodes) => {
+        for (const node of nodes) {
+            if (isText(node)) {
+                parts.push(node.data);
+            }
+            else if (isTag(node)) {
+                if (HIDDEN_ELEMENTS.has(node.name)) {
+                    continue;
+                }
+                const block = BLOCK_ELEMENTS.has(node.name);
+                if (block) {
+                    parts.push(" ");
+                }
+                walk(node.children);
+                if (block) {
+                    parts.push(" ");
+                }
+            }
+            else if (hasChildren(node)) {
+                walk(node.children);
+            }
+        }
+    };
+    walk(body.children);
+    return collapseWhitespace(parts.join(""));
+};
+export const extractPage = (html, options) => {
+    const doc = parseDocument(html);
+    const hasDoctype = /^\s*<!doctype\s+html/iu.test(html);
+    const baseEl = queryOne(doc, "base[href]");
+    const baseHref = baseEl ? attr(baseEl, "href") : undefined;
+    const base = (baseHref && resolveUrl(baseHref, options.url)) || options.url;
+    const siteHost = options.siteHost ?? hostOf(options.url) ?? "";
+    const titles = queryAll(doc, "title").map((el) => collapseWhitespace(text(el)));
+    const metaDescriptions = queryAll(doc, "meta[name]")
+        .filter((el) => (attr(el, "name") ?? "").trim().toLowerCase() === "description")
+        .map((el) => collapseWhitespace(attr(el, "content") ?? ""));
+    const metaRobots = queryAll(doc, "meta[name]")
+        .filter((el) => ["robots", "googlebot", "googlebot-news", "bingbot"].includes((attr(el, "name") ?? "").trim().toLowerCase()))
+        .map((el) => collapseWhitespace(attr(el, "content") ?? "").toLowerCase())
+        .filter(Boolean);
+    const canonicals = queryAll(doc, "link[rel]")
+        .filter((el) => relList(attr(el, "rel")).includes("canonical"))
+        .map((el) => (attr(el, "href") ?? "").trim());
+    const headings = queryAll(doc, "h1, h2, h3, h4, h5, h6").map((el) => ({
+        level: Number(el.name.slice(1)),
+        text: collapseWhitespace(text(el)),
+    }));
+    const og = {};
+    const twitter = {};
+    for (const el of queryAll(doc, "meta[property], meta[name]")) {
+        const key = (attr(el, "property") ?? attr(el, "name") ?? "")
+            .trim()
+            .toLowerCase();
+        const content = collapseWhitespace(attr(el, "content") ?? "");
+        if (!key) {
+            continue;
+        }
+        if (key.startsWith("og:") && !(key in og)) {
+            og[key.slice(3)] = content;
+        }
+        else if (key.startsWith("twitter:") && !(key in twitter)) {
+            twitter[key.slice(8)] = content;
+        }
+    }
+    const images = queryAll(doc, "img").map((el) => ({
+        src: attr(el, "src") ?? attr(el, "data-src") ?? null,
+        alt: attr(el, "alt") ?? null,
+        width: attr(el, "width") ?? null,
+        height: attr(el, "height") ?? null,
+        loading: attr(el, "loading") ?? null,
+    }));
+    const links = queryAll(doc, "a[href]").map((el) => {
+        const href = (attr(el, "href") ?? "").trim();
+        // Fragment-only and empty hrefs are not links to other documents.
+        const url = href === "" || href.startsWith("#") ? null : resolveUrl(href, base);
+        const rel = attr(el, "rel") ?? null;
+        const relTokens = relList(rel);
+        const linkText = collapseWhitespace(text(el));
+        const ariaLabel = collapseWhitespace(attr(el, "aria-label") ?? "");
+        const title = collapseWhitespace(attr(el, "title") ?? "");
+        const imgAlt = queryAll(el, "img[alt]")
+            .map((img) => collapseWhitespace(attr(img, "alt") ?? ""))
+            .find(Boolean);
+        return {
+            href,
+            url,
+            text: linkText,
+            rel,
+            internal: url
+                ? isSameSite(url, siteHost, options.includeSubdomains)
+                : false,
+            nofollow: relTokens.includes("nofollow"),
+            hasAccessibleName: Boolean(linkText || ariaLabel || title || imgAlt),
+        };
+    });
+    const hreflang = queryAll(doc, "link[rel]")
+        .filter((el) => relList(attr(el, "rel")).includes("alternate") && attr(el, "hreflang"))
+        .map((el) => {
+        const href = (attr(el, "href") ?? "").trim();
+        return {
+            lang: (attr(el, "hreflang") ?? "").trim(),
+            href,
+            url: resolveUrl(href, base),
+        };
+    });
+    const structured = extractStructuredData(doc);
+    const bodyText = visibleText(doc);
+    const htmlEl = queryOne(doc, "html");
+    const viewportEl = queryOne(doc, 'meta[name="viewport" i]');
+    return {
+        url: options.url,
+        hasDoctype,
+        lang: attrOrNull(htmlEl ? attr(htmlEl, "lang") : undefined),
+        charset: detectCharset(doc),
+        viewport: attrOrNull(viewportEl ? attr(viewportEl, "content") : undefined),
+        titles,
+        title: firstNonEmpty(titles),
+        metaDescriptions,
+        metaDescription: firstNonEmpty(metaDescriptions),
+        metaRobots,
+        canonicals,
+        canonical: canonicals[0] ? resolveUrl(canonicals[0], base) : null,
+        headings,
+        h1s: headings.flatMap((h) => (h.level === 1 ? [h.text] : [])),
+        og,
+        twitter,
+        images,
+        links,
+        hreflang,
+        structuredData: structured.items,
+        structuredDataErrors: structured.errors,
+        text: bodyText,
+        wordCount: wordCount(bodyText),
+        htmlBytes: Buffer.byteLength(html, "utf-8"),
+        hasBaseHref: Boolean(baseHref),
+    };
+};
+//# sourceMappingURL=extract.js.map

package/dist/core/html/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../src/core/html/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AASxD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAEzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAW1D,MAAM,OAAO,GAAG,CAAC,GAA8B,EAAY,EAAE,CAC3D,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE1D,MAAM,UAAU,GAAG,CAAC,KAAyB,EAAiB,EAAE,CAC9D,KAAK,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;AAExB,MAAM,aAAa,GAAG,CAAC,MAAgB,EAAiB,EAAE;IACxD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC;YACN,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,GAAa,EAAiB,EAAE;IACrD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEtE,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,IAAI,CACtD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,CACxE,CAAC;IAEF,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAE7D,OAAO,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC;AACxC,CAAC,CAAC;AAEF,yDAAyD;AACzD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AAEH,wFAAwF;AACxF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,GAAG;IACH,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,SAAS;IACT,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,YAAY;IACZ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,YAAY;CACb,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,GAAa,EAAU,EAAE;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAEnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE5C,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEpB,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEpB,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,IAAY,EACZ,OAAuB,EACR,EAAE;IACjB,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAE/D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAC/C,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC7B,CAAC;IAEF,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;SACjD,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CACxE;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;SAC3C,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACb,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC,QAAQ,CAC3D,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAC9C,CACF;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;SACxE,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;SAC1C,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;SAC9D,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAkB,QAAQ,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC,GAAG,CACzE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACP,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,EAAE,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;KACnC,CAAC,CACH,CAAC;IAEF,MAAM,EAAE,GAA2B,EAAE,CAAC;IACtC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,4BAA4B,CAAC,EAAE,CAAC;QAC7D,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;aACzD,IAAI,EAAE;aACN,WAAW,EAAE,CAAC;QAEjB,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1C,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAgB,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5D,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,IAAI;QACpD,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI;QAC5B,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI;QAChC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,IAAI;QAClC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,MAAM,KAAK,GAAe,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7C,kEAAkE;QAClE,MAAM,GAAG,GACP,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAEtE,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC;aACpC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACxD,IAAI,CAAC,OAAO,CAAC,CAAC;QAEjB,OAAO;YACL,IAAI;YACJ,GAAG;YACH,IAAI,EAAE,QAAQ;YACd,GAAG;YACH,QAAQ,EAAE,GAAG;gBACX,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBACtD,CAAC,CAAC,KAAK;YACT,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS,IAAI,KAAK,IAAI,MAAM,CAAC;SACrE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAmB,QAAQ,CAAC,GAAG,EAAE,WAAW,CAAC;SACxD,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CACzE;SACA,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7C,OAAO;YACL,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YACzC,IAAI;YACJ,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;SAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAE9C,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IAE5D,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,UAAU;QACV,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3D,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC;QAC3B,QAAQ,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM;QACN,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC;QAC5B,gBAAgB;QAChB,eAAe,EAAE,aAAa,CAAC,gBAAgB,CAAC;QAChD,UAAU;QACV,UAAU;QACV,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACjE,QAAQ;QACR,GAAG,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,EAAE;QACF,OAAO;QACP,MAAM;QACN,KAAK;QACL,QAAQ;QACR,cAAc,EAAE,UAAU,CAAC,KAAK;QAChC,oBAAoB,EAAE,UAAU,CAAC,MAAM;QACvC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC9B,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QAC3C,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC;KAC/B,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/html/structured-data.d.ts

@@ -0,0 +1,21 @@
+import type { StructuredDataError, StructuredDataItem } from "../types.js";
+import type { Document } from "./dom.js";
+/** A value JSON.parse returns: all that a JSON-LD block or a microdata item can hold. */
+export type JsonValue = string | number | boolean | null | JsonValue[] | JsonObject;
+/** A JSON-LD node or a microdata item: whatever properties the page gave it. */
+export interface JsonObject {
+    [property: string]: JsonValue;
+}
+/** Whether a structured-data value is a string. */
+export declare const isString: (value: JsonValue | undefined) => value is string;
+/** Whether a structured-data value is an object with properties, rather than a list or a scalar. */
+export declare const isJsonObject: (value: JsonValue | undefined) => value is JsonObject;
+export interface StructuredDataExtraction {
+    items: StructuredDataItem[];
+    errors: StructuredDataError[];
+}
+/** `https://schema.org/Article` -> `Article`, `schema:Article` -> `Article`. */
+export declare const shortType: (type: string) => string;
+export declare const contextString: (value: JsonValue | undefined) => string | null;
+export declare const typesOf: (value: JsonValue | undefined) => string[];
+export declare const extractStructuredData: (doc: Document) => StructuredDataExtraction;

package/dist/core/html/structured-data.js

@@ -0,0 +1,220 @@
+import { isTag } from "domhandler";
+import { attr, childElements, hasAttr, queryAll, text } from "./dom.js";
+import { collapseWhitespace } from "./text.js";
+/** Whether a structured-data value is a string. */
+export const isString = (value) => typeof value === "string";
+/** Whether a structured-data value is an object with properties, rather than a list or a scalar. */
+export const isJsonObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+const stripCdata = (source) => source
+    .replace(/^\s*<!--/u, "")
+    .replace(/-->\s*$/u, "")
+    .replace(/^\s*\/\*\s*<!\[CDATA\[\s*\*\//u, "")
+    .replace(/\/\*\s*\]\]>\s*\*\/\s*$/u, "")
+    .replace(/^\s*<!\[CDATA\[/u, "")
+    .replace(/\]\]>\s*$/u, "");
+/** `https://schema.org/Article` -> `Article`, `schema:Article` -> `Article`. */
+export const shortType = (type) => {
+    const t = type.trim();
+    const slash = t.lastIndexOf("/");
+    const colon = t.lastIndexOf(":");
+    const cut = Math.max(slash, t.startsWith("http") ? -1 : colon);
+    return cut >= 0 ? t.slice(cut + 1) : t;
+};
+export const contextString = (value) => {
+    if (isString(value)) {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        for (const v of value) {
+            const s = contextString(v);
+            if (s) {
+                return s;
+            }
+        }
+        return null;
+    }
+    if (isJsonObject(value)) {
+        const vocab = value["@vocab"];
+        if (isString(vocab)) {
+            return vocab;
+        }
+    }
+    return null;
+};
+export const typesOf = (value) => {
+    if (isString(value)) {
+        return [shortType(value)];
+    }
+    if (Array.isArray(value)) {
+        return value.filter(isString).map(shortType);
+    }
+    return [];
+};
+const flattenJsonLd = (node, inheritedContext) => {
+    if (Array.isArray(node)) {
+        return node.flatMap((n) => flattenJsonLd(n, inheritedContext));
+    }
+    if (!isJsonObject(node)) {
+        return [];
+    }
+    const context = contextString(node["@context"]) ?? inheritedContext;
+    const graph = node["@graph"];
+    if (Array.isArray(graph)) {
+        return graph.flatMap((n) => flattenJsonLd(n, context));
+    }
+    return [
+        { source: "json-ld", types: typesOf(node["@type"]), context, data: node },
+    ];
+};
+/* ---------------------------------------------------------------------------
+ * Microdata
+ * ------------------------------------------------------------------------- */
+/** Elements whose microdata value is always one attribute, empty when it is missing. */
+const URL_VALUE_ATTRIBUTES = new Map([
+    ["meta", "content"],
+    ["img", "src"],
+    ["audio", "src"],
+    ["embed", "src"],
+    ["iframe", "src"],
+    ["source", "src"],
+    ["track", "src"],
+    ["video", "src"],
+    ["a", "href"],
+    ["area", "href"],
+    ["link", "href"],
+    ["object", "data"],
+]);
+/** Elements whose microdata value is an attribute other than `content`, or else their text. */
+const TEXT_VALUE_ATTRIBUTES = new Map([
+    ["data", "value"],
+    ["meter", "value"],
+    ["time", "datetime"],
+]);
+const microdataValue = (el) => {
+    const tag = el.name.toLowerCase();
+    const urlAttribute = URL_VALUE_ATTRIBUTES.get(tag);
+    if (urlAttribute) {
+        return attr(el, urlAttribute) ?? "";
+    }
+    return (attr(el, TEXT_VALUE_ATTRIBUTES.get(tag) ?? "content") ??
+        collapseWhitespace(text(el)));
+};
+/** Walk descendants, stopping at nested itemscopes (they are added as a single property). */
+const collectProps = (root, out) => {
+    for (const child of childElements(root)) {
+        if (hasAttr(child, "itemprop")) {
+            out.push(child);
+        }
+        if (!hasAttr(child, "itemscope")) {
+            collectProps(child, out);
+        }
+    }
+};
+/** Whether an element sits inside another itemscope, which makes it a property, not an item. */
+const hasItemscopeAncestor = (el) => {
+    for (let p = el.parent; p && isTag(p); p = p.parent) {
+        if (hasAttr(p, "itemscope")) {
+            return true;
+        }
+    }
+    return false;
+};
+const microdataObject = (scope) => {
+    const obj = {};
+    const itemtype = attr(scope, "itemtype");
+    if (itemtype) {
+        const types = itemtype.split(/\s+/u).filter(Boolean);
+        const [first] = types;
+        obj["@type"] = types.length === 1 && first ? first : types;
+        if (first) {
+            try {
+                const u = new URL(first);
+                obj["@context"] = `${u.protocol}//${u.host}`;
+            }
+            catch {
+                /* not a URL */
+            }
+        }
+    }
+    const itemid = attr(scope, "itemid");
+    if (itemid) {
+        obj["@id"] = itemid;
+    }
+    const props = [];
+    collectProps(scope, props);
+    for (const el of props) {
+        const names = (attr(el, "itemprop") ?? "").split(/\s+/u).filter(Boolean);
+        const value = hasAttr(el, "itemscope")
+            ? microdataObject(el)
+            : microdataValue(el);
+        for (const name of names) {
+            const existing = obj[name];
+            if (existing === undefined) {
+                obj[name] = value;
+            }
+            else if (Array.isArray(existing)) {
+                existing.push(value);
+            }
+            else {
+                obj[name] = [existing, value];
+            }
+        }
+    }
+    return obj;
+};
+const extractMicrodata = (doc) => {
+    const items = [];
+    for (const el of queryAll(doc, "[itemscope]")) {
+        // Top-level scopes only: nested scopes become property values.
+        if (hasItemscopeAncestor(el)) {
+            continue;
+        }
+        const data = microdataObject(el);
+        items.push({
+            source: "microdata",
+            types: typesOf(data["@type"]),
+            context: contextString(data["@context"]),
+            data,
+        });
+    }
+    return items;
+};
+export const extractStructuredData = (doc) => {
+    const items = [];
+    const errors = [];
+    for (const el of queryAll(doc, "script[type]")) {
+        const type = (attr(el, "type") ?? "").trim().toLowerCase();
+        if (type !== "application/ld+json") {
+            continue;
+        }
+        const raw = text(el).trim();
+        if (!raw) {
+            errors.push({
+                source: "json-ld",
+                message: "Empty JSON-LD script block",
+                snippet: "",
+            });
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(stripCdata(raw));
+        }
+        catch (error) {
+            errors.push({
+                source: "json-ld",
+                message: `Invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
+                snippet: raw.slice(0, 200),
+            });
+            continue;
+        }
+        for (const node of flattenJsonLd(parsed, null)) {
+            items.push(node);
+        }
+    }
+    for (const item of extractMicrodata(doc)) {
+        items.push(item);
+    }
+    return { items, errors };
+};
+//# sourceMappingURL=structured-data.js.map

package/dist/core/html/structured-data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-data.js","sourceRoot":"","sources":["../../../src/core/html/structured-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAGnC,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAExE,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAgB/C,mDAAmD;AACnD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,KAA4B,EAAmB,EAAE,CACxE,OAAO,KAAK,KAAK,QAAQ,CAAC;AAE5B,oGAAoG;AACpG,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,KAA4B,EACP,EAAE,CACvB,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAOvE,MAAM,UAAU,GAAG,CAAC,MAAc,EAAU,EAAE,CAC5C,MAAM;KACH,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;KACxB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;KACvB,OAAO,CAAC,gCAAgC,EAAE,EAAE,CAAC;KAC7C,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC;KACvC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;KAC/B,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAE/B,gFAAgF;AAChF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE;IAChD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAE/D,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAA4B,EAAiB,EAAE;IAC3E,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAE3B,IAAI,CAAC,EAAE,CAAC;gBACN,OAAO,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,KAA4B,EAAY,EAAE;IAChE,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CACpB,IAAe,EACf,gBAA+B,EACT,EAAE;IACxB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,gBAAgB,CAAC;IACpE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE7B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO;QACL,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;KAC1E,CAAC;AACJ,CAAC,CAAC;AAEF;;+EAE+E;AAE/E,wFAAwF;AACxF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,CAAC,MAAM,EAAE,SAAS,CAAC;IACnB,CAAC,KAAK,EAAE,KAAK,CAAC;IACd,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,QAAQ,EAAE,KAAK,CAAC;IACjB,CAAC,QAAQ,EAAE,KAAK,CAAC;IACjB,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,GAAG,EAAE,MAAM,CAAC;IACb,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,QAAQ,EAAE,MAAM,CAAC;CACnB,CAAC,CAAC;AAEH,+FAA+F;AAC/F,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC;IACpC,CAAC,MAAM,EAAE,OAAO,CAAC;IACjB,CAAC,OAAO,EAAE,OAAO,CAAC;IAClB,CAAC,MAAM,EAAE,UAAU,CAAC;CACrB,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,EAAW,EAAU,EAAE;IAC7C,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,YAAY,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEnD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,CACL,IAAI,CAAC,EAAE,EAAE,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC;QACrD,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC7B,CAAC;AACJ,CAAC,CAAC;AAEF,6FAA6F;AAC7F,MAAM,YAAY,GAAG,CAAC,IAAa,EAAE,GAAc,EAAQ,EAAE;IAC3D,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,CAAC;YACjC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,oBAAoB,GAAG,CAAC,EAAW,EAAW,EAAE;IACpD,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,IAAI,OAAO,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,KAAc,EAAc,EAAE;IACrD,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAEzC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACtB,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QAE3D,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAErC,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzE,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC;YACpC,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC;YACrB,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;YAE3B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YACpB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,GAAa,EAAwB,EAAE;IAC/D,MAAM,KAAK,GAAyB,EAAE,CAAC;IAEvC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,CAAC;QAC9C,+DAA+D;QAC/D,IAAI,oBAAoB,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7B,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI;SACL,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,GAAa,EACa,EAAE;IAC5B,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3D,IAAI,IAAI,KAAK,qBAAqB,EAAE,CAAC;YACnC,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE5B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,4BAA4B;gBACrC,OAAO,EAAE,EAAE;aACZ,CAAC,CAAC;YAEH,SAAS;QACX,CAAC;QAED,IAAI,MAAiB,CAAC;QAEtB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,iBAAiB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBAClF,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aAC3B,CAAC,CAAC;YAEH,SAAS;QACX,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC,CAAC"}

package/dist/core/html/text.d.ts

@@ -0,0 +1,23 @@
+/** Text analysis helpers: word counts, readability, placeholder detection. */
+export declare const words: (text: string) => string[];
+export declare const wordCount: (text: string) => number;
+export declare const collapseWhitespace: (text: string) => string;
+export declare const sentenceCount: (text: string) => number;
+/** English syllable heuristic: vowel groups minus silent trailing e, min 1. */
+export declare const syllables: (word: string) => number;
+/**
+ * Flesch Reading Ease (English formula). Returns null when there is not enough text.
+ * 90-100 very easy, 60-70 plain English, 30-50 difficult, <30 very difficult.
+ */
+export declare const fleschReadingEase: (text: string) => number | null;
+export declare const hasPlaceholderText: (text: string) => boolean;
+export declare const GENERIC_TITLES: Set<string>;
+export declare const isGenericTitle: (title: string) => boolean;
+export declare const GENERIC_ANCHORS: Set<string>;
+export declare const isGenericAnchor: (text: string) => boolean;
+export declare const normalizeForComparison: (text: string) => string;
+export declare const includesKeyword: (haystack: string | null | undefined, keyword: string) => boolean;
+export declare const keywordOccurrences: (text: string, keyword: string) => number;
+/** 5-word shingles for near-duplicate detection. */
+export declare const shingles: (text: string, size?: number) => Set<string>;
+export declare const jaccard: (a: Set<string>, b: Set<string>) => number;

package/dist/core/html/text.js

@@ -0,0 +1,152 @@
+/** Text analysis helpers: word counts, readability, placeholder detection. */
+const WORD_RE = /[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu;
+export const words = (text) => text.match(WORD_RE) ?? [];
+export const wordCount = (text) => words(text).length;
+export const collapseWhitespace = (text) => text.replaceAll(/\s+/gu, " ").trim();
+export const sentenceCount = (text) => {
+    const parts = text
+        .split(/[.!?]+(?:\s+|$)/u)
+        .filter((s) => s.trim().length > 0);
+    return Math.max(1, parts.length);
+};
+/** English syllable heuristic: vowel groups minus silent trailing e, min 1. */
+export const syllables = (word) => {
+    const w = word.toLowerCase().replaceAll(/[^a-z]/gu, "");
+    if (!w) {
+        return 0;
+    }
+    if (w.length <= 3) {
+        return 1;
+    }
+    let count = (w.replace(/e$/u, "").match(/[aeiouy]+/gu) ?? []).length;
+    if (w.endsWith("le") && !/[aeiouy]le$/u.test(w)) {
+        count += 1;
+    }
+    return Math.max(1, count);
+};
+/**
+ * Flesch Reading Ease (English formula). Returns null when there is not enough text.
+ * 90-100 very easy, 60-70 plain English, 30-50 difficult, <30 very difficult.
+ */
+export const fleschReadingEase = (text) => {
+    const ws = words(text);
+    if (ws.length < 100) {
+        return null;
+    }
+    const sentences = sentenceCount(text);
+    const syl = ws.reduce((sum, w) => sum + syllables(w), 0);
+    const score = 206.835 - 1.015 * (ws.length / sentences) - 84.6 * (syl / ws.length);
+    return Math.round(score * 10) / 10;
+};
+const PLACEHOLDER_RE = /lorem ipsum|dolor sit amet|consectetur adipiscing/iu;
+export const hasPlaceholderText = (text) => PLACEHOLDER_RE.test(text);
+export const GENERIC_TITLES = new Set([
+    "home",
+    "homepage",
+    "index",
+    "untitled",
+    "untitled page",
+    "untitled document",
+    "document",
+    "new page",
+    "page",
+    "title",
+    "welcome",
+    "react app",
+    "vite app",
+    "vite + react",
+    "vite + react + ts",
+    "vite + vue",
+    "vite + svelte",
+    "create next app",
+    "next app",
+    "nuxt app",
+    "nuxt",
+    "astro",
+    "my app",
+    "my site",
+    "my website",
+    "website",
+    "default",
+    "test",
+    "hello world",
+    "svelte app",
+    "angular app",
+    "gatsby default starter",
+    "remix app",
+    "tanstack start starter",
+    "example",
+]);
+export const isGenericTitle = (title) => GENERIC_TITLES.has(collapseWhitespace(title).toLowerCase());
+export const GENERIC_ANCHORS = new Set([
+    "click here",
+    "here",
+    "read more",
+    "learn more",
+    "more",
+    "link",
+    "this",
+    "this page",
+    "continue",
+    "continue reading",
+    "details",
+    "more info",
+    "info",
+    "go",
+    "page",
+    "article",
+    "website",
+    "url",
+    "www",
+]);
+export const isGenericAnchor = (text) => {
+    const t = collapseWhitespace(text)
+        .toLowerCase()
+        .replace(/[.!…»›>]+$/u, "")
+        .trim();
+    return GENERIC_ANCHORS.has(t);
+};
+export const normalizeForComparison = (text) => collapseWhitespace(text).toLowerCase();
+export const includesKeyword = (haystack, keyword) => {
+    if (!haystack) {
+        return false;
+    }
+    return normalizeForComparison(haystack).includes(normalizeForComparison(keyword));
+};
+export const keywordOccurrences = (text, keyword) => {
+    const hay = normalizeForComparison(text);
+    const needle = normalizeForComparison(keyword);
+    if (!needle) {
+        return 0;
+    }
+    let count = 0;
+    let idx = hay.indexOf(needle);
+    while (idx !== -1) {
+        count += 1;
+        idx = hay.indexOf(needle, idx + needle.length);
+    }
+    return count;
+};
+/** 5-word shingles for near-duplicate detection. */
+export const shingles = (text, size = 5) => {
+    const ws = words(normalizeForComparison(text));
+    const out = new Set();
+    for (let i = 0; i + size <= ws.length; i += 1) {
+        out.add(ws.slice(i, i + size).join(" "));
+    }
+    return out;
+};
+export const jaccard = (a, b) => {
+    if (a.size === 0 && b.size === 0) {
+        return 1;
+    }
+    let inter = 0;
+    const [small, large] = a.size < b.size ? [a, b] : [b, a];
+    for (const s of small) {
+        if (large.has(s)) {
+            inter += 1;
+        }
+    }
+    return inter / (a.size + b.size - inter);
+};
+//# sourceMappingURL=text.js.map

package/dist/core/html/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../src/core/html/text.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAE9E,MAAM,OAAO,GAAG,gCAAgC,CAAC;AAEjD,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAY,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;AAE3E,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAEtE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAU,EAAE,CACzD,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAEvC,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAU,EAAE;IACpD,MAAM,KAAK,GAAG,IAAI;SACf,KAAK,CAAC,kBAAkB,CAAC;SACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE;IAChD,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAExD,IAAI,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAErE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAChD,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAiB,EAAE;IAC/D,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAEvB,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEzD,MAAM,KAAK,GACT,OAAO,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;IAEvE,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;AACrC,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,qDAAqD,CAAC;AAE7E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAW,EAAE,CAC1D,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IACpC,MAAM;IACN,UAAU;IACV,OAAO;IACP,UAAU;IACV,eAAe;IACf,mBAAmB;IACnB,UAAU;IACV,UAAU;IACV,MAAM;IACN,OAAO;IACP,SAAS;IACT,WAAW;IACX,UAAU;IACV,cAAc;IACd,mBAAmB;IACnB,YAAY;IACZ,eAAe;IACf,iBAAiB;IACjB,UAAU;IACV,UAAU;IACV,MAAM;IACN,OAAO;IACP,QAAQ;IACR,SAAS;IACT,YAAY;IACZ,SAAS;IACT,SAAS;IACT,MAAM;IACN,aAAa;IACb,YAAY;IACZ,aAAa;IACb,wBAAwB;IACxB,WAAW;IACX,wBAAwB;IACxB,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,KAAa,EAAW,EAAE,CACvD,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAE9D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IACrC,YAAY;IACZ,MAAM;IACN,WAAW;IACX,YAAY;IACZ,MAAM;IACN,MAAM;IACN,MAAM;IACN,WAAW;IACX,UAAU;IACV,kBAAkB;IAClB,SAAS;IACT,WAAW;IACX,MAAM;IACN,IAAI;IACJ,MAAM;IACN,SAAS;IACT,SAAS;IACT,KAAK;IACL,KAAK;CACN,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAW,EAAE;IACvD,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC;SAC/B,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAC;IAEV,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC7D,kBAAkB,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;AAEzC,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,QAAmC,EACnC,OAAe,EACN,EAAE;IACX,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAC9C,sBAAsB,CAAC,OAAO,CAAC,CAChC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAE,OAAe,EAAU,EAAE;IAC1E,MAAM,GAAG,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9B,OAAO,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,KAAK,IAAI,CAAC,CAAC;QACX,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,oDAAoD;AACpD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,IAAI,GAAG,CAAC,EAAe,EAAE;IAC9D,MAAM,EAAE,GAAG,KAAK,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAc,EAAE,CAAc,EAAU,EAAE;IAChE,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEzD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACjB,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAED,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;AAC3C,CAAC,CAAC"}