@rivi-software/agentic-seo-check 0.1.0 → 0.2.0

package/dist/core/fetch.js

@@ -1,119 +1,165 @@
-import { VERSION } from '../version.js';
+import { VERSION } from "../version.js";
 export const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; agentic-seo-check/${VERSION}; +https://github.com/rivi-software/agentic-seo-check)`;
-const KEPT_HEADERS = ['content-type', 'x-robots-tag', 'link', 'content-language', 'cache-control', 'last-modified', 'location', 'content-length', 'vary'];
+const KEPT_HEADERS = [
+    "content-type",
+    "x-robots-tag",
+    "link",
+    "content-language",
+    "cache-control",
+    "last-modified",
+    "location",
+    "content-length",
+    "vary",
+];
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
-/** GET a URL, following redirects manually so the chain is recorded. */
-export async function fetchPage(url, options = {}, readBody = true) {
-    const timeoutMs = options.timeoutMs ?? 15_000;
-    const maxRedirects = options.maxRedirects ?? 10;
-    const started = performance.now();
-    const chain = [];
-    let current = url;
-    for (let hop = 0; hop <= maxRedirects; hop++) {
-        const controller = new AbortController();
-        const timer = setTimeout(() => controller.abort(), timeoutMs);
-        try {
-            const res = await fetch(current, {
-                method: 'GET',
-                redirect: 'manual',
-                signal: controller.signal,
-                headers: {
-                    'user-agent': options.userAgent ?? DEFAULT_USER_AGENT,
-                    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                    'accept-language': 'en,*;q=0.5',
-                    ...(options.headers ?? {}),
-                },
-            });
-            const headers = pickHeaders(res.headers);
-            if (REDIRECT_STATUSES.has(res.status) && headers['location']) {
-                chain.push({ url: current, status: res.status });
-                let nextUrl;
-                try {
-                    nextUrl = new URL(headers['location'], current).href;
-                }
-                catch {
-                    return result(url, current, res.status, headers, null, chain, started, `Invalid Location header: ${headers['location']}`);
-                }
-                await res.body?.cancel().catch(() => undefined);
-                if (chain.some((h) => h.url === nextUrl) || nextUrl === current) {
-                    return result(url, nextUrl, res.status, headers, null, chain, started, 'Redirect loop');
-                }
-                current = nextUrl;
-                continue;
-            }
-            let body = null;
-            if (readBody)
-                body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
-            else
-                await res.body?.cancel().catch(() => undefined);
-            return result(url, current, res.status, headers, body, chain, started);
-        }
-        catch (err) {
-            const message = err instanceof Error ? (err.name === 'AbortError' ? `Timed out after ${timeoutMs} ms` : err.message) : String(err);
-            return result(url, current, null, {}, null, chain, started, message);
-        }
-        finally {
-            clearTimeout(timer);
+const pickHeaders = (headers) => {
+    const out = {};
+    for (const name of KEPT_HEADERS) {
+        const value = headers.get(name);
+        if (value !== null) {
+            out[name] = value;
         }
     }
-    return result(url, current, null, {}, null, chain, started, `More than ${maxRedirects} redirects`);
-}
-/** Status/redirect check without downloading the body. */
-export function probeUrl(url, options = {}) {
-    return fetchPage(url, options, false);
-}
-async function readLimited(res, maxBytes) {
-    if (!res.body)
-        return '';
+    return out;
+};
+const result = (requestedUrl, finalUrl, status, headers, body, chain, started, error) => {
+    const r = {
+        requestedUrl,
+        finalUrl,
+        status,
+        headers,
+        body,
+        redirectChain: chain,
+        timeMs: Math.round(performance.now() - started),
+        contentType: headers["content-type"] ?? null,
+    };
+    if (error) {
+        r.error = error;
+    }
+    return r;
+};
+const readLimited = async (res, maxBytes) => {
+    if (!res.body) {
+        return "";
+    }
     const reader = res.body.getReader();
     const chunks = [];
     let total = 0;
+    let truncated = false;
     for (;;) {
+        // oxlint-disable-next-line eslint/no-await-in-loop -- a stream hands over its chunks one after another
         const { done, value } = await reader.read();
-        if (done)
+        if (done) {
             break;
+        }
         if (value) {
             chunks.push(value);
             total += value.byteLength;
             if (total >= maxBytes) {
-                await reader.cancel().catch(() => undefined);
+                truncated = true;
                 break;
             }
         }
     }
-    const charset = /charset=([\w-]+)/i.exec(res.headers.get('content-type') ?? '')?.[1];
+    if (truncated) {
+        await reader.cancel().catch(() => null);
+    }
+    const charset = /charset=(?<charset>[\w-]+)/iu.exec(res.headers.get("content-type") ?? "")?.groups?.charset;
     try {
-        return new TextDecoder(charset ?? 'utf-8').decode(Buffer.concat(chunks));
+        return new TextDecoder(charset ?? "utf-8").decode(Buffer.concat(chunks));
     }
     catch {
-        return new TextDecoder('utf-8').decode(Buffer.concat(chunks));
+        return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
     }
-}
-function pickHeaders(headers) {
-    const out = {};
-    for (const name of KEPT_HEADERS) {
-        const value = headers.get(name);
-        if (value !== null)
-            out[name] = value;
+};
+const fetchHop = async (walk, current) => {
+    const { url, options, readBody, timeoutMs, started, chain } = walk;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(current, {
+            method: "GET",
+            redirect: "manual",
+            signal: controller.signal,
+            headers: {
+                "user-agent": options.userAgent ?? DEFAULT_USER_AGENT,
+                accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "accept-language": "en,*;q=0.5",
+                ...options.headers,
+            },
+        });
+        const headers = pickHeaders(res.headers);
+        if (REDIRECT_STATUSES.has(res.status) && headers["location"]) {
+            chain.push({ url: current, status: res.status });
+            let nextUrl;
+            try {
+                nextUrl = new URL(headers["location"], current).href;
+            }
+            catch {
+                return {
+                    done: result(url, current, res.status, headers, null, chain, started, `Invalid Location header: ${headers["location"]}`),
+                };
+            }
+            await res.body?.cancel().catch(() => null);
+            if (chain.some((h) => h.url === nextUrl) || nextUrl === current) {
+                return {
+                    done: result(url, nextUrl, res.status, headers, null, chain, started, "Redirect loop"),
+                };
+            }
+            return { next: nextUrl };
+        }
+        let body = null;
+        if (readBody) {
+            body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
+        }
+        else {
+            await res.body?.cancel().catch(() => null);
+        }
+        return {
+            done: result(url, current, res.status, headers, body, chain, started),
+        };
     }
-    return out;
-}
-function result(requestedUrl, finalUrl, status, headers, body, chain, started, error) {
-    const r = {
-        requestedUrl,
-        finalUrl,
-        status,
-        headers,
-        body,
-        redirectChain: chain,
-        timeMs: Math.round(performance.now() - started),
-        contentType: headers['content-type'] ?? null,
+    catch (error) {
+        let message = String(error);
+        if (error instanceof Error) {
+            message =
+                error.name === "AbortError"
+                    ? `Timed out after ${timeoutMs} ms`
+                    : error.message;
+        }
+        return {
+            done: result(url, current, null, {}, null, chain, started, message),
+        };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+};
+/** GET a URL, following redirects manually so the chain is recorded. */
+export const fetchPage = async (url, options = {}, readBody = true) => {
+    const maxRedirects = options.maxRedirects ?? 10;
+    const walk = {
+        url,
+        options,
+        readBody,
+        timeoutMs: options.timeoutMs ?? 15_000,
+        started: performance.now(),
+        chain: [],
     };
-    if (error)
-        r.error = error;
-    return r;
-}
-export function toResponseInfo(r) {
+    let current = url;
+    for (let hop = 0; hop <= maxRedirects; hop += 1) {
+        // oxlint-disable-next-line eslint/no-await-in-loop -- each request goes where the one before it redirected
+        const outcome = await fetchHop(walk, current);
+        if ("done" in outcome) {
+            return outcome.done;
+        }
+        current = outcome.next;
+    }
+    return result(url, current, null, {}, null, walk.chain, walk.started, `More than ${maxRedirects} redirects`);
+};
+/** Status/redirect check without downloading the body. */
+export const probeUrl = (url, options = {}) => fetchPage(url, options, false);
+export const toResponseInfo = (r) => {
     const info = {
         status: r.status,
         finalUrl: r.finalUrl,
@@ -122,14 +168,19 @@ export function toResponseInfo(r) {
         contentType: r.contentType,
         headers: r.headers,
     };
-    if (r.error)
+    if (r.error) {
         info.error = r.error;
+    }
     return info;
-}
-export function isHtmlResponse(r) {
-    const ct = r.contentType?.toLowerCase() ?? '';
-    if (ct)
-        return ct.includes('text/html') || ct.includes('application/xhtml');
-    return typeof r.body === 'string' && /<html|<!doctype/i.test(r.body.slice(0, 2000));
-}
+};
+export const isHtmlResponse = (r) => {
+    const ct = r.contentType?.toLowerCase() ?? "";
+    if (ct) {
+        return ct.includes("text/html") || ct.includes("application/xhtml");
+    }
+    if (!r.body) {
+        return false;
+    }
+    return /<html|<!doctype/iu.test(r.body.slice(0, 2000));
+};
 //# sourceMappingURL=fetch.js.map

package/dist/core/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG,CAAC,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;AAuB1J,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE,EAAE,QAAQ,GAAG,IAAI;IACtF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAChD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;gBAC/B,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;oBACrD,MAAM,EAAE,iEAAiE;oBACzE,iBAAiB,EAAE,YAAY;oBAC/B,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;iBAC3B;aACF,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;gBACvD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBAC5H,CAAC;gBACD,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAChD,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;oBAChE,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;gBAC1F,CAAC;gBACD,OAAO,GAAG,OAAO,CAAC;gBAClB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,GAAkB,IAAI,CAAC;YAC/B,IAAI,QAAQ;gBAAE,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;;gBAC1E,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACzE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,mBAAmB,SAAS,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnI,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,YAAY,YAAY,CAAC,CAAC;AACrG,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,UAAwB,EAAE;IAC9D,OAAO,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAa,EAAE,QAAgB;IACxD,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,SAAS,CAAC;QACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI;YAAE,MAAM;QAChB,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAC1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC7C,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,OAAgB;IACnC,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,MAAM,CAAC,YAAoB,EAAE,QAAgB,EAAE,MAAqB,EAAE,OAA+B,EAAE,IAAmB,EAAE,KAAoB,EAAE,OAAe,EAAE,KAAc;IACxL,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IACF,IAAI,KAAK;QAAE,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAc;IAC3C,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IACF,IAAI,CAAC,CAAC,KAAK;QAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAClC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAc;IAC3C,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC9C,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC5E,OAAO,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACtF,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;CACP,CAAC;AAuBF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACC,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAE1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CACjD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CACtC,EAAE,MAAM,EAAE,OAAO,CAAC;IAEnB,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC,CAAC;AAeF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,GAAkB,IAAI,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;SACtE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}

package/dist/core/html/answer.d.ts

@@ -0,0 +1,4 @@
+import type { AnswerSignals } from "../types.js";
+import type { Document } from "./dom.js";
+/** Reads what the AEO rules need from a parsed document; `bodyText` is its visible body text. */
+export declare const extractAnswerSignals: (doc: Document, bodyText: string) => AnswerSignals;

package/dist/core/html/answer.js

@@ -0,0 +1,198 @@
+import { hasChildren, isTag, isText } from "domhandler";
+import { attr, childElements, HIDDEN_ELEMENTS, queryAll, queryOne, text, visibleText, } from "./dom.js";
+import { collapseWhitespace, wordCount } from "./text.js";
+/** Meta names whose content holds robots directives for one crawler or for all of them. */
+const ROBOTS_META_NAMES = new Set([
+    "robots",
+    "googlebot",
+    "googlebot-news",
+    "bingbot",
+]);
+/** Containers FAQ widgets put a question in when it is not a heading. */
+const QUESTION_CONTAINERS = new Set(["summary", "dt"]);
+/** Elements an answer usually starts with. */
+const ANSWER_BLOCKS = new Set([
+    "p",
+    "li",
+    "dd",
+    "blockquote",
+    "td",
+    "figcaption",
+    "pre",
+]);
+/** Page chrome: lists and tables here are navigation, not content. */
+const CHROME = new Set(["nav", "header", "footer", "aside"]);
+/** Roots JavaScript frameworks render into; empty in the HTML when the page renders client-side. */
+const APP_ROOTS = [
+    "#root",
+    "#app",
+    "#__next",
+    "#__nuxt",
+    "#___gatsby",
+    "#svelte",
+    "app-root",
+    "[data-reactroot]",
+    "[ng-app]",
+];
+/*
+ * Written dates in the visible text. Every quantifier is bounded and every alternative is a fixed
+ * word, so each regular expression stays linear on long page text.
+ */
+const DATE_PATTERNS = [
+    /\b\d{4}-\d{2}-\d{2}\b/u,
+    /\b\d{1,2}[./-]\d{1,2}[./-]\d{4}\b/u,
+    /\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.? \d{1,2}(?:st|nd|rd|th)?,? \d{4}\b/iu,
+    /\b\d{1,2}(?:st|nd|rd|th)?\.? (?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.?,? \d{4}\b/iu,
+];
+const MAX_QUESTIONS = 100;
+/** A question ends with a question mark; a long "heading" is a paragraph styled as one. */
+const isQuestion = (value) => value.length > 0 && value.length <= 300 && /[?？]$/u.test(value);
+const isHeading = (el) => /^h[1-6]$/u.test(el.name);
+const inChrome = (el) => {
+    for (let p = el.parent; p && isTag(p); p = p.parent) {
+        if (CHROME.has(p.name)) {
+            return true;
+        }
+    }
+    return false;
+};
+/**
+ * What a heading, `<summary>` or `<dt>` does to the question being read: "subsection" goes on with
+ * its section, "content" is ordinary text, a question starts a new section and null ends it.
+ */
+const headingEffect = (el, current) => {
+    const heading = isHeading(el);
+    const label = collapseWhitespace(visibleText(el));
+    const question = isQuestion(label);
+    const level = heading ? Number(el.name.slice(1)) : 0;
+    if (!question) {
+        // "Why choose us?" answered by one card per reason, each under its own subheading, is answered.
+        if (heading && current && current.level > 0 && level > current.level) {
+            return "subsection";
+        }
+        // A <summary> or <dt> ends a section only when it asks a question itself.
+        return heading ? null : "content";
+    }
+    return {
+        level,
+        text: label.slice(0, 200),
+        sectionWords: 0,
+        firstBlockWords: null,
+    };
+};
+/** Question headings in document order, with how much text follows each before the next one. */
+const findQuestions = (body) => {
+    const questions = [];
+    let current = null;
+    const walk = (nodes) => {
+        for (const node of nodes) {
+            if (isText(node)) {
+                if (current) {
+                    current.sectionWords += wordCount(node.data);
+                }
+                continue;
+            }
+            if (!isTag(node)) {
+                if (hasChildren(node)) {
+                    walk(node.children);
+                }
+                continue;
+            }
+            if (HIDDEN_ELEMENTS.has(node.name)) {
+                continue;
+            }
+            if (isHeading(node) || QUESTION_CONTAINERS.has(node.name)) {
+                const effect = headingEffect(node, current);
+                if (effect === "subsection") {
+                    continue;
+                }
+                if (effect !== "content") {
+                    current = effect;
+                    if (current && questions.length < MAX_QUESTIONS) {
+                        questions.push(current);
+                    }
+                    continue;
+                }
+            }
+            if (current &&
+                current.firstBlockWords === null &&
+                ANSWER_BLOCKS.has(node.name)) {
+                current.firstBlockWords = wordCount(visibleText(node));
+            }
+            walk(node.children);
+        }
+    };
+    walk(body.children);
+    return questions;
+};
+/** A short selector for a `data-nosnippet` element: its tag plus its id or first class. */
+const nosnippetSelector = (el) => {
+    const id = attr(el, "id")?.trim();
+    const firstClass = attr(el, "class")?.trim().split(/\s+/u)[0];
+    let { name } = el;
+    if (id) {
+        name += `#${id}`;
+    }
+    else if (firstClass) {
+        name += `.${firstClass}`;
+    }
+    return `${name.slice(0, 80)}[data-nosnippet]`;
+};
+const hasNosnippetAncestor = (el) => {
+    for (let p = el.parent; p && isTag(p); p = p.parent) {
+        if (Object.hasOwn(p.attribs, "data-nosnippet")) {
+            return true;
+        }
+    }
+    return false;
+};
+const emptyAppRoot = (body) => {
+    for (const selector of APP_ROOTS) {
+        const el = queryOne(body, selector);
+        if (el && wordCount(visibleText(el)) === 0) {
+            return selector;
+        }
+    }
+    return null;
+};
+const emptySignals = (robotsMeta) => ({
+    robotsMeta,
+    questions: [],
+    lists: 0,
+    tables: 0,
+    nosnippetElements: 0,
+    nosnippetWords: 0,
+    nosnippetSelectors: [],
+    emptyAppRoot: null,
+    noscriptAsksForJavaScript: false,
+    timeElements: 0,
+    mentionsDate: false,
+});
+/** Reads what the AEO rules need from a parsed document; `bodyText` is its visible body text. */
+export const extractAnswerSignals = (doc, bodyText) => {
+    const robotsMeta = queryAll(doc, "meta[name]").flatMap((el) => {
+        const name = (attr(el, "name") ?? "").trim().toLowerCase();
+        const content = collapseWhitespace(attr(el, "content") ?? "").toLowerCase();
+        return ROBOTS_META_NAMES.has(name) && content ? [{ name, content }] : [];
+    });
+    const body = queryOne(doc, "body");
+    if (!body) {
+        return emptySignals(robotsMeta);
+    }
+    const nosnippet = queryAll(body, "[data-nosnippet]").filter((el) => !hasNosnippetAncestor(el));
+    return {
+        robotsMeta,
+        questions: findQuestions(body),
+        lists: queryAll(body, "ul, ol").filter((el) => childElements(el).filter((c) => c.name === "li").length >= 2 &&
+            !inChrome(el)).length,
+        tables: queryAll(body, "table").filter((el) => queryAll(el, "tr").length >= 2 && !inChrome(el)).length,
+        nosnippetElements: nosnippet.length,
+        nosnippetWords: nosnippet.reduce((sum, el) => sum + wordCount(visibleText(el)), 0),
+        nosnippetSelectors: nosnippet.slice(0, 10).map(nosnippetSelector),
+        emptyAppRoot: emptyAppRoot(body),
+        noscriptAsksForJavaScript: queryAll(body, "noscript").some((el) => /javascript/iu.test(text(el))),
+        timeElements: queryAll(body, "time").length,
+        mentionsDate: DATE_PATTERNS.some((re) => re.test(bodyText)),
+    };
+};
+//# sourceMappingURL=answer.js.map

package/dist/core/html/answer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer.js","sourceRoot":"","sources":["../../../src/core/html/answer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGxD,OAAO,EACL,IAAI,EACJ,aAAa,EACb,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,WAAW,GACZ,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE1D,2FAA2F;AAC3F,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;CACV,CAAC,CAAC;AAEH,yEAAyE;AACzE,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;AAEvD,8CAA8C;AAC9C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,YAAY;IACZ,IAAI;IACJ,YAAY;IACZ,KAAK;CACN,CAAC,CAAC;AAEH,sEAAsE;AACtE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;AAE7D,oGAAoG;AACpG,MAAM,SAAS,GAAG;IAChB,OAAO;IACP,MAAM;IACN,SAAS;IACT,SAAS;IACT,YAAY;IACZ,SAAS;IACT,UAAU;IACV,kBAAkB;IAClB,UAAU;CACX,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG;IACpB,wBAAwB;IACxB,oCAAoC;IACpC,wLAAwL;IACxL,2LAA2L;CAC5L,CAAC;AAEF,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,2FAA2F;AAC3F,MAAM,UAAU,GAAG,CAAC,KAAa,EAAW,EAAE,CAC5C,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAElE,MAAM,SAAS,GAAG,CAAC,EAAW,EAAW,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;AAEtE,MAAM,QAAQ,GAAG,CAAC,EAAW,EAAW,EAAE;IACxC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG,CACpB,EAAW,EACX,OAA+B,EACoB,EAAE;IACrD,MAAM,OAAO,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,kBAAkB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,gGAAgG;QAChG,IAAI,OAAO,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YACrE,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,0EAA0E;QAC1E,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,OAAO;QACL,KAAK;QACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACzB,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,IAAI;KACtB,CAAC;AACJ,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,aAAa,GAAG,CAAC,IAAa,EAAqB,EAAE;IACzD,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,IAAI,OAAO,GAA2B,IAAI,CAAC;IAE3C,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/C,CAAC;gBAED,SAAS;YACX,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtB,CAAC;gBAED,SAAS;YACX,CAAC;YAED,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1D,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAE5C,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;oBAC5B,SAAS;gBACX,CAAC;gBAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,OAAO,GAAG,MAAM,CAAC;oBAEjB,IAAI,OAAO,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;wBAChD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC1B,CAAC;oBAED,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IACE,OAAO;gBACP,OAAO,CAAC,eAAe,KAAK,IAAI;gBAChC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAC5B,CAAC;gBACD,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEpB,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF,2FAA2F;AAC3F,MAAM,iBAAiB,GAAG,CAAC,EAAW,EAAU,EAAE;IAChD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9D,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IAElB,IAAI,EAAE,EAAE,CAAC;QACP,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;IACnB,CAAC;SAAM,IAAI,UAAU,EAAE,CAAC;QACtB,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC;AAChD,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,EAAW,EAAW,EAAE;IACpD,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC;YAC/C,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAa,EAAiB,EAAE;IACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAEpC,IAAI,EAAE,IAAI,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,UAAuC,EAAE,EAAE,CAAC,CAAC;IACjE,UAAU;IACV,SAAS,EAAE,EAAE;IACb,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,iBAAiB,EAAE,CAAC;IACpB,cAAc,EAAE,CAAC;IACjB,kBAAkB,EAAE,EAAE;IACtB,YAAY,EAAE,IAAI;IAClB,yBAAyB,EAAE,KAAK;IAChC,YAAY,EAAE,CAAC;IACf,YAAY,EAAE,KAAK;CACpB,CAAC,CAAC;AAEH,iGAAiG;AACjG,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAAa,EACb,QAAgB,EACD,EAAE;IACjB,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAE5E,OAAO,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAEnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,YAAY,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,MAAM,CACzD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAClC,CAAC;IAEF,OAAO;QACL,UAAU;QACV,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC;QAC9B,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,CACpC,CAAC,EAAE,EAAE,EAAE,CACL,aAAa,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC;YAC5D,CAAC,QAAQ,CAAC,EAAE,CAAC,CAChB,CAAC,MAAM;QACR,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,MAAM,CACpC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CACxD,CAAC,MAAM;QACR,iBAAiB,EAAE,SAAS,CAAC,MAAM;QACnC,cAAc,EAAE,SAAS,CAAC,MAAM,CAC9B,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,EAC7C,CAAC,CACF;QACD,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACjE,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;QAChC,yBAAyB,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAChE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC9B;QACD,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM;QAC3C,YAAY,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC5D,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/html/dom.d.ts

@@ -0,0 +1,17 @@
+import type { AnyNode, Document, Element } from "domhandler";
+export type { AnyNode, Document, Element } from "domhandler";
+/** Elements whose content is never visible page text. */
+export declare const HIDDEN_ELEMENTS: Set<string>;
+/** The text a reader sees under `root`: hidden elements left out, whitespace collapsed. */
+export declare const visibleText: (root: AnyNode) => string;
+/** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
+export declare const parseDocument: (html: string) => Document;
+/** Descendants of `root` matching a CSS selector, in document order. */
+export declare const queryAll: (root: AnyNode, selector: string) => Element[];
+/** First descendant of `root` matching a CSS selector. */
+export declare const queryOne: (root: AnyNode, selector: string) => Element | null;
+export declare const attr: (el: Element, name: string) => string | undefined;
+export declare const hasAttr: (el: Element, name: string) => boolean;
+/** DOM `textContent`: concatenated text of all descendants, comments excluded. */
+export declare const text: (node: AnyNode | AnyNode[]) => string;
+export declare const childElements: (el: Element) => Element[];

package/dist/core/html/dom.js

@@ -0,0 +1,83 @@
+import { selectAll, selectOne } from "css-select";
+import { hasChildren, isTag, isText } from "domhandler";
+import { textContent } from "domutils";
+import { parse } from "parse5";
+import { adapter } from "parse5-htmlparser2-tree-adapter";
+import { collapseWhitespace } from "./text.js";
+/** Elements whose content is never visible page text. */
+export const HIDDEN_ELEMENTS = new Set([
+    "script",
+    "style",
+    "noscript",
+    "template",
+    "svg",
+    "iframe",
+    "head",
+]);
+/** Elements that break words apart; padded with spaces so adjacent words don't fuse. */
+const BLOCK_ELEMENTS = new Set([
+    "p",
+    "div",
+    "li",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "br",
+    "tr",
+    "td",
+    "th",
+    "section",
+    "article",
+    "header",
+    "footer",
+    "nav",
+    "blockquote",
+    "pre",
+    "dd",
+    "dt",
+    "figcaption",
+]);
+/** The text a reader sees under `root`: hidden elements left out, whitespace collapsed. */
+export const visibleText = (root) => {
+    const parts = [];
+    const walk = (nodes) => {
+        for (const node of nodes) {
+            if (isText(node)) {
+                parts.push(node.data);
+            }
+            else if (isTag(node)) {
+                if (HIDDEN_ELEMENTS.has(node.name)) {
+                    continue;
+                }
+                const block = BLOCK_ELEMENTS.has(node.name);
+                if (block) {
+                    parts.push(" ");
+                }
+                walk(node.children);
+                if (block) {
+                    parts.push(" ");
+                }
+            }
+            else if (hasChildren(node)) {
+                walk(node.children);
+            }
+        }
+    };
+    walk(hasChildren(root) ? root.children : [root]);
+    return collapseWhitespace(parts.join(""));
+};
+/** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
+export const parseDocument = (html) => parse(html, { scriptingEnabled: true, treeAdapter: adapter });
+/** Descendants of `root` matching a CSS selector, in document order. */
+export const queryAll = (root, selector) => selectAll(selector, root);
+/** First descendant of `root` matching a CSS selector. */
+export const queryOne = (root, selector) => selectOne(selector, root);
+export const attr = (el, name) => Object.hasOwn(el.attribs, name) ? el.attribs[name] : undefined;
+export const hasAttr = (el, name) => Object.hasOwn(el.attribs, name);
+/** DOM `textContent`: concatenated text of all descendants, comments excluded. */
+export const text = (node) => textContent(node);
+export const childElements = (el) => el.children.filter(isTag);
+//# sourceMappingURL=dom.js.map

package/dist/core/html/dom.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAE1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAI/C,yDAAyD;AACzD,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IACrC,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,KAAK;IACL,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AAEH,wFAAwF;AACxF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,GAAG;IACH,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,SAAS;IACT,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,YAAY;IACZ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,YAAY;CACb,CAAC,CAAC;AAEH,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAa,EAAU,EAAE;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE5C,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEpB,IAAI,KAAK,EAAE,CAAC;oBACV,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjD,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}