@citation-media/html-parser 0.1.0 → 0.1.2

package/dist/clean.js

@@ -1,4 +1,4 @@
-import { looksBinary } from "./text.js";
+import { looksBinary, readAttribute } from "./text.js";
 import { absoluteUrl, isResource, linkKind } from "./urls.js";
 const cookieConsent = [
     "[id^='brlbs-cmpnt']",
@@ -19,6 +19,10 @@ const remove = {
 };
 /** The cleaned page as HTML. */
 export const cleanHtml = (input, options) => {
+    // An empty page has nothing to stream; some runtimes never finish transforming an empty body.
+    if (!(input instanceof Response) && input.trim() === "") {
+        return Promise.resolve("");
+    }
     const base = new URL(options.url);
     const removed = options.remove ?? {};
     let rewriter = new HTMLRewriter();
@@ -122,7 +126,7 @@ export const cleanHtml = (input, options) => {
     if (links) {
         rewriter = rewriter.on("a", {
             element(element) {
-                const href = element.getAttribute("href");
+                const href = readAttribute(element, "href");
                 if (!href) {
                     return;
                 }

package/dist/parse.js

@@ -1,5 +1,5 @@
 import { MarkdownWriter } from "./markdown.js";
-import { collapseWhitespace, decodeEntities, removeHyphenation, } from "./text.js";
+import { collapseWhitespace, decodeEntities, readAttribute, removeHyphenation, } from "./text.js";
 import { absoluteUrl, isResource, linkKind } from "./urls.js";
 const blockTags = new Set([
     "br",
@@ -56,18 +56,28 @@ const hiddenFromMarkdown = [
     ".visually-hidden",
 ];
 const maxHeadings = 200;
-const maxImages = 500;
 const maxResources = 500;
-// Follows an element to its end tag. Void elements such as <img> have none, and HTMLRewriter throws
-// for them; elements closed implicitly, such as <li> without </li>, end with their parent.
-const enter = (element, onEnd) => {
+/**
+ * Follows an element to its end tag. HTMLRewriter keeps only the last end-tag handler an element
+ * gets, so every handler adds its action to the element's list and registers the whole list; the
+ * last registration then runs them all. Void elements such as <img> have no end tag, and
+ * HTMLRewriter throws for them; elements closed implicitly, such as <li> without </li>, end with
+ * their parent.
+ */
+const enter = (pass, element, onEnd) => {
+    const actions = [...pass.endActions, onEnd];
     try {
-        element.onEndTag(onEnd);
-        return true;
+        element.onEndTag(() => {
+            for (const action of actions) {
+                action();
+            }
+        });
     }
     catch {
         return false;
     }
+    pass.endActions = actions;
+    return true;
 };
 const clean = (text) => collapseWhitespace(removeHyphenation(decodeEntities(text))).trim();
 const addLink = (links, found, base) => {
@@ -96,7 +106,7 @@ const collecting = (pass) => pass.inContext > 0;
 const writing = (pass) => collecting(pass) && pass.hidden === 0 && pass.markdown !== undefined;
 const hide = (pass) => ({
     element(element) {
-        if (enter(element, () => (pass.hidden -= 1))) {
+        if (enter(pass, element, () => (pass.hidden -= 1))) {
             pass.hidden += 1;
         }
     },
@@ -108,7 +118,7 @@ const onStructure = (rewriter, pass) => {
         next = next.on(context, {
             element(element) {
                 pass.contextFound = true;
-                if (enter(element, () => (pass.inContext -= 1))) {
+                if (enter(pass, element, () => (pass.inContext -= 1))) {
                     pass.inContext += 1;
                 }
             },
@@ -127,7 +137,7 @@ const onStructure = (rewriter, pass) => {
             if (!collecting(pass)) {
                 return;
             }
-            const classes = element.getAttribute("class")?.split(/\s+/u) ?? [];
+            const classes = readAttribute(element, "class")?.split(/\s+/u) ?? [];
             for (const prefix of classPrefixes) {
                 if (classes.some((name) => name.startsWith(prefix))) {
                     pass.classCounts[prefix] = (pass.classCounts[prefix] ?? 0) + 1;
@@ -138,7 +148,7 @@ const onStructure = (rewriter, pass) => {
                 return;
             }
             pass.markdown?.open(tag);
-            enter(element, () => {
+            enter(pass, element, () => {
                 if (writing(pass)) {
                     pass.markdown?.close(tag);
                 }
@@ -148,7 +158,7 @@ const onStructure = (rewriter, pass) => {
 };
 const onLinks = (rewriter, pass) => rewriter.on("a[href]", {
     element(element) {
-        const href = element.getAttribute("href") ?? "";
+        const href = readAttribute(element, "href") ?? "";
         if (!collecting(pass) || !href) {
             return;
         }
@@ -156,16 +166,16 @@ const onLinks = (rewriter, pass) => rewriter.on("a[href]", {
         const text = [];
         pass.link = text;
         // Attributes are readable only while this handler runs, not in the end-tag handler.
-        const label = element.getAttribute("aria-label") ?? undefined;
-        const rel = element.getAttribute("rel") ?? undefined;
-        const target = element.getAttribute("target") ?? undefined;
+        const label = readAttribute(element, "aria-label") ?? undefined;
+        const rel = readAttribute(element, "rel") ?? undefined;
+        const target = readAttribute(element, "target") ?? undefined;
         const written = writing(pass) &&
             pass.markdownOptions?.links !== false &&
             !href.startsWith("#");
         if (written) {
             pass.markdown?.openLink(url);
         }
-        enter(element, () => {
+        enter(pass, element, () => {
             if (written) {
                 pass.markdown?.closeLink();
             }
@@ -178,23 +188,25 @@ const onLinks = (rewriter, pass) => rewriter.on("a[href]", {
 });
 const onImages = (rewriter, pass) => rewriter.on("img", {
     element(element) {
-        // Lazy-loading scripts often keep the real address in data-src until the image scrolls in.
-        const src = element.getAttribute("src") ?? element.getAttribute("data-src");
-        if (!collecting(pass) || !src || src.startsWith("data:")) {
+        // Lazy-loading scripts keep the real address in data-src and a placeholder in src.
+        const src = [
+            readAttribute(element, "src"),
+            readAttribute(element, "data-src"),
+            readAttribute(element, "data-lazy-src"),
+        ].find((value) => value && !value.startsWith("data:"));
+        if (!collecting(pass) || !src) {
             return;
         }
         const url = absoluteUrl(src, pass.base);
-        const alt = element.getAttribute("alt");
-        if (pass.options.images &&
-            pass.images.size < maxImages &&
-            !pass.images.has(url)) {
+        const alt = readAttribute(element, "alt");
+        if (pass.options.images && !pass.images.has(url)) {
             pass.images.set(url, {
                 alt: alt === null ? null : clean(alt),
-                height: element.getAttribute("height") ?? undefined,
-                loading: element.getAttribute("loading") ?? undefined,
+                height: readAttribute(element, "height") ?? undefined,
+                loading: readAttribute(element, "loading") ?? undefined,
                 srcset: element.hasAttribute("srcset"),
                 url,
-                width: element.getAttribute("width") ?? undefined,
+                width: readAttribute(element, "width") ?? undefined,
             });
         }
         if (writing(pass) && pass.markdownOptions?.images) {
@@ -208,11 +220,11 @@ const onResources = (rewriter, pass) => {
         .on("script", {
         element(element) {
             if (resources.scripts.length < maxResources) {
-                const src = element.getAttribute("src");
+                const src = readAttribute(element, "src");
                 resources.scripts.push({
                     attributes: [...element.attributes].flatMap(([name]) => name ? [name] : []),
                     src: src ? absoluteUrl(src, base) : undefined,
-                    type: element.getAttribute("type") ?? undefined,
+                    type: readAttribute(element, "type") ?? undefined,
                 });
             }
         },
@@ -220,7 +232,7 @@ const onResources = (rewriter, pass) => {
         .on("iframe[src]", {
         element(element) {
             if (resources.frames.length < maxResources) {
-                resources.frames.push(absoluteUrl(element.getAttribute("src") ?? "", base));
+                resources.frames.push(absoluteUrl(readAttribute(element, "src") ?? "", base));
             }
         },
     })
@@ -228,8 +240,8 @@ const onResources = (rewriter, pass) => {
         element(element) {
             if (resources.links.length < maxResources) {
                 resources.links.push({
-                    href: absoluteUrl(element.getAttribute("href") ?? "", base),
-                    rel: element.getAttribute("rel") ?? "",
+                    href: absoluteUrl(readAttribute(element, "href") ?? "", base),
+                    rel: readAttribute(element, "rel") ?? "",
                 });
             }
         },
@@ -245,13 +257,13 @@ const metaFields = new Map([
 const onMeta = (rewriter, pass) => rewriter
     .on("html", {
     element(element) {
-        pass.meta.language = element.getAttribute("lang") ?? undefined;
+        pass.meta.language = readAttribute(element, "lang") ?? undefined;
     },
 })
     .on("title", {
     element(element) {
         pass.title = [];
-        enter(element, () => {
+        enter(pass, element, () => {
             pass.meta.title = clean((pass.title ?? []).join(""));
             pass.title = undefined;
         });
@@ -259,18 +271,18 @@ const onMeta = (rewriter, pass) => rewriter
 })
     .on("meta", {
     element(element) {
-        const name = element.getAttribute("name") ??
-            element.getAttribute("property") ??
+        const name = readAttribute(element, "name") ??
+            readAttribute(element, "property") ??
             "";
         const field = metaFields.get(name.toLowerCase());
         if (field) {
-            pass.meta[field] = element.getAttribute("content") ?? undefined;
+            pass.meta[field] = readAttribute(element, "content") ?? undefined;
         }
     },
 })
     .on("link[rel='canonical']", {
     element(element) {
-        const href = element.getAttribute("href");
+        const href = readAttribute(element, "href");
         pass.meta.canonical = href ? absoluteUrl(href, pass.base) : undefined;
     },
 });
@@ -282,7 +294,7 @@ const onHeadings = (rewriter, pass) => rewriter.on("h1, h2, h3, h4, h5, h6", {
         const level = Number(element.tagName.slice(1));
         pass.headingCounts[level - 1] = (pass.headingCounts[level - 1] ?? 0) + 1;
         pass.heading = { level, text: [] };
-        enter(element, () => {
+        enter(pass, element, () => {
             const text = clean(pass.heading?.text.join("") ?? "");
             if (pass.heading && text && pass.headings.length < maxHeadings) {
                 pass.headings.push({ level: pass.heading.level, text });
@@ -338,6 +350,7 @@ export const parseHtml = async (input, options) => {
         base: new URL(options.url),
         classCounts: Object.fromEntries((options.classPrefixes ?? []).map((prefix) => [prefix, 0])),
         contextFound: false,
+        endActions: [],
         heading: undefined,
         headingCounts: [0, 0, 0, 0, 0, 0],
         headings: [],
@@ -356,7 +369,17 @@ export const parseHtml = async (input, options) => {
         resources: { frames: [], links: [], scripts: [] },
         title: undefined,
     };
-    let rewriter = onImages(onLinks(onStructure(new HTMLRewriter(), pass), pass), pass);
+    // An empty page has nothing to stream; some runtimes never finish transforming an empty body.
+    if (!(input instanceof Response) && input.trim() === "") {
+        return resultOf(pass);
+    }
+    // Registered first, so every element starts with an empty list of end-tag actions.
+    const first = new HTMLRewriter().on("*", {
+        element() {
+            pass.endActions = [];
+        },
+    });
+    let rewriter = onImages(onLinks(onStructure(first, pass), pass), pass);
     if (options.resources) {
         rewriter = onResources(rewriter, pass);
     }

package/dist/text.d.ts

@@ -14,3 +14,8 @@ export declare const collapseWhitespace: (text: string) => string;
  * conservative so umlauts and other normal characters never trigger it.
  */
 export declare const looksBinary: (text: string) => boolean;
+/**
+ * An attribute's value with its entities decoded. HTMLRewriter returns attributes as written in the
+ * source, so `href="?a=1&b=2"` would otherwise keep its `&`.
+ */
+export declare const readAttribute: (element: Element, name: string) => string | null;

package/dist/text.js

@@ -3,38 +3,261 @@
  * parser collects chunks per run of text and decodes them here, so an entity split across two
  * chunks is still decoded correctly.
  */
+// The named entities of HTML 4, which cover what pages use in practice; numeric ones work anyway.
 const named = new Map(Object.entries({
-    Auml: "Ä",
-    Ouml: "Ö",
-    Uuml: "Ü",
+    AElig: "\u00C6",
+    Aacute: "\u00C1",
+    Acirc: "\u00C2",
+    Agrave: "\u00C0",
+    Alpha: "\u0391",
+    Aring: "\u00C5",
+    Atilde: "\u00C3",
+    Auml: "\u00C4",
+    Beta: "\u0392",
+    Ccedil: "\u00C7",
+    Chi: "\u03A7",
+    Dagger: "\u2021",
+    Delta: "\u0394",
+    ETH: "\u00D0",
+    Eacute: "\u00C9",
+    Ecirc: "\u00CA",
+    Egrave: "\u00C8",
+    Epsilon: "\u0395",
+    Eta: "\u0397",
+    Euml: "\u00CB",
+    Gamma: "\u0393",
+    Iacute: "\u00CD",
+    Icirc: "\u00CE",
+    Igrave: "\u00CC",
+    Iota: "\u0399",
+    Iuml: "\u00CF",
+    Kappa: "\u039A",
+    Lambda: "\u039B",
+    Mu: "\u039C",
+    Ntilde: "\u00D1",
+    Nu: "\u039D",
+    OElig: "\u0152",
+    Oacute: "\u00D3",
+    Ocirc: "\u00D4",
+    Ograve: "\u00D2",
+    Omega: "\u03A9",
+    Omicron: "\u039F",
+    Oslash: "\u00D8",
+    Otilde: "\u00D5",
+    Ouml: "\u00D6",
+    Phi: "\u03A6",
+    Pi: "\u03A0",
+    Prime: "\u2033",
+    Psi: "\u03A8",
+    Rho: "\u03A1",
+    Scaron: "\u0160",
+    Sigma: "\u03A3",
+    THORN: "\u00DE",
+    Tau: "\u03A4",
+    Theta: "\u0398",
+    Uacute: "\u00DA",
+    Ucirc: "\u00DB",
+    Ugrave: "\u00D9",
+    Upsilon: "\u03A5",
+    Uuml: "\u00DC",
+    Xi: "\u039E",
+    Yacute: "\u00DD",
+    Yuml: "\u0178",
+    Zeta: "\u0396",
+    aacute: "\u00E1",
+    acirc: "\u00E2",
+    acute: "\u00B4",
+    aelig: "\u00E6",
+    agrave: "\u00E0",
+    alefsym: "\u2135",
+    alpha: "\u03B1",
     amp: "&",
+    and: "\u2227",
+    ang: "\u2220",
     apos: "'",
-    auml: "ä",
-    bdquo: "„",
-    bull: "•",
-    copy: "©",
-    euro: "€",
+    aring: "\u00E5",
+    asymp: "\u2248",
+    atilde: "\u00E3",
+    auml: "\u00E4",
+    bdquo: "\u201E",
+    beta: "\u03B2",
+    brvbar: "\u00A6",
+    bull: "\u2022",
+    cap: "\u2229",
+    ccedil: "\u00E7",
+    cedil: "\u00B8",
+    cent: "\u00A2",
+    chi: "\u03C7",
+    circ: "\u02C6",
+    clubs: "\u2663",
+    cong: "\u2245",
+    copy: "\u00A9",
+    crarr: "\u21B5",
+    cup: "\u222A",
+    curren: "\u00A4",
+    dArr: "\u21D3",
+    dagger: "\u2020",
+    darr: "\u2193",
+    deg: "\u00B0",
+    delta: "\u03B4",
+    diams: "\u2666",
+    divide: "\u00F7",
+    eacute: "\u00E9",
+    ecirc: "\u00EA",
+    egrave: "\u00E8",
+    empty: "\u2205",
+    emsp: "\u2003",
+    ensp: "\u2002",
+    epsilon: "\u03B5",
+    equiv: "\u2261",
+    eta: "\u03B7",
+    eth: "\u00F0",
+    euml: "\u00EB",
+    euro: "\u20AC",
+    exist: "\u2203",
+    fnof: "\u0192",
+    forall: "\u2200",
+    frac12: "\u00BD",
+    frac14: "\u00BC",
+    frac34: "\u00BE",
+    frasl: "\u2044",
+    gamma: "\u03B3",
+    ge: "\u2265",
     gt: ">",
-    hellip: "…",
-    laquo: "«",
-    ldquo: "“",
-    lsquo: "‘",
+    hArr: "\u21D4",
+    harr: "\u2194",
+    hearts: "\u2665",
+    hellip: "\u2026",
+    iacute: "\u00ED",
+    icirc: "\u00EE",
+    iexcl: "\u00A1",
+    igrave: "\u00EC",
+    image: "\u2111",
+    infin: "\u221E",
+    int: "\u222B",
+    iota: "\u03B9",
+    iquest: "\u00BF",
+    isin: "\u2208",
+    iuml: "\u00EF",
+    kappa: "\u03BA",
+    lArr: "\u21D0",
+    lambda: "\u03BB",
+    lang: "\u2329",
+    laquo: "\u00AB",
+    larr: "\u2190",
+    lceil: "\u2308",
+    ldquo: "\u201C",
+    le: "\u2264",
+    lfloor: "\u230A",
+    lowast: "\u2217",
+    loz: "\u25CA",
+    lrm: "\u200E",
+    lsaquo: "\u2039",
+    lsquo: "\u2018",
     lt: "<",
-    mdash: "—",
-    middot: "·",
-    nbsp: " ",
-    ndash: "–",
-    ouml: "ö",
+    macr: "\u00AF",
+    mdash: "\u2014",
+    micro: "\u00B5",
+    middot: "\u00B7",
+    minus: "\u2212",
+    mu: "\u03BC",
+    nabla: "\u2207",
+    nbsp: "\u00A0",
+    ndash: "\u2013",
+    ne: "\u2260",
+    ni: "\u220B",
+    not: "\u00AC",
+    notin: "\u2209",
+    nsub: "\u2284",
+    ntilde: "\u00F1",
+    nu: "\u03BD",
+    oacute: "\u00F3",
+    ocirc: "\u00F4",
+    oelig: "\u0153",
+    ograve: "\u00F2",
+    oline: "\u203E",
+    omega: "\u03C9",
+    omicron: "\u03BF",
+    oplus: "\u2295",
+    or: "\u2228",
+    ordf: "\u00AA",
+    ordm: "\u00BA",
+    oslash: "\u00F8",
+    otilde: "\u00F5",
+    otimes: "\u2297",
+    ouml: "\u00F6",
+    para: "\u00B6",
+    part: "\u2202",
+    permil: "\u2030",
+    perp: "\u22A5",
+    phi: "\u03C6",
+    pi: "\u03C0",
+    piv: "\u03D6",
+    plusmn: "\u00B1",
+    pound: "\u00A3",
+    prime: "\u2032",
+    prod: "\u220F",
+    prop: "\u221D",
+    psi: "\u03C8",
     quot: '"',
-    raquo: "»",
-    rdquo: "”",
-    reg: "®",
-    rsquo: "’",
-    sbquo: "‚",
+    rArr: "\u21D2",
+    radic: "\u221A",
+    rang: "\u232A",
+    raquo: "\u00BB",
+    rarr: "\u2192",
+    rceil: "\u2309",
+    rdquo: "\u201D",
+    real: "\u211C",
+    reg: "\u00AE",
+    rfloor: "\u230B",
+    rho: "\u03C1",
+    rlm: "\u200F",
+    rsaquo: "\u203A",
+    rsquo: "\u2019",
+    sbquo: "\u201A",
+    scaron: "\u0161",
+    sdot: "\u22C5",
+    sect: "\u00A7",
     shy: "",
-    szlig: "ß",
-    trade: "™",
-    uuml: "ü",
+    sigma: "\u03C3",
+    sigmaf: "\u03C2",
+    sim: "\u223C",
+    spades: "\u2660",
+    sub: "\u2282",
+    sube: "\u2286",
+    sum: "\u2211",
+    sup: "\u2283",
+    sup1: "\u00B9",
+    sup2: "\u00B2",
+    sup3: "\u00B3",
+    supe: "\u2287",
+    szlig: "\u00DF",
+    tau: "\u03C4",
+    there4: "\u2234",
+    theta: "\u03B8",
+    thetasym: "\u03D1",
+    thinsp: "\u2009",
+    thorn: "\u00FE",
+    tilde: "\u02DC",
+    times: "\u00D7",
+    trade: "\u2122",
+    uArr: "\u21D1",
+    uacute: "\u00FA",
+    uarr: "\u2191",
+    ucirc: "\u00FB",
+    ugrave: "\u00F9",
+    uml: "\u00A8",
+    upsih: "\u03D2",
+    upsilon: "\u03C5",
+    uuml: "\u00FC",
+    weierp: "\u2118",
+    xi: "\u03BE",
+    yacute: "\u00FD",
+    yen: "\u00A5",
+    yuml: "\u00FF",
+    zeta: "\u03B6",
+    zwj: "\u200D",
+    zwnj: "\u200C",
 }));
 const fromCodePoint = (value) => {
     try {
@@ -75,3 +298,11 @@ export const looksBinary = (text) => {
     const control = text.match(controlCharacters)?.length ?? 0;
     return (text.includes("\u0000") || control > 20 || control / text.length > 0.05);
 };
+/**
+ * An attribute's value with its entities decoded. HTMLRewriter returns attributes as written in the
+ * source, so `href="?a=1&amp;b=2"` would otherwise keep its `&amp;`.
+ */
+export const readAttribute = (element, name) => {
+    const value = element.getAttribute(name);
+    return value === null ? null : decodeEntities(value);
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@citation-media/html-parser",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Streaming HTML parser for Cloudflare Workers: links, images, resources, and Markdown without a DOM, built on HTMLRewriter.",
   "keywords": [
     "cloudflare-workers",