entities 6.0.0 → 7.0.0

package/src/decode.ts

@@ -1,6 +1,7 @@
+import { fromCodePoint, replaceCodePoint } from "./decode-codepoint.js";
 import { htmlDecodeTree } from "./generated/decode-data-html.js";
 import { xmlDecodeTree } from "./generated/decode-data-xml.js";
-import { replaceCodePoint, fromCodePoint } from "./decode-codepoint.js";
+import { BinTrieFlags } from "./internal/bin-trie-flags.js";
 
 const enum CharCodes {
     NUM = 35, // "#"
@@ -20,12 +21,6 @@ const enum CharCodes {
 /** Bit that needs to be set to convert an upper case ASCII character to lower case */
 const TO_LOWER_BIT = 0b10_0000;
 
-export enum BinTrieFlags {
-    VALUE_LENGTH = 0b1100_0000_0000_0000,
-    BRANCH_LENGTH = 0b0011_1111_1000_0000,
-    JUMP_TABLE = 0b0000_0000_0111_1111,
-}
-
 function isNumber(code: number): boolean {
     return code >= CharCodes.ZERO && code <= CharCodes.NINE;
 }
@@ -89,6 +84,7 @@ export interface EntityErrorProducer {
 export class EntityDecoder {
     constructor(
         /** The tree used to decode entities. */
+        // biome-ignore lint/correctness/noUnusedPrivateClassMembers: False positive
         private readonly decodeTree: Uint16Array,
         /**
          * The function that is called when a codepoint is decoded.
@@ -198,21 +194,6 @@ export class EntityDecoder {
         return this.stateNumericDecimal(input, offset);
     }
 
-    private addToNumericResult(
-        input: string,
-        start: number,
-        end: number,
-        base: number,
-    ): void {
-        if (start !== end) {
-            const digitCount = end - start;
-            this.result =
-                this.result * Math.pow(base, digitCount) +
-                Number.parseInt(input.substr(start, digitCount), base);
-            this.consumed += digitCount;
-        }
-    }
-
     /**
      * Parses a hexadecimal numeric entity.
      *
@@ -223,21 +204,22 @@ export class EntityDecoder {
      * @returns The number of characters that were consumed, or -1 if the entity is incomplete.
      */
     private stateNumericHex(input: string, offset: number): number {
-        const startIndex = offset;
-
         while (offset < input.length) {
             const char = input.charCodeAt(offset);
             if (isNumber(char) || isHexadecimalCharacter(char)) {
-                offset += 1;
+                // Convert hex digit to value (0-15); 'a'/'A' -> 10.
+                const digit =
+                    char <= CharCodes.NINE
+                        ? char - CharCodes.ZERO
+                        : (char | TO_LOWER_BIT) - CharCodes.LOWER_A + 10;
+                this.result = this.result * 16 + digit;
+                this.consumed++;
+                offset++;
             } else {
-                this.addToNumericResult(input, startIndex, offset, 16);
                 return this.emitNumericEntity(char, 3);
             }
         }
-
-        this.addToNumericResult(input, startIndex, offset, 16);
-
-        return -1;
+        return -1; // Incomplete entity
     }
 
     /**
@@ -250,21 +232,17 @@ export class EntityDecoder {
      * @returns The number of characters that were consumed, or -1 if the entity is incomplete.
      */
     private stateNumericDecimal(input: string, offset: number): number {
-        const startIndex = offset;
-
         while (offset < input.length) {
             const char = input.charCodeAt(offset);
             if (isNumber(char)) {
-                offset += 1;
+                this.result = this.result * 10 + (char - CharCodes.ZERO);
+                this.consumed++;
+                offset++;
             } else {
-                this.addToNumericResult(input, startIndex, offset, 10);
                 return this.emitNumericEntity(char, 2);
             }
         }
-
-        this.addToNumericResult(input, startIndex, offset, 10);
-
-        return -1;
+        return -1; // Incomplete entity
     }
 
     /**
@@ -321,12 +299,78 @@ export class EntityDecoder {
     private stateNamedEntity(input: string, offset: number): number {
         const { decodeTree } = this;
         let current = decodeTree[this.treeIndex];
-        // The mask is the number of bytes of the value, including the current byte.
+        // The length is the number of bytes of the value, including the current byte.
         let valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14;
 
-        for (; offset < input.length; offset++, this.excess++) {
+        while (offset < input.length) {
+            // Handle compact runs (possibly inline): valueLength == 0 and SEMI_REQUIRED bit set.
+            if (valueLength === 0 && (current & BinTrieFlags.FLAG13) !== 0) {
+                const runLength =
+                    (current & BinTrieFlags.BRANCH_LENGTH) >> 7; /* 2..63 */
+                const firstChar = current & BinTrieFlags.JUMP_TABLE;
+                // Fast-fail if we don't have enough remaining input for the full run (incomplete entity)
+                if (offset + runLength > input.length) return -1;
+                // Verify first char
+                if (input.charCodeAt(offset) !== firstChar) {
+                    return this.result === 0
+                        ? 0
+                        : this.emitNotTerminatedNamedEntity();
+                }
+                offset++;
+                this.excess++;
+                // Remaining characters after the first
+                const remaining = runLength - 1;
+                // Iterate over packed 2-char words
+                for (let runPos = 1; runPos < runLength; runPos += 2) {
+                    const packedWord =
+                        decodeTree[this.treeIndex + 1 + ((runPos - 1) >> 1)];
+                    const low = packedWord & 0xff;
+                    if (input.charCodeAt(offset) !== low) {
+                        return this.result === 0
+                            ? 0
+                            : this.emitNotTerminatedNamedEntity();
+                    }
+                    offset++;
+                    this.excess++;
+                    const high = (packedWord >> 8) & 0xff;
+                    if (runPos + 1 < runLength) {
+                        if (input.charCodeAt(offset) !== high) {
+                            return this.result === 0
+                                ? 0
+                                : this.emitNotTerminatedNamedEntity();
+                        }
+                        offset++;
+                        this.excess++;
+                    }
+                }
+                this.treeIndex += 1 + ((remaining + 1) >> 1);
+                current = decodeTree[this.treeIndex];
+                valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14;
+            }
+
+            if (offset >= input.length) break;
+
             const char = input.charCodeAt(offset);
 
+            /*
+             * Implicit semicolon handling for nodes that require a semicolon but
+             * don't have an explicit ';' branch stored in the trie. If we have
+             * a value on the current node, it requires a semicolon, and the
+             * current input character is a semicolon, emit the entity using the
+             * current node (without descending further).
+             */
+            if (
+                char === CharCodes.SEMI &&
+                valueLength !== 0 &&
+                (current & BinTrieFlags.FLAG13) !== 0
+            ) {
+                return this.emitNamedEntityData(
+                    this.treeIndex,
+                    valueLength,
+                    this.consumed + this.excess,
+                );
+            }
+
             this.treeIndex = determineBranch(
                 decodeTree,
                 current,
@@ -361,12 +405,18 @@ export class EntityDecoder {
                 }
 
                 // If we encounter a non-terminated (legacy) entity while parsing strictly, then ignore it.
-                if (this.decodeMode !== DecodingMode.Strict) {
+                if (
+                    this.decodeMode !== DecodingMode.Strict &&
+                    (current & BinTrieFlags.FLAG13) === 0
+                ) {
                     this.result = this.treeIndex;
                     this.consumed += this.excess;
                     this.excess = 0;
                 }
             }
+            // Increment offset & excess for next iteration
+            offset++;
+            this.excess++;
         }
 
         return -1;
@@ -407,7 +457,8 @@ export class EntityDecoder {
 
         this.emitCodePoint(
             valueLength === 1
-                ? decodeTree[result] & ~BinTrieFlags.VALUE_LENGTH
+                ? decodeTree[result] &
+                      ~(BinTrieFlags.VALUE_LENGTH | BinTrieFlags.FLAG13)
                 : decodeTree[result + 1],
             consumed,
         );
@@ -540,22 +591,28 @@ export function determineBranch(
             : decodeTree[nodeIndex + value] - 1;
     }
 
-    // Case 3: Multiple branches encoded in dictionary
+    // Case 3: Multiple branches encoded in packed dictionary (two keys per uint16)
+    const packedKeySlots = (branchCount + 1) >> 1;
 
-    // Binary search for the character.
-    let lo = nodeIndex;
-    let hi = lo + branchCount - 1;
+    /*
+     * Treat packed keys as a virtual sorted array of length `branchCount`.
+     * Key(i) = low byte for even i, high byte for odd i in slot i>>1.
+     */
+    let lo = 0;
+    let hi = branchCount - 1;
 
     while (lo <= hi) {
         const mid = (lo + hi) >>> 1;
-        const midValue = decodeTree[mid];
+        const slot = mid >> 1;
+        const packed = decodeTree[nodeIndex + slot];
+        const midKey = (packed >> ((mid & 1) * 8)) & 0xff;
 
-        if (midValue < char) {
+        if (midKey < char) {
             lo = mid + 1;
-        } else if (midValue > char) {
+        } else if (midKey > char) {
             hi = mid - 1;
         } else {
-            return decodeTree[mid + branchCount];
+            return decodeTree[nodeIndex + packedKeySlots + mid];
         }
     }
 
@@ -609,12 +666,11 @@ export function decodeXML(xmlString: string): string {
     return xmlDecoder(xmlString, DecodingMode.Strict);
 }
 
-// Re-export for use by eg. htmlparser2
-export { htmlDecodeTree } from "./generated/decode-data-html.js";
-export { xmlDecodeTree } from "./generated/decode-data-xml.js";
-
 export {
     decodeCodePoint,
-    replaceCodePoint,
     fromCodePoint,
+    replaceCodePoint,
 } from "./decode-codepoint.js";
+// Re-export for use by eg. htmlparser2
+export { htmlDecodeTree } from "./generated/decode-data-html.js";
+export { xmlDecodeTree } from "./generated/decode-data-xml.js";

package/src/encode.spec.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from "vitest";
+import { describe, expect, it } from "vitest";
 import * as entities from "./index.js";
 
 describe("Encode->decode test", () => {

package/src/encode.ts

@@ -1,7 +1,17 @@
+import { getCodePoint, XML_BITSET_VALUE } from "./escape.js";
 import { htmlTrie } from "./generated/encode-html.js";
-import { xmlReplacer, getCodePoint } from "./escape.js";
 
-const htmlReplacer = /[\t\n\f!-,./:-@[-`{-}\u0080-\uFFFF]/g;
+/**
+ * We store the characters to consider as a compact bitset for fast lookups.
+ */
+const HTML_BITSET = /* #__PURE__ */ new Uint32Array([
+    0x16_00, // Bits for 09,0A,0C
+    0xfc_00_ff_fe, // 32..63 -> 21-2D (minus space), 2E,2F,3A-3F
+    0xf8_00_00_01, // 64..95 -> 40, 5B-5F
+    0x38_00_00_01, // 96..127-> 60, 7B-7D
+]);
+
+const XML_BITSET = /* #__PURE__ */ new Uint32Array([0, XML_BITSET_VALUE, 0, 0]);
 
 /**
  * Encodes all characters in the input using HTML entities. This includes
@@ -15,7 +25,7 @@ const htmlReplacer = /[\t\n\f!-,./:-@[-`{-}\u0080-\uFFFF]/g;
  * (eg. `ü`) will be used.
  */
 export function encodeHTML(input: string): string {
-    return encodeHTMLTrieRe(htmlReplacer, input);
+    return encodeHTMLTrieRe(HTML_BITSET, input);
 }
 /**
  * Encodes all non-ASCII characters, as well as characters not valid in HTML
@@ -26,52 +36,58 @@ export function encodeHTML(input: string): string {
  * (eg. `ü`) will be used.
  */
 export function encodeNonAsciiHTML(input: string): string {
-    return encodeHTMLTrieRe(xmlReplacer, input);
+    return encodeHTMLTrieRe(XML_BITSET, input);
 }
 
-function encodeHTMLTrieRe(regExp: RegExp, input: string): string {
-    let returnValue = "";
-    let lastIndex = 0;
-    let match;
+function encodeHTMLTrieRe(bitset: Uint32Array, input: string): string {
+    let out: string | undefined;
+    let last = 0; // Start of the next untouched slice.
+    const { length } = input;
 
-    while ((match = regExp.exec(input)) !== null) {
-        const { index } = match;
-        returnValue += input.substring(lastIndex, index);
+    for (let index = 0; index < length; index++) {
         const char = input.charCodeAt(index);
-        let next = htmlTrie.get(char);
+        // Skip ASCII characters that don't need encoding
+        if (char < 0x80 && !((bitset[char >>> 5] >>> char) & 1)) {
+            continue;
+        }
 
-        if (typeof next === "object") {
-            // We are in a branch. Try to match the next char.
-            if (index + 1 < input.length) {
+        if (out === undefined) out = input.substring(0, index);
+        else if (last !== index) out += input.substring(last, index);
+
+        let node = htmlTrie.get(char);
+
+        if (typeof node === "object") {
+            if (index + 1 < length) {
                 const nextChar = input.charCodeAt(index + 1);
                 const value =
-                    typeof next.n === "number"
-                        ? next.n === nextChar
-                            ? next.o
+                    typeof node.next === "number"
+                        ? node.next === nextChar
+                            ? node.nextValue
                             : undefined
-                        : next.n.get(nextChar);
+                        : node.next.get(nextChar);
 
                 if (value !== undefined) {
-                    returnValue += value;
-                    lastIndex = regExp.lastIndex += 1;
+                    out += value;
+                    index++;
+                    last = index + 1;
                     continue;
                 }
             }
-
-            next = next.v;
+            node = node.value;
         }
 
-        // We might have a tree node without a value; skip and use a numeric entity.
-        if (next === undefined) {
+        if (node === undefined) {
             const cp = getCodePoint(input, index);
-            returnValue += `&#x${cp.toString(16)};`;
-            // Increase by 1 if we have a surrogate pair
-            lastIndex = regExp.lastIndex += Number(cp !== char);
+            out += `&#x${cp.toString(16)};`;
+            if (cp !== char) index++;
+            last = index + 1;
         } else {
-            returnValue += next;
-            lastIndex = index + 1;
+            out += node;
+            last = index + 1;
         }
     }
 
-    return returnValue + input.substr(lastIndex);
+    if (out === undefined) return input;
+    if (last < length) out += input.substr(last);
+    return out;
 }

package/src/escape.spec.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from "vitest";
+import { describe, expect, it } from "vitest";
 import * as entities from "./index.js";
 
 describe("escape HTML", () => {

package/src/escape.ts

@@ -1,5 +1,3 @@
-export const xmlReplacer: RegExp = /["$&'<>\u0080-\uFFFF]/g;
-
 const xmlCodeMap = new Map([
     [34, "&quot;"],
     [38, "&amp;"],
@@ -22,39 +20,54 @@ export const getCodePoint: (c: string, index: number) => number =
         : // http://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
           (input: string, index: number): number => input.codePointAt(index)!;
 
+/**
+ * Bitset for ASCII characters that need to be escaped in XML.
+ */
+export const XML_BITSET_VALUE = 0x50_00_00_c4; // 32..63 -> 34 ("),38 (&),39 ('),60 (<),62 (>)
+
 /**
  * Encodes all non-ASCII characters, as well as characters not valid in XML
- * documents using XML entities.
+ * documents using XML entities. Uses a fast bitset scan instead of RegExp.
  *
- * If a character has no equivalent entity, a
- * numeric hexadecimal reference (eg. `&#xfc;`) will be used.
+ * If a character has no equivalent entity, a numeric hexadecimal reference
+ * (eg. `&#xfc;`) will be used.
  */
 export function encodeXML(input: string): string {
-    let returnValue = "";
-    let lastIndex = 0;
-    let match;
+    let out: string | undefined;
+    let last = 0;
+    const { length } = input;
 
-    while ((match = xmlReplacer.exec(input)) !== null) {
-        const { index } = match;
+    for (let index = 0; index < length; index++) {
         const char = input.charCodeAt(index);
-        const next = xmlCodeMap.get(char);
-
-        if (next === undefined) {
-            returnValue += `${input.substring(lastIndex, index)}&#x${getCodePoint(
-                input,
-                index,
-            ).toString(16)};`;
-            // Increase by 1 if we have a surrogate pair
-            lastIndex = xmlReplacer.lastIndex += Number(
-                (char & 0xfc_00) === 0xd8_00,
-            );
-        } else {
-            returnValue += input.substring(lastIndex, index) + next;
-            lastIndex = index + 1;
+
+        // Check for ASCII chars that don't need escaping
+        if (
+            char < 0x80 &&
+            (((XML_BITSET_VALUE >>> char) & 1) === 0 || char >= 64 || char < 32)
+        ) {
+            continue;
         }
+
+        if (out === undefined) out = input.substring(0, index);
+        else if (last !== index) out += input.substring(last, index);
+
+        if (char < 64) {
+            // Known replacement
+            out += xmlCodeMap.get(char)!;
+            last = index + 1;
+            continue;
+        }
+
+        // Non-ASCII: encode as numeric entity (handle surrogate pair)
+        const cp = getCodePoint(input, index);
+        out += `&#x${cp.toString(16)};`;
+        if (cp !== char) index++; // Skip trailing surrogate
+        last = index + 1;
     }
 
-    return returnValue + input.substr(lastIndex);
+    if (out === undefined) return input;
+    if (last < length) out += input.substr(last);
+    return out;
 }
 
 /**
@@ -83,7 +96,7 @@ function getEscaper(
     map: Map<number, string>,
 ): (data: string) => string {
     return function escape(data: string): string {
-        let match;
+        let match: RegExpExecArray | null;
         let lastIndex = 0;
         let result = "";