entities 4.5.0 → 6.0.0

package/package.json

@@ -1,90 +1,116 @@
 {
     "name": "entities",
-    "version": "4.5.0",
+    "version": "6.0.0",
     "description": "Encode & decode XML and HTML entities with ease & speed",
-    "author": "Felix Boehm <me@feedic.com>",
-    "funding": "https://github.com/fb55/entities?sponsor=1",
-    "sideEffects": false,
     "keywords": [
-        "entity",
-        "decoding",
-        "encoding",
-        "html",
-        "xml",
-        "html entities"
+        "html entities",
+        "entity decoder",
+        "entity encoding",
+        "html decoding",
+        "html encoding",
+        "xml decoding",
+        "xml encoding"
     ],
-    "directories": {
-        "lib": "lib/"
+    "repository": {
+        "type": "git",
+        "url": "git://github.com/fb55/entities.git"
     },
-    "main": "lib/index.js",
-    "types": "lib/index.d.ts",
-    "module": "lib/esm/index.js",
+    "funding": "https://github.com/fb55/entities?sponsor=1",
+    "license": "BSD-2-Clause",
+    "author": "Felix Boehm <me@feedic.com>",
+    "sideEffects": false,
+    "type": "module",
     "exports": {
         ".": {
-            "require": "./lib/index.js",
-            "import": "./lib/esm/index.js"
+            "import": {
+                "types": "./dist/esm/index.d.ts",
+                "default": "./dist/esm/index.js"
+            },
+            "require": {
+                "types": "./dist/commonjs/index.d.ts",
+                "default": "./dist/commonjs/index.js"
+            }
         },
-        "./lib/decode.js": {
-            "require": "./lib/decode.js",
-            "import": "./lib/esm/decode.js"
+        "./decode": {
+            "import": {
+                "types": "./dist/esm/decode.d.ts",
+                "default": "./dist/esm/decode.js"
+            },
+            "require": {
+                "types": "./dist/commonjs/decode.d.ts",
+                "default": "./dist/commonjs/decode.js"
+            }
         },
-        "./lib/escape.js": {
-            "require": "./lib/escape.js",
-            "import": "./lib/esm/escape.js"
+        "./escape": {
+            "import": {
+                "types": "./dist/esm/escape.d.ts",
+                "default": "./dist/esm/escape.js"
+            },
+            "require": {
+                "types": "./dist/commonjs/escape.d.ts",
+                "default": "./dist/commonjs/escape.js"
+            }
         }
     },
+    "main": "./dist/commonjs/index.js",
+    "module": "./dist/esm/index.js",
+    "types": "./dist/commonjs/index.d.ts",
     "files": [
-        "lib/**/*"
+        "decode.js",
+        "escape.js",
+        "dist",
+        "src"
     ],
-    "engines": {
-        "node": ">=0.12"
-    },
-    "devDependencies": {
-        "@types/jest": "^28.1.8",
-        "@types/node": "^18.15.11",
-        "@typescript-eslint/eslint-plugin": "^5.58.0",
-        "@typescript-eslint/parser": "^5.58.0",
-        "eslint": "^8.38.0",
-        "eslint-config-prettier": "^8.8.0",
-        "eslint-plugin-node": "^11.1.0",
-        "jest": "^28.1.3",
-        "prettier": "^2.8.7",
-        "ts-jest": "^28.0.8",
-        "typedoc": "^0.24.1",
-        "typescript": "^5.0.4"
-    },
     "scripts": {
-        "test": "npm run test:jest && npm run lint",
-        "test:jest": "jest",
-        "lint": "npm run lint:es && npm run lint:prettier",
-        "lint:es": "eslint .",
-        "lint:prettier": "npm run prettier -- --check",
+        "build:docs": "typedoc --hideGenerator src/index.ts",
+        "build:encode-trie": "node --import=tsx scripts/write-encode-map.ts",
+        "build:trie": "node --import=tsx scripts/write-decode-map.ts",
         "format": "npm run format:es && npm run format:prettier",
         "format:es": "npm run lint:es -- --fix",
         "format:prettier": "npm run prettier -- --write",
+        "lint": "npm run lint:es && npm run lint:ts && npm run lint:prettier",
+        "lint:es": "eslint . --ignore-path .gitignore",
+        "lint:prettier": "npm run prettier -- --check",
+        "lint:ts": "tsc --noEmit",
+        "prepublishOnly": "tshy",
         "prettier": "prettier '**/*.{ts,md,json,yml}'",
-        "build": "npm run build:cjs && npm run build:esm",
-        "build:cjs": "tsc --sourceRoot https://raw.githubusercontent.com/fb55/entities/$(git rev-parse HEAD)/src/",
-        "build:esm": "npm run build:cjs -- --module esnext --target es2019 --outDir lib/esm && echo '{\"type\":\"module\"}' > lib/esm/package.json",
-        "build:docs": "typedoc --hideGenerator src/index.ts",
-        "build:trie": "ts-node scripts/write-decode-map.ts",
-        "build:encode-trie": "ts-node scripts/write-encode-map.ts",
-        "prepare": "npm run build"
+        "test": "npm run test:vi && npm run lint",
+        "test:vi": "vitest run"
     },
-    "repository": {
-        "type": "git",
-        "url": "git://github.com/fb55/entities.git"
+    "prettier": {
+        "proseWrap": "always",
+        "tabWidth": 4
     },
-    "license": "BSD-2-Clause",
-    "jest": {
-        "preset": "ts-jest",
-        "coverageProvider": "v8",
-        "moduleNameMapper": {
-            "^(.*)\\.js$": "$1"
-        }
+    "devDependencies": {
+        "@types/node": "^22.10.2",
+        "@typescript-eslint/eslint-plugin": "^8.18.1",
+        "@typescript-eslint/parser": "^8.18.0",
+        "@vitest/coverage-v8": "^2.1.8",
+        "eslint": "^8.57.1",
+        "eslint-config-prettier": "^9.1.0",
+        "eslint-plugin-n": "^17.15.1",
+        "eslint-plugin-unicorn": "^56.0.1",
+        "prettier": "^3.4.2",
+        "tshy": "^3.0.2",
+        "tsx": "^4.19.2",
+        "typedoc": "^0.27.5",
+        "typescript": "^5.7.2",
+        "vitest": "^2.0.2"
     },
-    "prettier": {
-        "tabWidth": 4,
-        "proseWrap": "always"
+    "engines": {
+        "node": ">=0.12"
+    },
+    "tshy": {
+        "exclude": [
+            "**/*.spec.ts",
+            "**/__fixtures__/*",
+            "**/__tests__/*",
+            "**/__snapshots__/*"
+        ],
+        "exports": {
+            ".": "./src/index.ts",
+            "./decode": "./src/decode.ts",
+            "./escape": "./src/escape.ts"
+        }
     }
 }

package/readme.md

@@ -4,16 +4,16 @@ Encode & decode HTML & XML entities with ease & speed.
 
 ## Features
 
--   😇 Tried and true: `entities` is used by many popular libraries; eg.
-    [`htmlparser2`](https://github.com/fb55/htmlparser2), the official
-    [AWS SDK](https://github.com/aws/aws-sdk-js-v3) and
-    [`commonmark`](https://github.com/commonmark/commonmark.js) use it to
-    process HTML entities.
--   ⚡️ Fast: `entities` is the fastest library for decoding HTML entities (as
-    of April 2022); see [performance](#performance).
--   🎛 Configurable: Get an output tailored for your needs. You are fine with
-    UTF8? That'll save you some bytes. Prefer to only have ASCII characters? We
-    can do that as well!
+- 😇 Tried and true: `entities` is used by many popular libraries; eg.
+  [`htmlparser2`](https://github.com/fb55/htmlparser2), the official
+  [AWS SDK](https://github.com/aws/aws-sdk-js-v3) and
+  [`commonmark`](https://github.com/commonmark/commonmark.js) use it to process
+  HTML entities.
+- ⚡️ Fast: `entities` is the fastest library for decoding HTML entities (as of
+  April 2022); see [performance](#performance).
+- 🎛 Configurable: Get an output tailored for your needs. You are fine with
+  UTF8? That'll save you some bytes. Prefer to only have ASCII characters? We
+  can do that as well!
 
 ## How to…
 
@@ -90,15 +90,15 @@ Instead, use the functions that you need directly.
 This library wouldn't be possible without the work of these individuals. Thanks
 to
 
--   [@mathiasbynens](https://github.com/mathiasbynens) for his explanations
-    about character encodings, and his library `he`, which was one of the
-    inspirations for `entities`
--   [@inikulin](https://github.com/inikulin) for his work on optimized tries for
-    decoding HTML entities for the `parse5` project
--   [@mdevils](https://github.com/mdevils) for taking on the challenge of
-    producing a quick entity library with his `html-entities` library.
-    `entities` would be quite a bit slower if there wasn't any competition.
-    Right now `entities` is on top, but we'll see how long that lasts!
+- [@mathiasbynens](https://github.com/mathiasbynens) for his explanations about
+  character encodings, and his library `he`, which was one of the inspirations
+  for `entities`
+- [@inikulin](https://github.com/inikulin) for his work on optimized tries for
+  decoding HTML entities for the `parse5` project
+- [@mdevils](https://github.com/mdevils) for taking on the challenge of
+  producing a quick entity library with his `html-entities` library. `entities`
+  would be quite a bit slower if there wasn't any competition. Right now
+  `entities` is on top, but we'll see how long that lasts!
 
 ---
 

package/src/decode-codepoint.ts

@@ -0,0 +1,81 @@
+// Adapted from https://github.com/mathiasbynens/he/blob/36afe179392226cf1b6ccdb16ebbb7a5a844d93a/src/he.js#L106-L134
+
+const decodeMap = new Map([
+    [0, 65_533],
+    // C1 Unicode control character reference replacements
+    [128, 8364],
+    [130, 8218],
+    [131, 402],
+    [132, 8222],
+    [133, 8230],
+    [134, 8224],
+    [135, 8225],
+    [136, 710],
+    [137, 8240],
+    [138, 352],
+    [139, 8249],
+    [140, 338],
+    [142, 381],
+    [145, 8216],
+    [146, 8217],
+    [147, 8220],
+    [148, 8221],
+    [149, 8226],
+    [150, 8211],
+    [151, 8212],
+    [152, 732],
+    [153, 8482],
+    [154, 353],
+    [155, 8250],
+    [156, 339],
+    [158, 382],
+    [159, 376],
+]);
+
+/**
+ * Polyfill for `String.fromCodePoint`. It is used to create a string from a Unicode code point.
+ */
+export const fromCodePoint: (...codePoints: number[]) => string =
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, n/no-unsupported-features/es-builtins
+    String.fromCodePoint ??
+    function (codePoint: number): string {
+        let output = "";
+
+        if (codePoint > 0xff_ff) {
+            codePoint -= 0x1_00_00;
+            output += String.fromCharCode(
+                ((codePoint >>> 10) & 0x3_ff) | 0xd8_00,
+            );
+            codePoint = 0xdc_00 | (codePoint & 0x3_ff);
+        }
+
+        output += String.fromCharCode(codePoint);
+        return output;
+    };
+
+/**
+ * Replace the given code point with a replacement character if it is a
+ * surrogate or is outside the valid range. Otherwise return the code
+ * point unchanged.
+ */
+export function replaceCodePoint(codePoint: number): number {
+    if (
+        (codePoint >= 0xd8_00 && codePoint <= 0xdf_ff) ||
+        codePoint > 0x10_ff_ff
+    ) {
+        return 0xff_fd;
+    }
+
+    return decodeMap.get(codePoint) ?? codePoint;
+}
+
+/**
+ * Replace the code point if relevant, then convert it to a string.
+ *
+ * @deprecated Use `fromCodePoint(replaceCodePoint(codePoint))` instead.
+ * @param codePoint The code point to decode.
+ * @returns The decoded code point.
+ */
+export function decodeCodePoint(codePoint: number): string {
+    return fromCodePoint(replaceCodePoint(codePoint));
+}

package/src/decode.spec.ts

@@ -0,0 +1,320 @@
+import { describe, it, expect, vitest } from "vitest";
+import * as entities from "./decode.js";
+
+describe("Decode test", () => {
+    const testcases = [
+        { input: "&", output: "&" },
+        { input: "&", output: "&" },
+        { input: "&", output: "&" },
+        { input: "&", output: "&" },
+        { input: "&", output: "&" },
+        { input: "&", output: "&" },
+        { input: "&", output: "&" },
+        { input: ":", output: ":" },
+        { input: ":", output: ":" },
+        { input: ":", output: ":" },
+        { input: ":", output: ":" },
+        { input: "&#", output: "&#" },
+        { input: "&>", output: "&>" },
+        { input: "id=770&#anchor", output: "id=770&#anchor" },
+    ];
+
+    for (const { input, output } of testcases) {
+        it(`should XML decode ${input}`, () =>
+            expect(entities.decodeXML(input)).toBe(output));
+        it(`should HTML decode ${input}`, () =>
+            expect(entities.decodeHTML(input)).toBe(output));
+    }
+
+    it("should HTML decode partial legacy entity", () => {
+        expect(entities.decodeHTMLStrict("&timesbar")).toBe("&timesbar");
+        expect(entities.decodeHTML("&timesbar")).toBe("×bar");
+    });
+
+    it("should HTML decode legacy entities according to spec", () =>
+        expect(entities.decodeHTML("?&image_uri=1&ℑ=2&image=3")).toBe(
+            "?&image_uri=1&ℑ=2&image=3",
+        ));
+
+    it("should back out of legacy entities", () =>
+        expect(entities.decodeHTML("&ampa")).toBe("&a"));
+
+    it("should not parse numeric entities in strict mode", () =>
+        expect(entities.decodeHTMLStrict("&#55")).toBe("&#55"));
+
+    it("should parse &nbsp followed by < (#852)", () =>
+        expect(entities.decodeHTML("&nbsp<")).toBe("\u00A0<"));
+
+    it("should decode trailing legacy entities", () => {
+        expect(entities.decodeHTML("&timesbar;&timesbar")).toBe("⨱×bar");
+    });
+
+    it("should decode multi-byte entities", () => {
+        expect(entities.decodeHTML("&NotGreaterFullEqual;")).toBe("≧̸");
+    });
+
+    it("should not decode legacy entities followed by text in attribute mode", () => {
+        expect(
+            entities.decodeHTML("&not", entities.DecodingMode.Attribute),
+        ).toBe("¬");
+
+        expect(
+            entities.decodeHTML("&noti", entities.DecodingMode.Attribute),
+        ).toBe("&noti");
+
+        expect(
+            entities.decodeHTML("&not=", entities.DecodingMode.Attribute),
+        ).toBe("&not=");
+
+        expect(entities.decodeHTMLAttribute("&notp")).toBe("&notp");
+        expect(entities.decodeHTMLAttribute("&notP")).toBe("&notP");
+        expect(entities.decodeHTMLAttribute("&not3")).toBe("&not3");
+    });
+});
+
+describe("EntityDecoder", () => {
+    it("should decode decimal entities", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        expect(decoder.write("&#5", 1)).toBe(-1);
+        expect(decoder.write("8;", 0)).toBe(5);
+
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 5);
+    });
+
+    it("should decode hex entities", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        expect(decoder.write("&#x3a;", 1)).toBe(6);
+
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 6);
+    });
+
+    it("should decode named entities", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        expect(decoder.write("&amp;", 1)).toBe(5);
+
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);
+    });
+
+    it("should decode legacy entities", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+        decoder.startEntity(entities.DecodingMode.Legacy);
+
+        expect(decoder.write("&amp", 1)).toBe(-1);
+
+        expect(callback).toHaveBeenCalledTimes(0);
+
+        expect(decoder.end()).toBe(4);
+
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 4);
+    });
+
+    it("should decode named entity written character by character", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        for (const c of "amp") {
+            expect(decoder.write(c, 0)).toBe(-1);
+        }
+        expect(decoder.write(";", 0)).toBe(5);
+
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);
+    });
+
+    it("should decode numeric entity written character by character", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        for (const c of "#x3a") {
+            expect(decoder.write(c, 0)).toBe(-1);
+        }
+        expect(decoder.write(";", 0)).toBe(6);
+
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 6);
+    });
+
+    it("should decode hex entities across several chunks", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        for (const chunk of ["#x", "cf", "ff", "d"]) {
+            expect(decoder.write(chunk, 0)).toBe(-1);
+        }
+
+        expect(decoder.write(";", 0)).toBe(9);
+        expect(callback).toHaveBeenCalledTimes(1);
+        expect(callback).toHaveBeenCalledWith(0xc_ff_fd, 9);
+    });
+
+    it("should not fail if nothing is written", () => {
+        const callback = vitest.fn();
+        const decoder = new entities.EntityDecoder(
+            entities.htmlDecodeTree,
+            callback,
+        );
+
+        expect(decoder.end()).toBe(0);
+        expect(callback).toHaveBeenCalledTimes(0);
+    });
+
+    describe("errors", () => {
+        it("should produce an error for a named entity without a semicolon", () => {
+            const errorHandlers = {
+                missingSemicolonAfterCharacterReference: vitest.fn(),
+                absenceOfDigitsInNumericCharacterReference: vitest.fn(),
+                validateNumericCharacterReference: vitest.fn(),
+            };
+            const callback = vitest.fn();
+            const decoder = new entities.EntityDecoder(
+                entities.htmlDecodeTree,
+                callback,
+                errorHandlers,
+            );
+
+            decoder.startEntity(entities.DecodingMode.Legacy);
+            expect(decoder.write("&amp;", 1)).toBe(5);
+            expect(callback).toHaveBeenCalledTimes(1);
+            expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);
+            expect(
+                errorHandlers.missingSemicolonAfterCharacterReference,
+            ).toHaveBeenCalledTimes(0);
+
+            decoder.startEntity(entities.DecodingMode.Legacy);
+            expect(decoder.write("&amp", 1)).toBe(-1);
+            expect(decoder.end()).toBe(4);
+
+            expect(callback).toHaveBeenCalledTimes(2);
+            expect(callback).toHaveBeenLastCalledWith("&".charCodeAt(0), 4);
+            expect(
+                errorHandlers.missingSemicolonAfterCharacterReference,
+            ).toHaveBeenCalledTimes(1);
+        });
+
+        it("should produce an error for a numeric entity without a semicolon", () => {
+            const errorHandlers = {
+                missingSemicolonAfterCharacterReference: vitest.fn(),
+                absenceOfDigitsInNumericCharacterReference: vitest.fn(),
+                validateNumericCharacterReference: vitest.fn(),
+            };
+            const callback = vitest.fn();
+            const decoder = new entities.EntityDecoder(
+                entities.htmlDecodeTree,
+                callback,
+                errorHandlers,
+            );
+
+            decoder.startEntity(entities.DecodingMode.Legacy);
+            expect(decoder.write("&#x3a", 1)).toBe(-1);
+            expect(decoder.end()).toBe(5);
+
+            expect(callback).toHaveBeenCalledTimes(1);
+            expect(callback).toHaveBeenCalledWith(0x3a, 5);
+            expect(
+                errorHandlers.missingSemicolonAfterCharacterReference,
+            ).toHaveBeenCalledTimes(1);
+            expect(
+                errorHandlers.absenceOfDigitsInNumericCharacterReference,
+            ).toHaveBeenCalledTimes(0);
+            expect(
+                errorHandlers.validateNumericCharacterReference,
+            ).toHaveBeenCalledTimes(1);
+            expect(
+                errorHandlers.validateNumericCharacterReference,
+            ).toHaveBeenCalledWith(0x3a);
+        });
+
+        it("should produce an error for numeric entities without digits", () => {
+            const errorHandlers = {
+                missingSemicolonAfterCharacterReference: vitest.fn(),
+                absenceOfDigitsInNumericCharacterReference: vitest.fn(),
+                validateNumericCharacterReference: vitest.fn(),
+            };
+            const callback = vitest.fn();
+            const decoder = new entities.EntityDecoder(
+                entities.htmlDecodeTree,
+                callback,
+                errorHandlers,
+            );
+
+            decoder.startEntity(entities.DecodingMode.Legacy);
+            expect(decoder.write("&#", 1)).toBe(-1);
+            expect(decoder.end()).toBe(0);
+
+            expect(callback).toHaveBeenCalledTimes(0);
+            expect(
+                errorHandlers.missingSemicolonAfterCharacterReference,
+            ).toHaveBeenCalledTimes(0);
+            expect(
+                errorHandlers.absenceOfDigitsInNumericCharacterReference,
+            ).toHaveBeenCalledTimes(1);
+            expect(
+                errorHandlers.absenceOfDigitsInNumericCharacterReference,
+            ).toHaveBeenCalledWith(2);
+            expect(
+                errorHandlers.validateNumericCharacterReference,
+            ).toHaveBeenCalledTimes(0);
+        });
+
+        it("should produce an error for hex entities without digits", () => {
+            const errorHandlers = {
+                missingSemicolonAfterCharacterReference: vitest.fn(),
+                absenceOfDigitsInNumericCharacterReference: vitest.fn(),
+                validateNumericCharacterReference: vitest.fn(),
+            };
+            const callback = vitest.fn();
+            const decoder = new entities.EntityDecoder(
+                entities.htmlDecodeTree,
+                callback,
+                errorHandlers,
+            );
+
+            decoder.startEntity(entities.DecodingMode.Legacy);
+            expect(decoder.write("&#x", 1)).toBe(-1);
+            expect(decoder.end()).toBe(0);
+
+            expect(callback).toHaveBeenCalledTimes(0);
+            expect(
+                errorHandlers.missingSemicolonAfterCharacterReference,
+            ).toHaveBeenCalledTimes(0);
+            expect(
+                errorHandlers.absenceOfDigitsInNumericCharacterReference,
+            ).toHaveBeenCalledTimes(1);
+            expect(
+                errorHandlers.validateNumericCharacterReference,
+            ).toHaveBeenCalledTimes(0);
+        });
+    });
+});