@chr33s/pdf-unicode-trie 5.0.0

package/src/index.ts

@@ -0,0 +1,4 @@
+export { default as builder } from "./builder.js";
+export { swap32LE } from "./swap.js";
+export * from "./unicode-trie.js";
+export { default } from "./unicode-trie.js";

package/src/swap.ts

@@ -0,0 +1,21 @@
+const isBigEndian = new Uint8Array(new Uint32Array([0x12345678]).buffer)[0] === 0x12;
+
+const swap = (array: Uint8Array, left: number, right: number): void => {
+  const temp = array[left];
+  array[left] = array[right];
+  array[right] = temp;
+};
+
+const swap32 = (array: Uint8Array): void => {
+  const { length } = array;
+  for (let index = 0; index < length; index += 4) {
+    swap(array, index, index + 3);
+    swap(array, index + 1, index + 2);
+  }
+};
+
+export const swap32LE = (array: Uint8Array): void => {
+  if (isBigEndian) {
+    swap32(array);
+  }
+};

package/src/unicode-trie.ts

@@ -0,0 +1,148 @@
+import { inflate } from "@chr33s/pdf-common";
+import { swap32LE } from "./swap.js";
+
+export type UnicodeTrieJSON = {
+  data: Uint32Array | Int32Array;
+  highStart: number;
+  errorValue: number;
+};
+
+export type UnicodeTrieInit = ArrayBufferLike | ArrayBufferView | UnicodeTrieJSON;
+
+const SHIFT_1 = 6 + 5;
+const SHIFT_2 = 5;
+const SHIFT_1_2 = SHIFT_1 - SHIFT_2;
+const OMITTED_BMP_INDEX_1_LENGTH = 0x10000 >> SHIFT_1;
+const INDEX_2_BLOCK_LENGTH = 1 << SHIFT_1_2;
+const INDEX_2_MASK = INDEX_2_BLOCK_LENGTH - 1;
+const INDEX_SHIFT = 2;
+const DATA_BLOCK_LENGTH = 1 << SHIFT_2;
+const DATA_MASK = DATA_BLOCK_LENGTH - 1;
+const LSCP_INDEX_2_OFFSET = 0x10000 >> SHIFT_2;
+const LSCP_INDEX_2_LENGTH = 0x400 >> SHIFT_2;
+const INDEX_2_BMP_LENGTH = LSCP_INDEX_2_OFFSET + LSCP_INDEX_2_LENGTH;
+const UTF8_2B_INDEX_2_OFFSET = INDEX_2_BMP_LENGTH;
+const UTF8_2B_INDEX_2_LENGTH = 0x800 >> 6;
+const INDEX_1_OFFSET = UTF8_2B_INDEX_2_OFFSET + UTF8_2B_INDEX_2_LENGTH;
+const DATA_GRANULARITY = 1 << INDEX_SHIFT;
+
+const isUnicodeTrieJSON = (value: UnicodeTrieInit): value is UnicodeTrieJSON => {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+
+  if (!("data" in value) || !("highStart" in value) || !("errorValue" in value)) {
+    return false;
+  }
+
+  const candidate = value as Partial<UnicodeTrieJSON>;
+  return (
+    (candidate.data instanceof Uint32Array || candidate.data instanceof Int32Array) &&
+    typeof candidate.highStart === "number" &&
+    typeof candidate.errorValue === "number"
+  );
+};
+
+const toUint8Array = (value: ArrayBufferLike | ArrayBufferView): Uint8Array => {
+  if (value instanceof Uint8Array) {
+    return value;
+  }
+
+  if (ArrayBuffer.isView(value)) {
+    return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+  }
+
+  if (value instanceof ArrayBuffer) {
+    return new Uint8Array(value);
+  }
+
+  if (typeof SharedArrayBuffer !== "undefined" && value instanceof SharedArrayBuffer) {
+    return new Uint8Array(value);
+  }
+
+  return new Uint8Array(value);
+};
+
+class UnicodeTrie {
+  readonly data: Uint32Array;
+  readonly highStart: number;
+  readonly errorValue: number;
+
+  private constructor(data: Uint32Array, highStart: number, errorValue: number) {
+    this.data = data;
+    this.highStart = highStart;
+    this.errorValue = errorValue;
+  }
+
+  /**
+   * Create a UnicodeTrie from a JSON object (synchronous)
+   */
+  static fromJSON(source: UnicodeTrieJSON): UnicodeTrie {
+    const typedData =
+      source.data instanceof Uint32Array ? source.data : new Uint32Array(source.data);
+    return new UnicodeTrie(typedData, source.highStart, source.errorValue);
+  }
+
+  /**
+   * Create a UnicodeTrie from compressed binary data (async)
+   */
+  static async fromBuffer(source: ArrayBufferLike | ArrayBufferView): Promise<UnicodeTrie> {
+    const bytes = toUint8Array(source);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const highStart = view.getUint32(0, true);
+    const errorValue = view.getUint32(4, true);
+    // uncompressedLength at offset 8 is not needed for async inflate
+
+    let payload = bytes.subarray(12);
+    payload = await inflate(payload, "deflate-raw");
+    payload = await inflate(payload, "deflate-raw");
+    swap32LE(payload);
+
+    const data = new Uint32Array(
+      payload.buffer,
+      payload.byteOffset,
+      payload.byteLength / Uint32Array.BYTES_PER_ELEMENT,
+    );
+
+    return new UnicodeTrie(data, highStart, errorValue);
+  }
+
+  /**
+   * Create a UnicodeTrie from any supported source (async)
+   */
+  static async create(source: UnicodeTrieInit): Promise<UnicodeTrie> {
+    if (isUnicodeTrieJSON(source)) {
+      return UnicodeTrie.fromJSON(source);
+    }
+    return UnicodeTrie.fromBuffer(source);
+  }
+
+  get(codePoint: number): number {
+    if (codePoint < 0 || codePoint > 0x10ffff) {
+      return this.errorValue;
+    }
+
+    if (codePoint < 0xd800 || (codePoint > 0xdbff && codePoint <= 0xffff)) {
+      const index = (this.data[codePoint >> SHIFT_2] << INDEX_SHIFT) + (codePoint & DATA_MASK);
+      return this.data[index];
+    }
+
+    if (codePoint <= 0xffff) {
+      const index =
+        (this.data[LSCP_INDEX_2_OFFSET + ((codePoint - 0xd800) >> SHIFT_2)] << INDEX_SHIFT) +
+        (codePoint & DATA_MASK);
+      return this.data[index];
+    }
+
+    if (codePoint < this.highStart) {
+      let index = this.data[INDEX_1_OFFSET - OMITTED_BMP_INDEX_1_LENGTH + (codePoint >> SHIFT_1)];
+      index = this.data[index + ((codePoint >> SHIFT_2) & INDEX_2_MASK)];
+      index = (index << INDEX_SHIFT) + (codePoint & DATA_MASK);
+      return this.data[index];
+    }
+
+    return this.data[this.data.length - DATA_GRANULARITY];
+  }
+}
+
+export default UnicodeTrie;

package/test/builder.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, test } from "vitest";
+import UnicodeTrieBuilder from "../src/builder.js";
+
+describe("UnicodeTrieBuilder", () => {
+  describe("constructor", () => {
+    test("should accept null initial value", () => {
+      const builder = new UnicodeTrieBuilder(null, 0);
+      expect(builder.get(0x0100)).toBe(0);
+    });
+
+    test("should accept null error value", () => {
+      const builder = new UnicodeTrieBuilder(0, null);
+      expect(builder.get(-1)).toBe(0);
+    });
+
+    test("should use default values when not provided", () => {
+      const builder = new UnicodeTrieBuilder();
+      expect(builder.get(0x0100)).toBe(0);
+      expect(builder.get(-1)).toBe(0);
+    });
+  });
+
+  describe("set", () => {
+    test("should set values for individual code points", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x0041, 1);
+      builder.set(0x0042, 2);
+      builder.set(0x0043, 3);
+
+      expect(builder.get(0x0041)).toBe(1);
+      expect(builder.get(0x0042)).toBe(2);
+      expect(builder.get(0x0043)).toBe(3);
+    });
+
+    test("should overwrite previous values", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x0041, 100);
+      expect(builder.get(0x0041)).toBe(100);
+
+      builder.set(0x0041, 200);
+      expect(builder.get(0x0041)).toBe(200);
+    });
+
+    test("should handle supplementary plane code points", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x1f600, 128512); // Emoji code point
+
+      expect(builder.get(0x1f600)).toBe(128512);
+    });
+  });
+
+  describe("setRange", () => {
+    test("should set range without overwrite", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x50, 99);
+      builder.setRange(0x40, 0x60, 50, false);
+
+      // Existing value should not be overwritten
+      expect(builder.get(0x50)).toBe(99);
+      // Other values in range should be set
+      expect(builder.get(0x40)).toBe(50);
+      expect(builder.get(0x60)).toBe(50);
+    });
+
+    test("should set range with overwrite", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x50, 99);
+      builder.setRange(0x40, 0x60, 50, true);
+
+      // Existing value should be overwritten
+      expect(builder.get(0x50)).toBe(50);
+      expect(builder.get(0x40)).toBe(50);
+      expect(builder.get(0x60)).toBe(50);
+    });
+
+    test("should handle single code point range", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.setRange(0x100, 0x100, 42, true);
+
+      expect(builder.get(0x100)).toBe(42);
+      expect(builder.get(0x0ff)).toBe(0);
+      expect(builder.get(0x101)).toBe(0);
+    });
+
+    test("should handle large ranges", () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.setRange(0x0, 0xffff, 1, true);
+
+      expect(builder.get(0x0)).toBe(1);
+      expect(builder.get(0x8000)).toBe(1);
+      expect(builder.get(0xffff)).toBe(1);
+      expect(builder.get(0x10000)).toBe(0); // Beyond range
+    });
+  });
+
+  describe("freeze", () => {
+    test("should return compacted trie", () => {
+      const builder = new UnicodeTrieBuilder(10, 20);
+      builder.set(0x1000, 100);
+
+      const trie = builder.freeze();
+      expect(trie.get(0x1000)).toBe(100);
+      expect(trie.get(0x1001)).toBe(10);
+      expect(trie.get(-1)).toBe(20);
+    });
+
+    test("should return same values as builder", () => {
+      const builder = new UnicodeTrieBuilder(5, 999);
+      builder.setRange(0x100, 0x1ff, 50, true);
+      builder.set(0x150, 75);
+
+      const trie = builder.freeze();
+
+      for (let cp = 0x100; cp <= 0x1ff; cp++) {
+        expect(trie.get(cp)).toBe(builder.get(cp));
+      }
+    });
+  });
+
+  describe("toBuffer", () => {
+    test("should produce buffer that can be loaded", async () => {
+      const builder = new UnicodeTrieBuilder(1, 2);
+      builder.set(0x5000, 500);
+
+      const buffer = await builder.toBuffer();
+      expect(buffer).toBeInstanceOf(Buffer);
+      expect(buffer.length).toBeGreaterThan(12); // Header is 12 bytes
+
+      // Verify header values
+      const highStart = buffer.readUInt32LE(0);
+      const errorValue = buffer.readUInt32LE(4);
+      expect(errorValue).toBe(2);
+      expect(highStart).toBeGreaterThan(0);
+    });
+
+    test("should produce valid compressed output", async () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.setRange(0x0, 0xff, 1, true);
+
+      const buffer = await builder.toBuffer();
+
+      // Import UnicodeTrie to verify
+      const { default: UnicodeTrie } = await import("../src/index.js");
+      const trie = await UnicodeTrie.create(buffer);
+
+      expect(trie.get(0x00)).toBe(1);
+      expect(trie.get(0xff)).toBe(1);
+      expect(trie.get(0x100)).toBe(0);
+    });
+  });
+});

package/test/swap.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, test } from "vitest";
+import { swap32LE } from "../src/swap.js";
+
+describe("swap32LE", () => {
+  test("should not modify array on little-endian systems", () => {
+    // Most modern systems are little-endian, so this is the common case
+    const input = new Uint8Array([0x12, 0x34, 0x56, 0x78]);
+
+    // On little-endian systems, this should be a no-op
+    // On big-endian systems, it will swap bytes
+    swap32LE(input);
+
+    // We check that the function runs without error
+    // The actual byte order depends on the system endianness
+    expect(input.length).toBe(4);
+  });
+
+  test("should handle empty array", () => {
+    const input = new Uint8Array([]);
+    expect(() => swap32LE(input)).not.toThrow();
+  });
+
+  test("should handle multiple 32-bit values", () => {
+    const input = new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]);
+    expect(() => swap32LE(input)).not.toThrow();
+    expect(input.length).toBe(8);
+  });
+
+  test("should work with array that is multiple of 4 bytes", () => {
+    const input = new Uint8Array(12);
+    for (let i = 0; i < 12; i++) {
+      input[i] = i;
+    }
+    expect(() => swap32LE(input)).not.toThrow();
+  });
+});

package/test/trie.test.ts

@@ -0,0 +1,125 @@
+import { describe, expect, test } from "vitest";
+import UnicodeTrieBuilder from "../src/builder.js";
+import UnicodeTrie from "../src/unicode-trie.js";
+
+describe("UnicodeTrie", () => {
+  describe("get", () => {
+    test("should return errorValue for negative code points", async () => {
+      const builder = new UnicodeTrieBuilder(0, 999);
+      const trie = builder.freeze();
+      expect(trie.get(-1)).toBe(999);
+      expect(trie.get(-100)).toBe(999);
+    });
+
+    test("should return errorValue for code points above 0x10FFFF", async () => {
+      const builder = new UnicodeTrieBuilder(0, 888);
+      const trie = builder.freeze();
+      expect(trie.get(0x110000)).toBe(888);
+      expect(trie.get(0x200000)).toBe(888);
+    });
+
+    test("should handle BMP code points (0x0000-0xFFFF)", async () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x0041, 65); // 'A'
+      builder.set(0x0061, 97); // 'a'
+      builder.set(0x0000, 1); // NUL
+      builder.set(0xffff, 2); // Last BMP
+
+      const trie = builder.freeze();
+      expect(trie.get(0x0041)).toBe(65);
+      expect(trie.get(0x0061)).toBe(97);
+      expect(trie.get(0x0000)).toBe(1);
+      expect(trie.get(0xffff)).toBe(2);
+    });
+
+    test("should handle lead surrogate range (0xD800-0xDBFF)", async () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0xd800, 100);
+      builder.set(0xdbff, 101);
+
+      const trie = builder.freeze();
+      expect(trie.get(0xd800)).toBe(100);
+      expect(trie.get(0xdbff)).toBe(101);
+    });
+
+    test("should handle supplementary code points (0x10000-0x10FFFF)", async () => {
+      const builder = new UnicodeTrieBuilder(0, 0);
+      builder.set(0x10000, 200); // First supplementary
+      builder.set(0x1f600, 201); // Emoji
+      builder.set(0x10ffff, 202); // Last valid
+
+      const trie = builder.freeze();
+      expect(trie.get(0x10000)).toBe(200);
+      expect(trie.get(0x1f600)).toBe(201);
+      expect(trie.get(0x10ffff)).toBe(202);
+    });
+
+    test("should return initial value for unset code points", async () => {
+      const builder = new UnicodeTrieBuilder(42, 0);
+      const trie = builder.freeze();
+
+      expect(trie.get(0x0100)).toBe(42);
+      expect(trie.get(0x5000)).toBe(42);
+    });
+  });
+
+  describe("fromJSON", () => {
+    test("should create trie from JSON object", async () => {
+      const builder = new UnicodeTrieBuilder(10, 666);
+      builder.set(0x0041, 100);
+      const frozen = builder.freeze();
+
+      const json = {
+        data: frozen.data,
+        highStart: frozen.highStart,
+        errorValue: frozen.errorValue,
+      };
+
+      const trie = UnicodeTrie.fromJSON(json);
+      expect(trie.get(0x0041)).toBe(100);
+      expect(trie.get(0x0042)).toBe(10);
+      expect(trie.get(-1)).toBe(666);
+    });
+
+    test("should accept Int32Array as data", () => {
+      const data = new Int32Array([1, 2, 3, 4]);
+      const json = {
+        data,
+        highStart: 0x110000,
+        errorValue: 0,
+      };
+
+      const trie = UnicodeTrie.fromJSON(json);
+      expect(trie.data).toBeInstanceOf(Uint32Array);
+    });
+  });
+
+  describe("create", () => {
+    test("should detect JSON format and use fromJSON", async () => {
+      const builder = new UnicodeTrieBuilder(5, 500);
+      builder.set(0x0100, 99);
+      const frozen = builder.freeze();
+
+      const json = {
+        data: frozen.data,
+        highStart: frozen.highStart,
+        errorValue: frozen.errorValue,
+      };
+
+      const trie = await UnicodeTrie.create(json);
+      expect(trie.get(0x0100)).toBe(99);
+      expect(trie.get(-1)).toBe(500);
+    });
+
+    test("should detect buffer format and use fromBuffer", async () => {
+      const builder = new UnicodeTrieBuilder(7, 700);
+      builder.set(0x0200, 77);
+
+      const buffer = await builder.toBuffer();
+      const trie = await UnicodeTrie.create(buffer);
+
+      expect(trie.get(0x0200)).toBe(77);
+      expect(trie.get(-1)).toBe(700);
+    });
+  });
+});