@chr33s/pdf-unicode-properties 5.0.0 → 5.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chr33s/pdf-unicode-properties",
-  "version": "5.0.0",
+  "version": "5.0.1",
   "description": "Provides fast access to unicode character properties",
   "type": "module",
   "sideEffects": false,

package/scripts/generate.ts

@@ -1,149 +0,0 @@
-import codePoints from "@chr33s/pdf-codepoints";
-import { base64, deflate } from "@chr33s/pdf-common";
-import { builder as UnicodeTrieBuilder } from "@chr33s/pdf-unicode-trie";
-import { writeFile } from "node:fs/promises";
-import path from "node:path";
-
-type IndexLookup = Record<string, number>;
-
-const bits = (value: number): number => (value > 0 ? (Math.log2(value) + 1) | 0 : 0);
-
-const numericValue = (numeric?: string | null): number => {
-  if (!numeric) {
-    return 0;
-  }
-
-  const fractionMatch = numeric.match(/^(-?\d+)\/(\d+)$/);
-  if (fractionMatch) {
-    const numerator = parseInt(fractionMatch[1], 10);
-    const denominator = parseInt(fractionMatch[2], 10);
-    return ((numerator + 12) << 4) + (denominator - 1);
-  }
-
-  if (/^\d0+$/.test(numeric)) {
-    const mantissa = parseInt(numeric[0]!, 10);
-    const exponent = numeric.length - 1;
-    return ((mantissa + 14) << 5) + (exponent - 2);
-  }
-
-  const value = parseInt(numeric, 10);
-  if (value <= 50) {
-    return 1 + value;
-  }
-
-  let mantissa = value;
-  let exponent = 0;
-  while (mantissa % 60 === 0) {
-    mantissa /= 60;
-    exponent += 1;
-  }
-
-  return ((mantissa + 0xbf) << 2) + (exponent - 1);
-};
-
-const addIndex = (
-  lookup: IndexLookup,
-  key: string | null | undefined,
-  currentCount: number,
-): number => {
-  const normalizedKey = key ?? "";
-  if (lookup[normalizedKey] == null) {
-    lookup[normalizedKey] = currentCount;
-    return currentCount + 1;
-  }
-  return currentCount;
-};
-
-const getIndex = (lookup: IndexLookup, key: string | null | undefined): number =>
-  lookup[key ?? ""] ?? 0;
-
-const srcDir = path.resolve(process.cwd(), "src");
-const trieFilePath = path.join(srcDir, "trie.js");
-const dataFilePath = path.join(srcDir, "data.js");
-
-const categories: IndexLookup = Object.create(null);
-const combiningClasses: IndexLookup = Object.create(null);
-const scripts: IndexLookup = Object.create(null);
-const eaws: IndexLookup = Object.create(null);
-
-let categoryCount = 0;
-let combiningClassCount = 0;
-let scriptCount = 0;
-let eawCount = 0;
-
-const entries = Array.from(codePoints);
-
-for (const entry of entries) {
-  if (!entry) {
-    continue;
-  }
-
-  categoryCount = addIndex(categories, entry.category, categoryCount);
-  combiningClassCount = addIndex(combiningClasses, entry.combiningClassName, combiningClassCount);
-  scriptCount = addIndex(scripts, entry.script, scriptCount);
-  eawCount = addIndex(eaws, entry.eastAsianWidth, eawCount);
-}
-
-const numberBits = 10;
-const combiningClassBits = bits(combiningClassCount - 1);
-const scriptBits = bits(scriptCount - 1);
-const eawBits = bits(eawCount - 1);
-
-const categoryShift = combiningClassBits + scriptBits + eawBits + numberBits;
-const combiningShift = scriptBits + eawBits + numberBits;
-const scriptShift = eawBits + numberBits;
-const eawShift = numberBits;
-
-const trie = new UnicodeTrieBuilder();
-for (const entry of entries) {
-  if (!entry) {
-    continue;
-  }
-
-  const category = getIndex(categories, entry.category);
-  const combiningClass = getIndex(combiningClasses, entry.combiningClassName);
-  const script = getIndex(scripts, entry.script);
-  const eaw = getIndex(eaws, entry.eastAsianWidth);
-  const numeric = numericValue(entry.numeric);
-
-  const val =
-    (category << categoryShift) |
-    (combiningClass << combiningShift) |
-    (script << scriptShift) |
-    (eaw << eawShift) |
-    numeric;
-
-  trie.set(entry.code, val);
-}
-
-const trieBuffer = await trie.toBuffer();
-const compressedTrie = await deflate(trieBuffer);
-const triePayload = base64.encode(
-  compressedTrie.buffer.slice(
-    compressedTrie.byteOffset,
-    compressedTrie.byteOffset + compressedTrie.byteLength,
-  ) as ArrayBuffer,
-);
-
-const emitModule = (value: string): string =>
-  `const payload = ${JSON.stringify(value)};\nexport default payload;\n`;
-
-await writeFile(trieFilePath, emitModule(triePayload));
-
-const encoder = new TextEncoder();
-const data = {
-  categories: Object.keys(categories),
-  combiningClasses: Object.keys(combiningClasses),
-  scripts: Object.keys(scripts),
-  eaw: Object.keys(eaws),
-};
-
-const dataBytes = encoder.encode(JSON.stringify(data));
-const compressedData = await deflate(dataBytes);
-const dataPayload = base64.encode(
-  compressedData.buffer.slice(
-    compressedData.byteOffset,
-    compressedData.byteOffset + compressedData.byteLength,
-  ) as ArrayBuffer,
-);
-await writeFile(dataFilePath, emitModule(dataPayload));

package/test/index.test.ts

@@ -1,91 +0,0 @@
-import { beforeAll, describe, expect, test } from "vitest";
-import createUnicodeProperties, { type UnicodePropertiesAPI } from "../src/index.js";
-
-const code = (char: string) => char.charCodeAt(0);
-
-describe("unicode-properties", () => {
-  let unicode: UnicodePropertiesAPI;
-
-  beforeAll(async () => {
-    unicode = await createUnicodeProperties();
-  });
-
-  test("getCategory", () => {
-    expect(unicode.getCategory(code("2"))).toBe("Nd");
-    expect(unicode.getCategory(code("x"))).toBe("Ll");
-  });
-
-  test("getCombiningClass", () => {
-    expect(unicode.getCombiningClass(code("x"))).toBe("Not_Reordered");
-    expect(unicode.getCombiningClass(code("́"))).toBe("Above");
-    expect(unicode.getCombiningClass(code("ٕ"))).toBe("Below");
-    expect(unicode.getCombiningClass(code("ٔ"))).toBe("Above");
-  });
-
-  test("getScript", () => {
-    expect(unicode.getScript(code("x"))).toBe("Latin");
-    expect(unicode.getScript(code("غ"))).toBe("Arabic");
-  });
-
-  test("getEastAsianWidth", () => {
-    expect(unicode.getEastAsianWidth(code("x"))).toBe("Na");
-    expect(unicode.getEastAsianWidth(code("杜"))).toBe("W");
-    expect(unicode.getEastAsianWidth(code("Æ"))).toBe("A");
-  });
-
-  test("getNumericValue", () => {
-    expect(unicode.getNumericValue(code("2"))).toBe(2);
-    expect(unicode.getNumericValue(code("x"))).toBeNull();
-  });
-
-  test("isAlphabetic", () => {
-    expect(unicode.isAlphabetic(code("x"))).toBe(true);
-    expect(unicode.isAlphabetic(code("2"))).toBe(false);
-  });
-
-  test("isDigit", () => {
-    expect(unicode.isDigit(code("x"))).toBe(false);
-    expect(unicode.isDigit(code("2"))).toBe(true);
-  });
-
-  test("isPunctuation", () => {
-    expect(unicode.isPunctuation(code("x"))).toBe(false);
-    expect(unicode.isPunctuation(code("."))).toBe(true);
-  });
-
-  test("isLowerCase", () => {
-    expect(unicode.isLowerCase(code("X"))).toBe(false);
-    expect(unicode.isLowerCase(code("2"))).toBe(false);
-    expect(unicode.isLowerCase(code("x"))).toBe(true);
-  });
-
-  test("isUpperCase", () => {
-    expect(unicode.isUpperCase(code("X"))).toBe(true);
-    expect(unicode.isUpperCase(code("2"))).toBe(false);
-    expect(unicode.isUpperCase(code("x"))).toBe(false);
-  });
-
-  test("isTitleCase", () => {
-    expect(unicode.isTitleCase(code("Dz"))).toBe(true);
-    expect(unicode.isTitleCase(code("2"))).toBe(false);
-    expect(unicode.isTitleCase(code("x"))).toBe(false);
-  });
-
-  test("isWhiteSpace", () => {
-    expect(unicode.isWhiteSpace(code(" "))).toBe(true);
-    expect(unicode.isWhiteSpace(code("2"))).toBe(false);
-    expect(unicode.isWhiteSpace(code("x"))).toBe(false);
-  });
-
-  test("isBaseForm", () => {
-    expect(unicode.isBaseForm(code("2"))).toBe(true);
-    expect(unicode.isBaseForm(code("x"))).toBe(true);
-    expect(unicode.isBaseForm(code("́"))).toBe(false);
-  });
-
-  test("isMark", () => {
-    expect(unicode.isMark(code("2"))).toBe(false);
-    expect(unicode.isMark(code("x"))).toBe(false);
-    expect(unicode.isMark(code("́"))).toBe(true);
-  });
-});

package/tsconfig.json

@@ -1,10 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "compilerOptions": {
-    "allowJs": true,
-    "rootDir": "src",
-    "outDir": "dist"
-  },
-  "include": ["src/**/*.ts", "src/**/*.js"],
-  "exclude": ["dist", "node_modules", "test"]
-}

package/tsconfig.typecheck.json

@@ -1,16 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "compilerOptions": {
-    "noEmit": true,
-    "rootDir": "."
-  },
-  "include": [
-    "**/*.ts",
-  ],
-  "exclude": [
-    "dist",
-    "node_modules",
-    "src/data.js",
-    "src/trie.js"
-  ]
-}

package/vitest.config.ts

@@ -1,12 +0,0 @@
-import { defineConfig } from "vitest/config";
-
-export default defineConfig({
-  test: {
-    environment: "node",
-    include: ["test/**/*.test.ts"],
-    coverage: {
-      provider: "v8",
-      reporter: ["text", "lcov"],
-    },
-  },
-});