grapheme-conformance 0.1.0

package/dist/index.js

@@ -0,0 +1,151 @@
+// src/parse.ts
+var BREAK = "\xF7";
+var NO_BREAK = "\xD7";
+function parseBreakTest(source) {
+  const vectors2 = [];
+  const lines = source.split(/\r\n|\r|\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const hash = lines[i].indexOf("#");
+    const body = (hash === -1 ? lines[i] : lines[i].slice(0, hash)).trim();
+    if (body === "") continue;
+    const tokens = body.split(/\s+/);
+    if (tokens[0] !== BREAK) continue;
+    const expected = [];
+    let cluster = "";
+    let malformed = false;
+    for (const token of tokens) {
+      if (token === BREAK) {
+        if (cluster !== "") expected.push(cluster);
+        cluster = "";
+      } else if (token === NO_BREAK) {
+        continue;
+      } else if (/^[0-9A-Fa-f]+$/.test(token)) {
+        cluster += String.fromCodePoint(parseInt(token, 16));
+      } else {
+        malformed = true;
+        break;
+      }
+    }
+    if (malformed) continue;
+    if (cluster !== "") expected.push(cluster);
+    if (expected.length === 0) continue;
+    vectors2.push({ input: expected.join(""), expected, line: i + 1 });
+  }
+  return vectors2;
+}
+
+// src/rules.ts
+var VIRAMA = /[्্੍્୍்్್്]/;
+var INDIC_SCRIPT = /[\p{Script=Devanagari}\p{Script=Bengali}\p{Script=Gurmukhi}\p{Script=Gujarati}\p{Script=Oriya}\p{Script=Tamil}\p{Script=Telugu}\p{Script=Kannada}\p{Script=Malayalam}]/u;
+var LETTER = /\p{L}/u;
+var MARK = /\p{M}/u;
+var EXT_PICT = /\p{Extended_Pictographic}/u;
+var ZWJ = "\u200D";
+function isRegionalIndicator(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 127462 && c <= 127487;
+}
+function isHangulJamo(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 4352 && c <= 4607 || c >= 43360 && c <= 43391 || c >= 55216 && c <= 55295;
+}
+function hasIndicConjunct(cps) {
+  for (let i = 0; i < cps.length - 1; i++) {
+    if (!VIRAMA.test(cps[i])) continue;
+    let j = i + 1;
+    while (j < cps.length && (cps[j] === ZWJ || MARK.test(cps[j]))) j++;
+    if (j < cps.length && LETTER.test(cps[j]) && INDIC_SCRIPT.test(cps[j])) {
+      return true;
+    }
+  }
+  return false;
+}
+function hasPictographicZwj(cps) {
+  for (let i = 1; i < cps.length - 1; i++) {
+    if (cps[i] === ZWJ && EXT_PICT.test(cps[i - 1]) && EXT_PICT.test(cps[i + 1])) {
+      return true;
+    }
+  }
+  return false;
+}
+function inferRule(input) {
+  const cps = [...input];
+  if (hasIndicConjunct(cps)) return "GB9c";
+  if (hasPictographicZwj(cps)) return "GB11";
+  if (cps.filter(isRegionalIndicator).length >= 2) return "GB12/GB13";
+  if (cps.some(isHangulJamo)) return "GB6/GB7/GB8";
+  return null;
+}
+
+// src/score.ts
+function toHex(input) {
+  return [...input].map((cp) => cp.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")).join(" ");
+}
+function sameClusters(a, b) {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+function score(segmenter, vectors2) {
+  const failures = [];
+  let passed = 0;
+  for (const vector of vectors2) {
+    let actual;
+    try {
+      const result = segmenter(vector.input);
+      actual = Array.isArray(result) && result.every((c) => typeof c === "string") ? [...result] : [];
+    } catch {
+      actual = [];
+    }
+    if (sameClusters(actual, vector.expected)) {
+      passed++;
+    } else {
+      failures.push({
+        line: vector.line,
+        input: vector.input,
+        inputHex: toHex(vector.input),
+        expected: [...vector.expected],
+        actual,
+        rule: inferRule(vector.input)
+      });
+    }
+  }
+  const total = vectors2.length;
+  return { passed, total, rate: total === 0 ? 1 : passed / total, failures };
+}
+
+// src/vectors.ts
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+var VERSIONS = ["15.0.0", "15.1.0", "16.0.0", "17.0.0"];
+function vectorsDir() {
+  let dir = dirname(fileURLToPath(import.meta.url));
+  for (let i = 0; i < 4; i++) {
+    const candidate = join(dir, "vectors");
+    if (existsSync(join(candidate, "GraphemeBreakTest-15.1.0.txt"))) return candidate;
+    const parent = resolve(dir, "..");
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(
+    "grapheme-conformance: vendored vectors/ directory not found next to the package"
+  );
+}
+function load() {
+  const dir = vectorsDir();
+  const out = /* @__PURE__ */ Object.create(null);
+  for (const version of VERSIONS) {
+    const source = readFileSync(join(dir, `GraphemeBreakTest-${version}.txt`), "utf8");
+    out[version] = parseBreakTest(source);
+  }
+  return out;
+}
+var vectors = load();
+export {
+  parseBreakTest,
+  score,
+  vectors
+};

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "grapheme-conformance",
+  "version": "0.1.0",
+  "description": "Score any JavaScript grapheme segmenter against Unicode's official GraphemeBreakTest.txt. Find out which string splitters are wrong, and about what.",
+  "keywords": [
+    "unicode",
+    "grapheme",
+    "uax29",
+    "text-segmentation",
+    "conformance",
+    "emoji",
+    "i18n",
+    "intl-segmenter",
+    "testing",
+    "javascript"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tamerkalla/grapheme-conformance.git"
+  },
+  "homepage": "https://github.com/tamerkalla/grapheme-conformance#readme",
+  "bugs": {
+    "url": "https://github.com/tamerkalla/grapheme-conformance/issues"
+  },
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "bin": {
+    "grapheme-conformance": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "vectors",
+    "README.md",
+    "SCOREBOARD.md",
+    "VERIFY.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "prescoreboard": "npm run build",
+    "scoreboard": "node scripts/scoreboard.mjs",
+    "scoreboard:check": "npm run build && node scripts/check-scoreboard.mjs",
+    "smoke": "npm run build && node scripts/smoke.mjs",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@types/node": "22.10.2",
+    "grapheme-splitter": "1.0.4",
+    "graphemer": "1.4.0",
+    "runes2": "1.1.4",
+    "tsup": "8.3.5",
+    "typescript": "5.7.2",
+    "unicode-segmenter": "0.17.3",
+    "vitest": "2.1.8"
+  }
+}