grapheme-conformance 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 grapheme-conformance contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,169 @@
+# grapheme-conformance
+
+**Your string splitter is probably wrong about Hindi.**
+
+Unicode publishes `GraphemeBreakTest.txt`: a machine-readable answer key that
+says exactly where a string may be split into user-perceived characters. This
+package scores any JavaScript grapheme segmenter against it, tells you which
+cases it got wrong, and gives you an exit code you can put in CI.
+
+It does not ship another segmenter.
+
+## The finding
+
+`क्षि` — Devanagari, four code points, one character a reader sees. Split it
+with a popular library and you get two clusters. `grapheme-splitter` and
+`graphemer` both break it; `runes2` breaks it into three.
+
+| input | splits wrongly in |
+|---|---|
+| `क्षि` Devanagari (`0915 094D 0937 093F`) | `grapheme-splitter` (2), `graphemer` (2), `runes2` (3) |
+| `e` + combining acute (`0065 0301`) | `runes2` (2) |
+| Hangul `한` (`1112 1161 11AB`) | `runes2` (2) |
+| pirate flag (`1F3F4 200D 2620 FE0F`) | `grapheme-splitter` (2) |
+
+Family emoji, skin-tone modifiers, regional-indicator flags, tag-sequence
+flags, ZWJ professions and keycaps pass in every library tested. Emoji is not
+the story; Indic script is. All seven of `graphemer`'s Unicode 15.1 failures
+are rule GB9c, added in that release for Indic conjunct clusters.
+
+## Scoreboard
+
+See [SCOREBOARD.md](./SCOREBOARD.md), regenerated by `npm run scoreboard`.
+Cases passed:
+
+| implementation | U15.1.0 (1187) | U16.0.0 (1093) |
+|---|---|---|
+| `Intl.Segmenter` (ICU, Node 22) | 1186 | 1092 |
+| `unicode-segmenter@0.17.3` | 1186 | 1092 |
+| `graphemer@1.4.0` | 1180 | 1086 |
+| `grapheme-splitter@1.0.4` | 1175 | 1081 |
+| `runes2@1.1.4` | 730 | 695 |
+
+`Intl.Segmenter`'s single failure is the input `2701 200D 2701`, a known ICU
+deviation — and one that arrived with a later ICU rather than an earlier one.
+Node 18.20.8 ships an ICU without it and passes every case (`1187` and `1093`);
+Node 20 and 22 split that input and score `1186` and `1092`.
+
+## Install
+
+```sh
+npm install --save-dev grapheme-conformance
+```
+
+Zero runtime dependencies. Dual ESM/CJS. Node 18+.
+
+## CLI
+
+```sh
+npx grapheme-conformance --module unicode-segmenter/grapheme \
+  --export splitGraphemes --version 16.0.0
+```
+
+```
+unicode-segmenter/grapheme (splitGraphemes)  GraphemeBreakTest 16.0.0
+  passed  1092/1093  99.91%
+  failed  1
+
+  line  input                             want  got   rule
+  1105  2701 200D 2701                    1     2     -
+```
+
+| flag | default | meaning |
+|---|---|---|
+| `--module` | required | module to load: a bare name, or a path relative to cwd |
+| `--export` | `default` | the export to score |
+| `--version` | `16.0.0` | vendored vectors: `15.0.0`, `15.1.0`, `16.0.0`, `17.0.0` |
+| `--min` | `1.0` | minimum pass rate before exiting non-zero |
+| `--limit` | `10` | failing cases to print |
+
+Exits `0` when the pass rate is at least `--min`, `1` when below, `2` on a
+usage or module-loading error. The exit code is the point: drop it into your
+own CI. Exports returning a generator or any other iterable are accepted.
+
+## API
+
+```ts
+import { parseBreakTest, score, vectors } from 'grapheme-conformance';
+
+const segmenter = new Intl.Segmenter('en', { granularity: 'grapheme' });
+const report = score(
+  (s) => [...segmenter.segment(s)].map((part) => part.segment),
+  vectors['16.0.0'],
+);
+
+report.passed; // 1092
+report.total; // 1093
+report.rate; // 0.999085...
+report.failures[0].inputHex; // '2701 200D 2701'
+```
+
+```ts
+type Segmenter = (input: string) => string[];
+
+interface Vector { input: string; expected: string[]; line: number }
+
+function parseBreakTest(source: string): Vector[];
+function score(segmenter: Segmenter, vectors: Vector[]): Report;
+
+interface Report { passed: number; total: number; rate: number; failures: Failure[] }
+
+interface Failure {
+  line: number;        // 1-based line number in the source .txt
+  input: string;
+  inputHex: string;    // 'space-separated UPPERCASE hex code points'
+  expected: string[];
+  actual: string[];
+  rule: string | null; // inferred rule id, or null
+}
+
+const vectors: Record<string, Vector[]>; // keyed by Unicode version
+```
+
+`parseBreakTest` takes a string and never touches the filesystem. `score` never
+throws: a segmenter that throws on an input yields a failure with `actual: []`.
+`failures` is ordered by `line`, and repeated calls return deeply equal reports.
+
+## Vectors
+
+`15.0.0`, `15.1.0`, `16.0.0` and `17.0.0` are vendored under `vectors/` and
+committed, fetched once from
+[unicode-org/unicodetools](https://github.com/unicode-org/unicodetools). Nothing
+is fetched at install, build or test time; the whole suite runs offline.
+
+## Rule inference
+
+Each failure carries a best-effort `rule`: `GB9c` (Indic conjunct), `GB11`
+(pictographic ZWJ sequence), `GB12/GB13` (regional indicators), `GB6/GB7/GB8`
+(Hangul jamo), or `null`.
+
+**This is a convenience for reading the scoreboard, not a correctness claim.**
+It inspects the input for characteristic code points; it does not implement
+UAX #29 and does not prove which rule an implementation actually got wrong.
+
+## A note on the numbers
+
+The scoreboard states counts. It does not rank or grade, and a lower number is
+not automatically a reason to switch libraries — segmenters trade conformance
+against size and speed, and which cases matter depends on the text you handle.
+What the numbers do show is that the disagreements are not exotic: they land on
+ordinary Devanagari.
+
+`Intl.Segmenter` is scored against whatever ICU the host Node ships, so its row
+moves with the runtime, and not monotonically: Node 18.20.8 scores higher than
+Node 22 because the `2701 200D 2701` deviation is a newer ICU behaviour. The
+pure-JS libraries are pinned to exact versions and score identically everywhere.
+
+## Verifying this build
+
+See [VERIFY.md](./VERIFY.md): one command, ten integers to compare.
+
+## Scope
+
+Grapheme clusters only. Word and sentence breaking are out of scope —
+`unicode-segmenter` does not implement them, which leaves too few
+implementations to be worth a scoreboard.
+
+## License
+
+MIT

package/SCOREBOARD.md

@@ -0,0 +1,27 @@
+# Scoreboard
+
+Cases passed against Unicode's official `GraphemeBreakTest.txt`, by Unicode
+version. Case counts are in the column headers.
+
+| implementation | 15.0.0 (602) | 15.1.0 (1187) | 16.0.0 (1093) | 17.0.0 (766) |
+|---|---|---|---|---|
+| `Intl.Segmenter` | 601/602 (99.83%) | 1186/1187 (99.92%) | 1092/1093 (99.91%) | 766/766 (100.00%) |
+| `unicode-segmenter` | 601/602 (99.83%) | 1186/1187 (99.92%) | 1092/1093 (99.91%) | 766/766 (100.00%) |
+| `graphemer` | 602/602 (100.00%) | 1180/1187 (99.41%) | 1086/1093 (99.36%) | 749/766 (97.78%) |
+| `grapheme-splitter` | 597/602 (99.17%) | 1175/1187 (98.99%) | 1081/1093 (98.90%) | 746/766 (97.39%) |
+| `runes2` | 412/602 (68.44%) | 730/1187 (61.50%) | 695/1093 (63.59%) | 501/766 (65.40%) |
+
+---
+
+Generated 2026-08-13 by `npm run scoreboard`. Versions under test:
+
+| implementation | version |
+|---|---|
+| `Intl.Segmenter` | ICU, Unicode 17.0 |
+| `unicode-segmenter` | 0.17.3 |
+| `graphemer` | 1.4.0 |
+| `grapheme-splitter` | 1.0.4 |
+| `runes2` | 1.1.4 |
+
+`Intl.Segmenter` is scored against the host ICU, so its row depends on the
+runtime the scoreboard was generated on. The other rows do not.

package/VERIFY.md

@@ -0,0 +1,58 @@
+# VERIFY
+
+Run one command:
+
+```
+npm run scoreboard
+```
+
+It builds the package, scores every library against every vendored copy of
+Unicode's `GraphemeBreakTest.txt`, writes `SCOREBOARD.md`, and prints the table
+below to stdout.
+
+## Expected output
+
+Compare ten integers. These two columns are the specified baseline.
+
+| implementation | U15.1.0 (1187) | U16.0.0 (1093) |
+|---|---|---|
+| `Intl.Segmenter` | 1186 | 1092 |
+| `unicode-segmenter` | 1186 | 1092 |
+| `graphemer` | 1180 | 1086 |
+| `grapheme-splitter` | 1175 | 1081 |
+| `runes2` | 730 | 695 |
+
+The printed table shows each cell as `passed/total (rate%)`; the integer to
+check is the one before the slash. If all ten match, the build is correct.
+
+## One caveat, and it is the only one
+
+`Intl.Segmenter` is not a fixed library. It is scored against whatever ICU the
+host Node ships, so its row moves with the runtime. Measured across the CI
+matrix:
+
+| runtime | 15.1.0 | 16.0.0 | `2701 200D 2701` |
+|---|---|---|---|
+| Node 22.22.2 (ICU 78.2) | 1186 | 1092 | split, the known deviation |
+| Node 20.x | 1186 | 1092 | split |
+| Node 18.20.8 | 1187 | 1093 | not split |
+
+Run `npm run scoreboard` on Node 22 or 20 to get the two baseline numbers. On
+Node 18 the `Intl.Segmenter` row reads `1187` and `1093` instead: that ICU
+predates the deviation and passes every case. The other four rows are pinned to
+exact versions and hold identically on every Node.
+
+The baseline was verified on Node 22.22.2 (ICU 78.2, Unicode 17.0).
+
+## Everything else
+
+```
+npm test        # 104 assertions, offline, including all ten integers above
+npm run typecheck
+npm run build
+npm run smoke   # loads the ESM and CJS entry points and scores Intl.Segmenter
+npm run scoreboard:check   # asserts SCOREBOARD.md is byte-identical to a fresh run
+```
+
+No network access is required by any of these. The vectors are committed under
+`vectors/` and are never fetched.

package/dist/cli.cjs

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+"use strict";
+
+// node_modules/tsup/assets/cjs_shims.js
+var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
+var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
+
+// src/cli.ts
+var import_node_module = require("module");
+var import_node_path2 = require("path");
+var import_node_url2 = require("url");
+
+// src/rules.ts
+var VIRAMA = /[्্੍્୍்్್്]/;
+var INDIC_SCRIPT = /[\p{Script=Devanagari}\p{Script=Bengali}\p{Script=Gurmukhi}\p{Script=Gujarati}\p{Script=Oriya}\p{Script=Tamil}\p{Script=Telugu}\p{Script=Kannada}\p{Script=Malayalam}]/u;
+var LETTER = /\p{L}/u;
+var MARK = /\p{M}/u;
+var EXT_PICT = /\p{Extended_Pictographic}/u;
+var ZWJ = "\u200D";
+function isRegionalIndicator(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 127462 && c <= 127487;
+}
+function isHangulJamo(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 4352 && c <= 4607 || c >= 43360 && c <= 43391 || c >= 55216 && c <= 55295;
+}
+function hasIndicConjunct(cps) {
+  for (let i = 0; i < cps.length - 1; i++) {
+    if (!VIRAMA.test(cps[i])) continue;
+    let j = i + 1;
+    while (j < cps.length && (cps[j] === ZWJ || MARK.test(cps[j]))) j++;
+    if (j < cps.length && LETTER.test(cps[j]) && INDIC_SCRIPT.test(cps[j])) {
+      return true;
+    }
+  }
+  return false;
+}
+function hasPictographicZwj(cps) {
+  for (let i = 1; i < cps.length - 1; i++) {
+    if (cps[i] === ZWJ && EXT_PICT.test(cps[i - 1]) && EXT_PICT.test(cps[i + 1])) {
+      return true;
+    }
+  }
+  return false;
+}
+function inferRule(input) {
+  const cps = [...input];
+  if (hasIndicConjunct(cps)) return "GB9c";
+  if (hasPictographicZwj(cps)) return "GB11";
+  if (cps.filter(isRegionalIndicator).length >= 2) return "GB12/GB13";
+  if (cps.some(isHangulJamo)) return "GB6/GB7/GB8";
+  return null;
+}
+
+// src/score.ts
+function toHex(input) {
+  return [...input].map((cp) => cp.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")).join(" ");
+}
+function sameClusters(a, b) {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+function score(segmenter, vectors2) {
+  const failures = [];
+  let passed = 0;
+  for (const vector of vectors2) {
+    let actual;
+    try {
+      const result = segmenter(vector.input);
+      actual = Array.isArray(result) && result.every((c) => typeof c === "string") ? [...result] : [];
+    } catch {
+      actual = [];
+    }
+    if (sameClusters(actual, vector.expected)) {
+      passed++;
+    } else {
+      failures.push({
+        line: vector.line,
+        input: vector.input,
+        inputHex: toHex(vector.input),
+        expected: [...vector.expected],
+        actual,
+        rule: inferRule(vector.input)
+      });
+    }
+  }
+  const total = vectors2.length;
+  return { passed, total, rate: total === 0 ? 1 : passed / total, failures };
+}
+
+// src/vectors.ts
+var import_node_fs = require("fs");
+var import_node_path = require("path");
+var import_node_url = require("url");
+
+// src/parse.ts
+var BREAK = "\xF7";
+var NO_BREAK = "\xD7";
+function parseBreakTest(source) {
+  const vectors2 = [];
+  const lines = source.split(/\r\n|\r|\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const hash = lines[i].indexOf("#");
+    const body = (hash === -1 ? lines[i] : lines[i].slice(0, hash)).trim();
+    if (body === "") continue;
+    const tokens = body.split(/\s+/);
+    if (tokens[0] !== BREAK) continue;
+    const expected = [];
+    let cluster = "";
+    let malformed = false;
+    for (const token of tokens) {
+      if (token === BREAK) {
+        if (cluster !== "") expected.push(cluster);
+        cluster = "";
+      } else if (token === NO_BREAK) {
+        continue;
+      } else if (/^[0-9A-Fa-f]+$/.test(token)) {
+        cluster += String.fromCodePoint(parseInt(token, 16));
+      } else {
+        malformed = true;
+        break;
+      }
+    }
+    if (malformed) continue;
+    if (cluster !== "") expected.push(cluster);
+    if (expected.length === 0) continue;
+    vectors2.push({ input: expected.join(""), expected, line: i + 1 });
+  }
+  return vectors2;
+}
+
+// src/vectors.ts
+var VERSIONS = ["15.0.0", "15.1.0", "16.0.0", "17.0.0"];
+function vectorsDir() {
+  let dir = (0, import_node_path.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl));
+  for (let i = 0; i < 4; i++) {
+    const candidate = (0, import_node_path.join)(dir, "vectors");
+    if ((0, import_node_fs.existsSync)((0, import_node_path.join)(candidate, "GraphemeBreakTest-15.1.0.txt"))) return candidate;
+    const parent = (0, import_node_path.resolve)(dir, "..");
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(
+    "grapheme-conformance: vendored vectors/ directory not found next to the package"
+  );
+}
+function load() {
+  const dir = vectorsDir();
+  const out = /* @__PURE__ */ Object.create(null);
+  for (const version of VERSIONS) {
+    const source = (0, import_node_fs.readFileSync)((0, import_node_path.join)(dir, `GraphemeBreakTest-${version}.txt`), "utf8");
+    out[version] = parseBreakTest(source);
+  }
+  return out;
+}
+var vectors = load();
+
+// src/cli.ts
+var USAGE = `Usage: grapheme-conformance --module <specifier> [options]
+  --module  <specifier>  module to load (bare name, or path relative to cwd)
+  --export  <name>       export to score                  (default: default)
+  --version <x.y.z>      vendored vectors: ${Object.keys(vectors).join(", ")}
+  --min     <0..1>       minimum pass rate                (default: 1.0)
+  --limit   <n>          failing cases to print           (default: 10)
+Exits non-zero when the pass rate is below --min.`;
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "-h" || arg === "--help") return { help: "true" };
+    if (!arg.startsWith("--")) continue;
+    const eq = arg.indexOf("=");
+    if (eq !== -1) out[arg.slice(2, eq)] = arg.slice(eq + 1);
+    else out[arg.slice(2)] = argv[++i] ?? "";
+  }
+  return out;
+}
+function fail(message) {
+  console.error(message);
+  process.exit(2);
+}
+async function load2(specifier, exportName) {
+  const relative = specifier.startsWith(".") || (0, import_node_path2.isAbsolute)(specifier);
+  const target = relative ? (0, import_node_url2.pathToFileURL)((0, import_node_path2.resolve)(process.cwd(), specifier)).href : specifier;
+  let module2;
+  try {
+    module2 = await import(target);
+  } catch {
+    const from = (0, import_node_module.createRequire)((0, import_node_path2.resolve)(process.cwd(), "noop.js"));
+    module2 = await import((0, import_node_url2.pathToFileURL)(from.resolve(specifier)).href);
+  }
+  const direct = module2[exportName];
+  const nested = module2.default?.[exportName];
+  const picked = typeof direct === "function" ? direct : nested;
+  if (typeof picked !== "function") fail(`No callable export '${exportName}' in '${specifier}'.`);
+  return (input) => {
+    const result = picked(input);
+    if (Array.isArray(result)) return result;
+    if (result && typeof result[Symbol.iterator] === "function") {
+      return [...result];
+    }
+    throw new TypeError("segmenter did not return an array of clusters");
+  };
+}
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help || !args.module) {
+    console.log(USAGE);
+    process.exit(args.module ? 0 : 2);
+  }
+  const version = args.version ?? "16.0.0";
+  const cases = vectors[version];
+  if (!cases) fail(`Unknown version '${version}'. Have: ${Object.keys(vectors).join(", ")}`);
+  const min = args.min === void 0 ? 1 : Number(args.min);
+  if (!Number.isFinite(min) || min < 0 || min > 1) fail("--min must be between 0 and 1.");
+  const limit = args.limit === void 0 ? 10 : Number(args.limit);
+  const exportName = args.export ?? "default";
+  const report = score(await load2(args.module, exportName), cases);
+  console.log(`${args.module} (${exportName})  GraphemeBreakTest ${version}`);
+  console.log(`  passed  ${report.passed}/${report.total}  ${(report.rate * 100).toFixed(2)}%`);
+  console.log(`  failed  ${report.failures.length}`);
+  if (report.failures.length > 0) {
+    console.log(`
+  ${"line".padEnd(6)}${"input".padEnd(34)}${"want".padEnd(6)}${"got".padEnd(6)}rule`);
+    for (const f of report.failures.slice(0, Math.max(0, limit))) {
+      const hex = f.inputHex.length > 32 ? `${f.inputHex.slice(0, 29)}...` : f.inputHex;
+      console.log(
+        `  ${String(f.line).padEnd(6)}${hex.padEnd(34)}${String(f.expected.length).padEnd(6)}${String(f.actual.length).padEnd(6)}${f.rule ?? "-"}`
+      );
+    }
+    if (report.failures.length > limit) console.log(`  ... and ${report.failures.length - limit} more`);
+  }
+  process.exit(report.rate < min ? 1 : 0);
+}
+main().catch((error) => fail(error instanceof Error ? error.message : String(error)));

package/dist/cli.d.cts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node