grapheme-conformance 0.1.0

package/dist/cli.js

@@ -0,0 +1,234 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { createRequire } from "node:module";
+import { isAbsolute, resolve as resolve2 } from "node:path";
+import { pathToFileURL } from "node:url";
+
+// src/rules.ts
+var VIRAMA = /[्্੍્୍்్್്]/;
+var INDIC_SCRIPT = /[\p{Script=Devanagari}\p{Script=Bengali}\p{Script=Gurmukhi}\p{Script=Gujarati}\p{Script=Oriya}\p{Script=Tamil}\p{Script=Telugu}\p{Script=Kannada}\p{Script=Malayalam}]/u;
+var LETTER = /\p{L}/u;
+var MARK = /\p{M}/u;
+var EXT_PICT = /\p{Extended_Pictographic}/u;
+var ZWJ = "\u200D";
+function isRegionalIndicator(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 127462 && c <= 127487;
+}
+function isHangulJamo(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 4352 && c <= 4607 || c >= 43360 && c <= 43391 || c >= 55216 && c <= 55295;
+}
+function hasIndicConjunct(cps) {
+  for (let i = 0; i < cps.length - 1; i++) {
+    if (!VIRAMA.test(cps[i])) continue;
+    let j = i + 1;
+    while (j < cps.length && (cps[j] === ZWJ || MARK.test(cps[j]))) j++;
+    if (j < cps.length && LETTER.test(cps[j]) && INDIC_SCRIPT.test(cps[j])) {
+      return true;
+    }
+  }
+  return false;
+}
+function hasPictographicZwj(cps) {
+  for (let i = 1; i < cps.length - 1; i++) {
+    if (cps[i] === ZWJ && EXT_PICT.test(cps[i - 1]) && EXT_PICT.test(cps[i + 1])) {
+      return true;
+    }
+  }
+  return false;
+}
+function inferRule(input) {
+  const cps = [...input];
+  if (hasIndicConjunct(cps)) return "GB9c";
+  if (hasPictographicZwj(cps)) return "GB11";
+  if (cps.filter(isRegionalIndicator).length >= 2) return "GB12/GB13";
+  if (cps.some(isHangulJamo)) return "GB6/GB7/GB8";
+  return null;
+}
+
+// src/score.ts
+function toHex(input) {
+  return [...input].map((cp) => cp.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")).join(" ");
+}
+function sameClusters(a, b) {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+function score(segmenter, vectors2) {
+  const failures = [];
+  let passed = 0;
+  for (const vector of vectors2) {
+    let actual;
+    try {
+      const result = segmenter(vector.input);
+      actual = Array.isArray(result) && result.every((c) => typeof c === "string") ? [...result] : [];
+    } catch {
+      actual = [];
+    }
+    if (sameClusters(actual, vector.expected)) {
+      passed++;
+    } else {
+      failures.push({
+        line: vector.line,
+        input: vector.input,
+        inputHex: toHex(vector.input),
+        expected: [...vector.expected],
+        actual,
+        rule: inferRule(vector.input)
+      });
+    }
+  }
+  const total = vectors2.length;
+  return { passed, total, rate: total === 0 ? 1 : passed / total, failures };
+}
+
+// src/vectors.ts
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// src/parse.ts
+var BREAK = "\xF7";
+var NO_BREAK = "\xD7";
+function parseBreakTest(source) {
+  const vectors2 = [];
+  const lines = source.split(/\r\n|\r|\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const hash = lines[i].indexOf("#");
+    const body = (hash === -1 ? lines[i] : lines[i].slice(0, hash)).trim();
+    if (body === "") continue;
+    const tokens = body.split(/\s+/);
+    if (tokens[0] !== BREAK) continue;
+    const expected = [];
+    let cluster = "";
+    let malformed = false;
+    for (const token of tokens) {
+      if (token === BREAK) {
+        if (cluster !== "") expected.push(cluster);
+        cluster = "";
+      } else if (token === NO_BREAK) {
+        continue;
+      } else if (/^[0-9A-Fa-f]+$/.test(token)) {
+        cluster += String.fromCodePoint(parseInt(token, 16));
+      } else {
+        malformed = true;
+        break;
+      }
+    }
+    if (malformed) continue;
+    if (cluster !== "") expected.push(cluster);
+    if (expected.length === 0) continue;
+    vectors2.push({ input: expected.join(""), expected, line: i + 1 });
+  }
+  return vectors2;
+}
+
+// src/vectors.ts
+var VERSIONS = ["15.0.0", "15.1.0", "16.0.0", "17.0.0"];
+function vectorsDir() {
+  let dir = dirname(fileURLToPath(import.meta.url));
+  for (let i = 0; i < 4; i++) {
+    const candidate = join(dir, "vectors");
+    if (existsSync(join(candidate, "GraphemeBreakTest-15.1.0.txt"))) return candidate;
+    const parent = resolve(dir, "..");
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(
+    "grapheme-conformance: vendored vectors/ directory not found next to the package"
+  );
+}
+function load() {
+  const dir = vectorsDir();
+  const out = /* @__PURE__ */ Object.create(null);
+  for (const version of VERSIONS) {
+    const source = readFileSync(join(dir, `GraphemeBreakTest-${version}.txt`), "utf8");
+    out[version] = parseBreakTest(source);
+  }
+  return out;
+}
+var vectors = load();
+
+// src/cli.ts
+var USAGE = `Usage: grapheme-conformance --module <specifier> [options]
+  --module  <specifier>  module to load (bare name, or path relative to cwd)
+  --export  <name>       export to score                  (default: default)
+  --version <x.y.z>      vendored vectors: ${Object.keys(vectors).join(", ")}
+  --min     <0..1>       minimum pass rate                (default: 1.0)
+  --limit   <n>          failing cases to print           (default: 10)
+Exits non-zero when the pass rate is below --min.`;
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "-h" || arg === "--help") return { help: "true" };
+    if (!arg.startsWith("--")) continue;
+    const eq = arg.indexOf("=");
+    if (eq !== -1) out[arg.slice(2, eq)] = arg.slice(eq + 1);
+    else out[arg.slice(2)] = argv[++i] ?? "";
+  }
+  return out;
+}
+function fail(message) {
+  console.error(message);
+  process.exit(2);
+}
+async function load2(specifier, exportName) {
+  const relative = specifier.startsWith(".") || isAbsolute(specifier);
+  const target = relative ? pathToFileURL(resolve2(process.cwd(), specifier)).href : specifier;
+  let module;
+  try {
+    module = await import(target);
+  } catch {
+    const from = createRequire(resolve2(process.cwd(), "noop.js"));
+    module = await import(pathToFileURL(from.resolve(specifier)).href);
+  }
+  const direct = module[exportName];
+  const nested = module.default?.[exportName];
+  const picked = typeof direct === "function" ? direct : nested;
+  if (typeof picked !== "function") fail(`No callable export '${exportName}' in '${specifier}'.`);
+  return (input) => {
+    const result = picked(input);
+    if (Array.isArray(result)) return result;
+    if (result && typeof result[Symbol.iterator] === "function") {
+      return [...result];
+    }
+    throw new TypeError("segmenter did not return an array of clusters");
+  };
+}
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help || !args.module) {
+    console.log(USAGE);
+    process.exit(args.module ? 0 : 2);
+  }
+  const version = args.version ?? "16.0.0";
+  const cases = vectors[version];
+  if (!cases) fail(`Unknown version '${version}'. Have: ${Object.keys(vectors).join(", ")}`);
+  const min = args.min === void 0 ? 1 : Number(args.min);
+  if (!Number.isFinite(min) || min < 0 || min > 1) fail("--min must be between 0 and 1.");
+  const limit = args.limit === void 0 ? 10 : Number(args.limit);
+  const exportName = args.export ?? "default";
+  const report = score(await load2(args.module, exportName), cases);
+  console.log(`${args.module} (${exportName})  GraphemeBreakTest ${version}`);
+  console.log(`  passed  ${report.passed}/${report.total}  ${(report.rate * 100).toFixed(2)}%`);
+  console.log(`  failed  ${report.failures.length}`);
+  if (report.failures.length > 0) {
+    console.log(`
+  ${"line".padEnd(6)}${"input".padEnd(34)}${"want".padEnd(6)}${"got".padEnd(6)}rule`);
+    for (const f of report.failures.slice(0, Math.max(0, limit))) {
+      const hex = f.inputHex.length > 32 ? `${f.inputHex.slice(0, 29)}...` : f.inputHex;
+      console.log(
+        `  ${String(f.line).padEnd(6)}${hex.padEnd(34)}${String(f.expected.length).padEnd(6)}${String(f.actual.length).padEnd(6)}${f.rule ?? "-"}`
+      );
+    }
+    if (report.failures.length > limit) console.log(`  ... and ${report.failures.length - limit} more`);
+  }
+  process.exit(report.rate < min ? 1 : 0);
+}
+main().catch((error) => fail(error instanceof Error ? error.message : String(error)));

package/dist/index.cjs

@@ -0,0 +1,184 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  parseBreakTest: () => parseBreakTest,
+  score: () => score,
+  vectors: () => vectors
+});
+module.exports = __toCommonJS(index_exports);
+
+// node_modules/tsup/assets/cjs_shims.js
+var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
+var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
+
+// src/parse.ts
+var BREAK = "\xF7";
+var NO_BREAK = "\xD7";
+function parseBreakTest(source) {
+  const vectors2 = [];
+  const lines = source.split(/\r\n|\r|\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const hash = lines[i].indexOf("#");
+    const body = (hash === -1 ? lines[i] : lines[i].slice(0, hash)).trim();
+    if (body === "") continue;
+    const tokens = body.split(/\s+/);
+    if (tokens[0] !== BREAK) continue;
+    const expected = [];
+    let cluster = "";
+    let malformed = false;
+    for (const token of tokens) {
+      if (token === BREAK) {
+        if (cluster !== "") expected.push(cluster);
+        cluster = "";
+      } else if (token === NO_BREAK) {
+        continue;
+      } else if (/^[0-9A-Fa-f]+$/.test(token)) {
+        cluster += String.fromCodePoint(parseInt(token, 16));
+      } else {
+        malformed = true;
+        break;
+      }
+    }
+    if (malformed) continue;
+    if (cluster !== "") expected.push(cluster);
+    if (expected.length === 0) continue;
+    vectors2.push({ input: expected.join(""), expected, line: i + 1 });
+  }
+  return vectors2;
+}
+
+// src/rules.ts
+var VIRAMA = /[्্੍્୍்్್്]/;
+var INDIC_SCRIPT = /[\p{Script=Devanagari}\p{Script=Bengali}\p{Script=Gurmukhi}\p{Script=Gujarati}\p{Script=Oriya}\p{Script=Tamil}\p{Script=Telugu}\p{Script=Kannada}\p{Script=Malayalam}]/u;
+var LETTER = /\p{L}/u;
+var MARK = /\p{M}/u;
+var EXT_PICT = /\p{Extended_Pictographic}/u;
+var ZWJ = "\u200D";
+function isRegionalIndicator(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 127462 && c <= 127487;
+}
+function isHangulJamo(cp) {
+  const c = cp.codePointAt(0);
+  return c >= 4352 && c <= 4607 || c >= 43360 && c <= 43391 || c >= 55216 && c <= 55295;
+}
+function hasIndicConjunct(cps) {
+  for (let i = 0; i < cps.length - 1; i++) {
+    if (!VIRAMA.test(cps[i])) continue;
+    let j = i + 1;
+    while (j < cps.length && (cps[j] === ZWJ || MARK.test(cps[j]))) j++;
+    if (j < cps.length && LETTER.test(cps[j]) && INDIC_SCRIPT.test(cps[j])) {
+      return true;
+    }
+  }
+  return false;
+}
+function hasPictographicZwj(cps) {
+  for (let i = 1; i < cps.length - 1; i++) {
+    if (cps[i] === ZWJ && EXT_PICT.test(cps[i - 1]) && EXT_PICT.test(cps[i + 1])) {
+      return true;
+    }
+  }
+  return false;
+}
+function inferRule(input) {
+  const cps = [...input];
+  if (hasIndicConjunct(cps)) return "GB9c";
+  if (hasPictographicZwj(cps)) return "GB11";
+  if (cps.filter(isRegionalIndicator).length >= 2) return "GB12/GB13";
+  if (cps.some(isHangulJamo)) return "GB6/GB7/GB8";
+  return null;
+}
+
+// src/score.ts
+function toHex(input) {
+  return [...input].map((cp) => cp.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")).join(" ");
+}
+function sameClusters(a, b) {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+function score(segmenter, vectors2) {
+  const failures = [];
+  let passed = 0;
+  for (const vector of vectors2) {
+    let actual;
+    try {
+      const result = segmenter(vector.input);
+      actual = Array.isArray(result) && result.every((c) => typeof c === "string") ? [...result] : [];
+    } catch {
+      actual = [];
+    }
+    if (sameClusters(actual, vector.expected)) {
+      passed++;
+    } else {
+      failures.push({
+        line: vector.line,
+        input: vector.input,
+        inputHex: toHex(vector.input),
+        expected: [...vector.expected],
+        actual,
+        rule: inferRule(vector.input)
+      });
+    }
+  }
+  const total = vectors2.length;
+  return { passed, total, rate: total === 0 ? 1 : passed / total, failures };
+}
+
+// src/vectors.ts
+var import_node_fs = require("fs");
+var import_node_path = require("path");
+var import_node_url = require("url");
+var VERSIONS = ["15.0.0", "15.1.0", "16.0.0", "17.0.0"];
+function vectorsDir() {
+  let dir = (0, import_node_path.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl));
+  for (let i = 0; i < 4; i++) {
+    const candidate = (0, import_node_path.join)(dir, "vectors");
+    if ((0, import_node_fs.existsSync)((0, import_node_path.join)(candidate, "GraphemeBreakTest-15.1.0.txt"))) return candidate;
+    const parent = (0, import_node_path.resolve)(dir, "..");
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(
+    "grapheme-conformance: vendored vectors/ directory not found next to the package"
+  );
+}
+function load() {
+  const dir = vectorsDir();
+  const out = /* @__PURE__ */ Object.create(null);
+  for (const version of VERSIONS) {
+    const source = (0, import_node_fs.readFileSync)((0, import_node_path.join)(dir, `GraphemeBreakTest-${version}.txt`), "utf8");
+    out[version] = parseBreakTest(source);
+  }
+  return out;
+}
+var vectors = load();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parseBreakTest,
+  score,
+  vectors
+});

package/dist/index.d.cts

@@ -0,0 +1,78 @@
+/**
+ * A function that splits a string into user-perceived characters.
+ *
+ * Both `Intl.Segmenter` wrappers and library calls fit this shape:
+ *
+ * ```ts
+ * const seg = new Intl.Segmenter('en', { granularity: 'grapheme' });
+ * const segmenter: Segmenter = (s) => [...seg.segment(s)].map((x) => x.segment);
+ * ```
+ */
+type Segmenter = (input: string) => string[];
+/** One case from `GraphemeBreakTest.txt`. */
+interface Vector {
+    /** The full input string, all clusters concatenated. */
+    input: string;
+    /** The clusters the input must split into. */
+    expected: string[];
+    /** 1-based line number in the source `.txt`. */
+    line: number;
+}
+/** A single case a segmenter got wrong. */
+interface Failure {
+    /** 1-based line number in the source `.txt`. */
+    line: number;
+    input: string;
+    /** Space-separated UPPERCASE hex code points, e.g. `'0915 094D 0937'`. */
+    inputHex: string;
+    expected: string[];
+    /** What the segmenter returned. `[]` if it threw. */
+    actual: string[];
+    /** Inferred rule id, or `null` when no rule matched. */
+    rule: string | null;
+}
+/** The result of scoring one segmenter against one set of vectors. */
+interface Report {
+    passed: number;
+    total: number;
+    /** `passed / total`, or `1` when there are no vectors. */
+    rate: number;
+    /** Every failing case, ordered by `line`. */
+    failures: Failure[];
+}
+
+/**
+ * Parse the text of a Unicode `GraphemeBreakTest.txt` into vectors.
+ *
+ * Pure: takes a string, touches no filesystem, no network.
+ *
+ * ```
+ * ÷ 0915 × 094D × 0937 ÷ 093F ÷  # comment
+ * ```
+ *
+ * `÷` means break, `×` means no break, hex code points sit between them and
+ * `#` starts a comment. Lines that are blank, or whose first token is not `÷`,
+ * are skipped.
+ */
+declare function parseBreakTest(source: string): Vector[];
+
+/**
+ * Run `segmenter` over every vector and report what it got wrong.
+ *
+ * Never throws. A segmenter that throws on an input, or returns anything that
+ * is not an array of strings, yields a failure with `actual: []`.
+ *
+ * `failures` is ordered by `line`, and repeated calls on the same inputs
+ * produce deeply equal reports.
+ */
+declare function score(segmenter: Segmenter, vectors: Vector[]): Report;
+
+/**
+ * The vendored official test vectors, keyed by Unicode version.
+ *
+ * Read from the committed `.txt` files at import time. Nothing is fetched, at
+ * build time or any other time.
+ */
+declare const vectors: Record<string, Vector[]>;
+
+export { type Failure, type Report, type Segmenter, type Vector, parseBreakTest, score, vectors };

package/dist/index.d.ts

@@ -0,0 +1,78 @@
+/**
+ * A function that splits a string into user-perceived characters.
+ *
+ * Both `Intl.Segmenter` wrappers and library calls fit this shape:
+ *
+ * ```ts
+ * const seg = new Intl.Segmenter('en', { granularity: 'grapheme' });
+ * const segmenter: Segmenter = (s) => [...seg.segment(s)].map((x) => x.segment);
+ * ```
+ */
+type Segmenter = (input: string) => string[];
+/** One case from `GraphemeBreakTest.txt`. */
+interface Vector {
+    /** The full input string, all clusters concatenated. */
+    input: string;
+    /** The clusters the input must split into. */
+    expected: string[];
+    /** 1-based line number in the source `.txt`. */
+    line: number;
+}
+/** A single case a segmenter got wrong. */
+interface Failure {
+    /** 1-based line number in the source `.txt`. */
+    line: number;
+    input: string;
+    /** Space-separated UPPERCASE hex code points, e.g. `'0915 094D 0937'`. */
+    inputHex: string;
+    expected: string[];
+    /** What the segmenter returned. `[]` if it threw. */
+    actual: string[];
+    /** Inferred rule id, or `null` when no rule matched. */
+    rule: string | null;
+}
+/** The result of scoring one segmenter against one set of vectors. */
+interface Report {
+    passed: number;
+    total: number;
+    /** `passed / total`, or `1` when there are no vectors. */
+    rate: number;
+    /** Every failing case, ordered by `line`. */
+    failures: Failure[];
+}
+
+/**
+ * Parse the text of a Unicode `GraphemeBreakTest.txt` into vectors.
+ *
+ * Pure: takes a string, touches no filesystem, no network.
+ *
+ * ```
+ * ÷ 0915 × 094D × 0937 ÷ 093F ÷  # comment
+ * ```
+ *
+ * `÷` means break, `×` means no break, hex code points sit between them and
+ * `#` starts a comment. Lines that are blank, or whose first token is not `÷`,
+ * are skipped.
+ */
+declare function parseBreakTest(source: string): Vector[];
+
+/**
+ * Run `segmenter` over every vector and report what it got wrong.
+ *
+ * Never throws. A segmenter that throws on an input, or returns anything that
+ * is not an array of strings, yields a failure with `actual: []`.
+ *
+ * `failures` is ordered by `line`, and repeated calls on the same inputs
+ * produce deeply equal reports.
+ */
+declare function score(segmenter: Segmenter, vectors: Vector[]): Report;
+
+/**
+ * The vendored official test vectors, keyed by Unicode version.
+ *
+ * Read from the committed `.txt` files at import time. Nothing is fetched, at
+ * build time or any other time.
+ */
+declare const vectors: Record<string, Vector[]>;
+
+export { type Failure, type Report, type Segmenter, type Vector, parseBreakTest, score, vectors };