@remnic/core 9.62.0 → 9.62.1

package/src/whitespace.test.ts

@@ -0,0 +1,178 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  collapseWhitespace,
+  displayWidth,
+  graphemeUnits,
+  padEndDisplay,
+  truncateCodePointSafe,
+  truncateGraphemeSafe,
+} from "./whitespace.js";
+
+function hasLoneSurrogate(value: string): boolean {
+  for (let index = 0; index < value.length; index += 1) {
+    const code = value.charCodeAt(index);
+    const isHigh = code >= 0xd800 && code <= 0xdbff;
+    const isLow = code >= 0xdc00 && code <= 0xdfff;
+    if (isHigh) {
+      const next = value.charCodeAt(index + 1);
+      if (next < 0xdc00 || next > 0xdfff) return true;
+      index += 1;
+    } else if (isLow) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function isGraphemePrefix(prefix: string, value: string): boolean {
+  let accumulated = "";
+  for (const unit of graphemeUnits(value)) {
+    if (accumulated === prefix) return true;
+    accumulated += unit;
+  }
+  return accumulated === prefix;
+}
+
+// Deterministic pseudo-random source so failures replay exactly.
+function lcg(seed: number): () => number {
+  let state = seed >>> 0;
+  return () => {
+    state = (state * 1664525 + 1013904223) >>> 0;
+    return state / 0x1_0000_0000;
+  };
+}
+
+// Every entry is one or more whole grapheme clusters covering the cases named
+// in issue #2195: astral emoji, ZWJ sequences, skin-tone modifiers,
+// regional-indicator flags, Hangul (precomposed and jamo), combining marks,
+// and astral kanji.
+const GRAPHEME_PIECES = [
+  "a",
+  "b",
+  " ",
+  "7",
+  "😀",
+  "👨‍👩‍👧‍👦",
+  "👍🏽",
+  "🇯🇵",
+  "🇺🇸",
+  "🇰🇷",
+  "한",
+  "한",
+  "e\u0301",
+  "k\u0331\u0301",
+  "क़",
+  "𠮷",
+  "日",
+  "本",
+  "語",
+] as const;
+
+test("truncateGraphemeSafe: property - valid Unicode, grapheme-aligned, within budget", () => {
+  const random = lcg(20260816);
+  for (let iteration = 0; iteration < 500; iteration += 1) {
+    let input = "";
+    const pieceCount = 1 + Math.floor(random() * 12);
+    for (let piece = 0; piece < pieceCount; piece += 1) {
+      input += GRAPHEME_PIECES[Math.floor(random() * GRAPHEME_PIECES.length)]!;
+    }
+    const maxChars = Math.floor(random() * (input.length + 2));
+    const output = truncateGraphemeSafe(input, maxChars);
+    assert.ok(
+      output.length <= maxChars,
+      `budget exceeded for ${JSON.stringify(input)} @ ${maxChars}: ${JSON.stringify(output)}`,
+    );
+    assert.ok(
+      !hasLoneSurrogate(output),
+      `lone surrogate for ${JSON.stringify(input)} @ ${maxChars}: ${JSON.stringify(output)}`,
+    );
+    assert.ok(
+      isGraphemePrefix(output, input),
+      `split grapheme for ${JSON.stringify(input)} @ ${maxChars}: ${JSON.stringify(output)}`,
+    );
+    if (input.length <= maxChars) {
+      assert.equal(output, input, `fits must pass through @ ${maxChars}`);
+    }
+  }
+});
+
+test("truncateGraphemeSafe: emoji, ZWJ, flags, jamo, and combining marks stay whole", () => {
+  assert.equal(truncateGraphemeSafe("hello world", 5), "hello");
+  assert.equal(truncateGraphemeSafe("hello", 50), "hello");
+  assert.equal(truncateGraphemeSafe("hello", 0), "");
+  assert.equal(truncateGraphemeSafe("👍👍", 2), "👍");
+  assert.equal(truncateGraphemeSafe("👍", 1), "", "a split emoji is dropped, never half-cut");
+  assert.equal(truncateGraphemeSafe("a👍b", 2), "a");
+  assert.equal(truncateGraphemeSafe("🇯🇵🇺🇸", 4), "🇯🇵");
+  assert.equal(truncateGraphemeSafe("🇯🇵🇺🇸", 3), "", "a split flag is dropped, never half-cut");
+  const family = "👨‍👩‍👧‍👦";
+  assert.equal(truncateGraphemeSafe(`${family}!`, family.length + 1), `${family}!`);
+  assert.equal(truncateGraphemeSafe(`${family}!`, family.length), family);
+  assert.equal(truncateGraphemeSafe(family, family.length - 1), "", "ZWJ sequence is atomic");
+  assert.equal(truncateGraphemeSafe("한글", 3), "한", "jamo run is one cluster");
+  assert.equal(truncateGraphemeSafe("e\u0301x", 2), "e\u0301", "base plus mark is one cluster");
+  assert.equal(truncateGraphemeSafe("e\u0301x", 1), "");
+});
+
+test("truncateCodePointSafe and collapseWhitespace keep prior behavior", () => {
+  assert.equal(collapseWhitespace(" a \n b "), "a b");
+  assert.equal(truncateCodePointSafe("abcd", 2), "ab");
+  assert.equal(truncateCodePointSafe("ab", 9), "ab");
+  assert.equal(truncateCodePointSafe("abc", 0), "");
+});
+
+test("displayWidth: CJK wide, fullwidth, halfwidth, marks, and emoji", () => {
+  assert.equal(displayWidth(""), 0);
+  assert.equal(displayWidth("abc"), 3);
+  assert.equal(displayWidth("日本語"), 6);
+  assert.equal(displayWidth("ハングル"), 8);
+  assert.equal(displayWidth("한국어"), 6);
+  assert.equal(displayWidth("ｱ"), 1, "halfwidth katakana is narrow");
+  assert.equal(displayWidth("ｆｕｌｌ"), 8, "fullwidth ASCII is wide");
+  assert.equal(displayWidth("e\u0301"), 1, "combining mark is zero width");
+  assert.equal(displayWidth("😀"), 2);
+  assert.equal(displayWidth("中日 memo"), 9, "2+2+1+4");
+});
+
+test("displayWidth: emoji sequences measure as one two-cell grapheme", () => {
+  assert.equal(displayWidth("👨‍👩‍👧‍👦"), 2, "ZWJ family emoji is one cluster");
+  assert.equal(displayWidth("🇯🇵"), 2, "regional-indicator flag is one cluster");
+  assert.equal(displayWidth("👍🏽"), 2, "skin-tone modifier joins the cluster");
+  assert.equal(displayWidth("❤️"), 2, "VS16 requests emoji presentation");
+  assert.equal(displayWidth("1️⃣"), 2, "keycap sequence is one cluster");
+  assert.equal(displayWidth("👨‍👩‍👧‍👦 文"), 5, "clusters compose: 2 + 1 + 2");
+  assert.equal(
+    padEndDisplay("👨‍👩‍👧‍👦", 6),
+    "👨‍👩‍👧‍👦" + " ".repeat(4),
+    "emoji rows pad by display width, not code units",
+  );
+});
+
+test("padEndDisplay: CJK rows align to one computed width", () => {
+  assert.equal(padEndDisplay("abc", 6), "abc   ");
+  assert.equal(padEndDisplay("abc", 2), "abc", "narrow target never truncates");
+  assert.equal(padEndDisplay("日本", 10), "日本" + " ".repeat(6));
+  const rows = ["機械学習", "nlp", "한국어", "x"];
+  const padded = rows.map((row) => padEndDisplay(row, 12));
+  for (const row of padded) {
+    assert.equal(displayWidth(row), 12);
+  }
+  assert.deepEqual(padded, [
+    "機械学習    ",
+    "nlp         ",
+    "한국어      ",
+    "x           ",
+  ]);
+  // Snapshot-style: the CLI topics table shape keeps its score column aligned
+  // for CJK rows, which plain padEnd misaligns.
+  const table = rows.slice(0, 3).map((term) => `  ${padEndDisplay(term, 12)} 0.123`);
+  assert.deepEqual(table, [
+    "  機械学習     0.123",
+    "  nlp          0.123",
+    "  한국어       0.123",
+  ]);
+  const tableWidth = displayWidth(table[0]!);
+  assert.ok(table.every((line) => displayWidth(line) === tableWidth));
+});

package/src/whitespace.ts

@@ -8,3 +8,122 @@ export function truncateCodePointSafe(value: string, maxChars: number): string {
   if (glyphs.length <= maxChars) return value;
   return glyphs.slice(0, Math.max(1, maxChars)).join("").trimEnd();
 }
+
+let graphemeSegmenter: Intl.Segmenter | null | undefined;
+
+function getGraphemeSegmenter(): Intl.Segmenter | null {
+  if (graphemeSegmenter === undefined) {
+    graphemeSegmenter =
+      typeof Intl !== "undefined" && typeof Intl.Segmenter === "function"
+        ? new Intl.Segmenter(undefined, { granularity: "grapheme" })
+        : null;
+  }
+  return graphemeSegmenter;
+}
+
+/**
+ * Longest prefix of `value` whose UTF-16 length stays within `maxChars`
+ * without splitting a surrogate pair or a grapheme cluster (ZWJ emoji
+ * sequences, flags, Hangul jamo runs, combining marks). A cluster wider than
+ * the budget is dropped whole rather than split. Falls back to code-point
+ * boundaries when Intl.Segmenter is unavailable.
+ */
+export function truncateGraphemeSafe(value: string, maxChars: number): string {
+  if (maxChars <= 0) return "";
+  if (value.length <= maxChars) return value;
+  let truncated = "";
+  for (const unit of graphemeUnits(value)) {
+    if (truncated.length + unit.length > maxChars) break;
+    truncated += unit;
+  }
+  return truncated;
+}
+
+/**
+ * Grapheme clusters of `value` as separate strings, or its code points when
+ * Intl.Segmenter is unavailable.
+ */
+export function graphemeUnits(value: string): string[] {
+  const segmenter = getGraphemeSegmenter();
+  if (segmenter === null) return [...value];
+  return Array.from(segmenter.segment(value), (segment) => segment.segment);
+}
+
+const COMBINING_MARK = /\p{M}/u;
+const FORMAT_CHAR = /\p{Cf}/u;
+const EMOJI_PRESENTATION = /\p{Emoji_Presentation}/u;
+const EMOJI_PRESENTATION_SELECTOR = "\uFE0F";
+
+// ponytail: pragmatic East_Asian_Width Wide/Fullwidth subset - contiguous
+// block ranges plus the emoji presentation planes. Sparse exceptions inside
+// those ranges (narrow unassigned code points, U+303F) do not matter for
+// terminal column alignment; switch to a full EAW table only if drift shows.
+const WIDE_CODE_POINT_RANGES: ReadonlyArray<readonly [number, number]> = [
+  [0x1100, 0x115f], // Hangul Jamo
+  [0x2e80, 0x303e], // CJK radicals, Kangxi radicals, CJK symbols and punctuation
+  [0x3041, 0x33ff], // Hiragana, Katakana, Bopomofo, Hangul compatibility jamo
+  [0x3400, 0x4dbf], // CJK unified ideographs extension A
+  [0x4e00, 0xa4cf], // CJK unified ideographs, Yi
+  [0xa960, 0xa97f], // Hangul Jamo extension A
+  [0xac00, 0xd7a3], // Hangul syllables
+  [0xf900, 0xfaff], // CJK compatibility ideographs
+  [0xfe10, 0xfe19], // Vertical forms
+  [0xfe30, 0xfe6b], // CJK compatibility forms
+  [0xff00, 0xff60], // Fullwidth ASCII and punctuation
+  [0xffe0, 0xffe6], // Fullwidth signs
+  [0x1f300, 0x1f64f], // Emoji symbols, pictographs, emoticons
+  [0x1f680, 0x1f6ff], // Transport and map
+  [0x1f900, 0x1f9ff], // Supplemental symbols
+  [0x1fa70, 0x1faff], // Symbols and pictographs extended-A
+  [0x20000, 0x2fffd], // CJK extensions B-F
+  [0x30000, 0x3fffd], // CJK extension G and later
+];
+
+function isWideGlyph(glyph: string): boolean {
+  const codePoint = glyph.codePointAt(0)!;
+  let low = 0;
+  let high = WIDE_CODE_POINT_RANGES.length - 1;
+  while (low <= high) {
+    const mid = (low + high) >> 1;
+    const [start, end] = WIDE_CODE_POINT_RANGES[mid]!;
+    if (codePoint < start) high = mid - 1;
+    else if (codePoint > end) low = mid + 1;
+    else return true;
+  }
+  return false;
+}
+
+/**
+ * Terminal cell width: East_Asian_Width Wide/Fullwidth glyphs count as 2,
+ * combining marks and format characters (ZWJ, variation selectors) as 0,
+ * everything else as 1. A grapheme cluster with emoji presentation - ZWJ
+ * sequences, flags, skin-tone modifiers - occupies two cells.
+ */
+export function displayWidth(value: string): number {
+  let width = 0;
+  for (const cluster of graphemeUnits(value)) {
+    width += graphemeClusterWidth(cluster);
+  }
+  return width;
+}
+
+function graphemeClusterWidth(cluster: string): number {
+  if (cluster.includes(EMOJI_PRESENTATION_SELECTOR)) return 2;
+  let emojiCluster = false;
+  let width = 0;
+  for (const glyph of cluster) {
+    if (COMBINING_MARK.test(glyph) || FORMAT_CHAR.test(glyph)) continue;
+    if (EMOJI_PRESENTATION.test(glyph)) emojiCluster = true;
+    width += isWideGlyph(glyph) ? 2 : 1;
+  }
+  return emojiCluster ? 2 : width;
+}
+
+/**
+ * padEnd analog that pads to a terminal cell width, so rows containing CJK
+ * or emoji keep their columns aligned.
+ */
+export function padEndDisplay(value: string, targetWidth: number): string {
+  const padding = targetWidth - displayWidth(value);
+  return padding > 0 ? value + " ".repeat(padding) : value;
+}

package/dist/chunk-6MKAMLQL.js

@@ -1,16 +0,0 @@
-// src/whitespace.ts
-function collapseWhitespace(value) {
-  return value.replace(/\s+/g, " ").trim();
-}
-function truncateCodePointSafe(value, maxChars) {
-  const glyphs = Array.from(value);
-  if (maxChars <= 0) return "";
-  if (glyphs.length <= maxChars) return value;
-  return glyphs.slice(0, Math.max(1, maxChars)).join("").trimEnd();
-}
-
-export {
-  collapseWhitespace,
-  truncateCodePointSafe
-};
-//# sourceMappingURL=chunk-6MKAMLQL.js.map

package/dist/chunk-6MKAMLQL.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/whitespace.ts"],"sourcesContent":["export function collapseWhitespace(value: string): string {\n  return value.replace(/\\s+/g, \" \").trim();\n}\n\nexport function truncateCodePointSafe(value: string, maxChars: number): string {\n  const glyphs = Array.from(value);\n  if (maxChars <= 0) return \"\";\n  if (glyphs.length <= maxChars) return value;\n  return glyphs.slice(0, Math.max(1, maxChars)).join(\"\").trimEnd();\n}\n"],"mappings":";AAAO,SAAS,mBAAmB,OAAuB;AACxD,SAAO,MAAM,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACzC;AAEO,SAAS,sBAAsB,OAAe,UAA0B;AAC7E,QAAM,SAAS,MAAM,KAAK,KAAK;AAC/B,MAAI,YAAY,EAAG,QAAO;AAC1B,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,SAAO,OAAO,MAAM,GAAG,KAAK,IAAI,GAAG,QAAQ,CAAC,EAAE,KAAK,EAAE,EAAE,QAAQ;AACjE;","names":[]}