@remnic/core 9.62.0 → 9.62.2

package/src/storage/tombstone-migration-sources.test.ts

@@ -0,0 +1,116 @@
+import * as assert from "node:assert/strict";
+import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { createTombstoneMigrationSourceContents } from "./tombstone-migration-sources.js";
+import type { MemoryFile } from "../types.js";
+
+function memory(id: string, content: string): MemoryFile {
+  return {
+    path: `facts/${id}.md`,
+    rawContent: content,
+    frontmatter: { id, category: "fact", status: "active" },
+  } as unknown as MemoryFile;
+}
+
+interface Harness {
+  dir: string;
+  ledgerPath: string;
+  collectCalls: number;
+  readsByPaths: string[][];
+  gateCollect: Promise<void>;
+  releaseCollect: () => void;
+}
+
+async function makeHarness(): Promise<Harness> {
+  const dir = await mkdtemp(path.join(tmpdir(), "tomb-migration-race-"));
+  const ledgerPath = path.join(dir, "tombstones.jsonl");
+  await writeFile(ledgerPath, "", "utf8");
+  let gateCollect: (() => void) | undefined;
+  const gate = new Promise<void>((resolve) => {
+    gateCollect = resolve;
+  });
+  const harness: Harness = {
+    dir,
+    ledgerPath,
+    collectCalls: 0,
+    readsByPaths: [],
+    gateCollect: gate,
+    releaseCollect: () => gateCollect?.(),
+  };
+  return harness;
+}
+
+test("an in-flight fill does not publish a stale cache after the ledger advances", async () => {
+  const h = await makeHarness();
+  try {
+    const generations: string[][] = [["facts/old-a.md"], ["facts/new-a.md", "facts/old-a.md"]];
+    const contents = new Map<string, string>([
+      ["facts/old-a.md", "old body"],
+      ["facts/new-a.md", "new body"],
+    ]);
+    const sourceContents = createTombstoneMigrationSourceContents({
+      tombstonesPath: () => h.ledgerPath,
+      collectTombstoneMigrationPaths: async () => {
+        const callIndex = h.collectCalls + 1;
+        h.collectCalls = callIndex;
+        if (callIndex === 1) {
+          // First fill stalls while a peer retires a legacy memory; the
+          // generation is bound at call entry so the stall cannot upgrade
+          // it to the fresh revision.
+          await h.gateCollect;
+        }
+        return generations[Math.min(callIndex - 1, generations.length - 1)]!;
+      },
+      readParsedMemoriesFromPaths: async (filePaths: string[]) => {
+        h.readsByPaths.push([...filePaths]);
+        return filePaths
+          .filter((filePath) => contents.has(filePath))
+          .map((filePath) => memory(path.basename(filePath, ".md"), contents.get(filePath)!));
+      },
+      storedContentIdentityCandidates: (content: string) => [content],
+    });
+
+    // The first callback requests an id whose path exists only in the new
+    // ledger revision, so a stale snapshot would miss it entirely.
+    const first = sourceContents(["new-a"]);
+    // Advance the ledger mtime so a later callback drops both caches while
+    // the first fill is still awaiting its (stale) path snapshot.
+    await utimes(h.ledgerPath, new Date(), new Date(Date.now() + 5000));
+    const second = sourceContents(["old-a"]);
+    h.releaseCollect();
+
+    // Without the revision guard, the first callback would miss on the
+    // stale snapshot, publish a full map built from it, and resolve empty.
+    const firstResult = await first;
+    assert.deepEqual([...firstResult.keys()], ["new-a"]);
+    const secondResult = await second;
+    assert.deepEqual([...secondResult.keys()], ["old-a"]);
+    assert.equal(h.collectCalls >= 2, true, "restart re-collects paths against the fresh revision");
+  } finally {
+    await rm(h.dir, { recursive: true, force: true });
+  }
+});
+
+test("a stable ledger revision reuses the collected paths across callbacks", async () => {
+  const h = await makeHarness();
+  try {
+    const sourceContents = createTombstoneMigrationSourceContents({
+      tombstonesPath: () => h.ledgerPath,
+      collectTombstoneMigrationPaths: async () => {
+        h.collectCalls += 1;
+        return ["facts/a.md"];
+      },
+      readParsedMemoriesFromPaths: async (filePaths: string[]) =>
+        filePaths.map((filePath) => memory(path.basename(filePath, ".md"), "body")),
+      storedContentIdentityCandidates: (content: string) => [content],
+    });
+    await sourceContents(["a"]);
+    await sourceContents(["a"]);
+    assert.equal(h.collectCalls, 1, "unchanged ledger mtime reuses the path snapshot");
+  } finally {
+    await rm(h.dir, { recursive: true, force: true });
+  }
+});

package/src/storage/tombstone-migration-sources.ts

@@ -0,0 +1,90 @@
+import { statSync } from "node:fs";
+import path from "node:path";
+import type { TombstoneStoreOptions } from "../lifecycle/tombstones.js";
+import type { MemoryFile } from "../types.js";
+
+/**
+ * Deps injected by `StorageManager.buildTombstoneStore` so this module owns no
+ * storage-god-file imports (issue #1533 ratchet).
+ */
+export interface TombstoneMigrationSourceDeps {
+  /** On-disk tombstone ledger path; its mtime scopes both caches below. */
+  tombstonesPath: () => string;
+  collectTombstoneMigrationPaths: () => Promise<string[]>;
+  readParsedMemoriesFromPaths: (filePaths: string[], batchSize?: number) => Promise<MemoryFile[]>;
+  storedContentIdentityCandidates: (content: string) => string[];
+}
+
+/**
+ * Build the `sourceContentsForMemoryIds` callback `TombstoneStore` uses to
+ * verify pre-Unicode rows against their retired source memories. The corpus
+ * path snapshot and the resolved source-content map are memoized for one
+ * ledger revision only (issue #2367): a peer process can create and retire a
+ * legacy-format memory after this snapshot, and the staleness reload must
+ * resolve the new row against a FRESH path list — otherwise the row stays
+ * unverified (withheld from the lookup tiers) until restart.
+ */
+export function createTombstoneMigrationSourceContents(
+  deps: TombstoneMigrationSourceDeps,
+): NonNullable<TombstoneStoreOptions["sourceContentsForMemoryIds"]> {
+  let sourcePathsPromise: Promise<string[]> | undefined;
+  let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
+  let ledgerMtimeMs: number | null = null;
+  const dropCachesIfLedgerChanged = (): void => {
+    let mtimeMs: number | null;
+    try {
+      mtimeMs = Math.floor(statSync(deps.tombstonesPath()).mtimeMs);
+    } catch {
+      return; // ENOENT / stat failure — keep whatever is cached.
+    }
+    if (mtimeMs === ledgerMtimeMs) return;
+    ledgerMtimeMs = mtimeMs;
+    sourcePathsPromise = undefined;
+    sourceContentsByIdPromise = undefined;
+  };
+  return async (sourceMemoryIds) => {
+    for (;;) {
+      dropCachesIfLedgerChanged();
+      // In-flight fills are bound to the ledger revision they started
+      // against: if the ledger advances while a fill awaits, restart
+      // against the fresh revision instead of publishing a cache built
+      // from a stale path snapshot (issue #2367 review round 2).
+      const revision = ledgerMtimeMs;
+      const requested = new Set(sourceMemoryIds);
+      const sourcePaths = await (sourcePathsPromise ??= deps.collectTombstoneMigrationPaths());
+      if (ledgerMtimeMs !== revision) continue;
+      const directPaths = sourcePaths.filter((filePath) =>
+        requested.has(path.basename(filePath, ".md"))
+      );
+      const contents = new Map<string, readonly string[]>();
+      for (const memory of await deps.readParsedMemoriesFromPaths(directPaths, 50)) {
+        const id = memory.frontmatter.id;
+        if (requested.has(id) && !contents.has(id)) {
+          contents.set(id, deps.storedContentIdentityCandidates(memory.content));
+        }
+      }
+      // Re-check after the direct-path read too: a concurrent drop during
+      // that await must not let this call publish a fill built from the
+      // stale snapshot into the freshly cleared slot (issue #2367 round 3).
+      if (ledgerMtimeMs !== revision) continue;
+      if (contents.size === requested.size) return contents;
+      const fill = (sourceContentsByIdPromise ??= (async () => {
+        const allContents = new Map<string, readonly string[]>();
+        for (const memory of await deps.readParsedMemoriesFromPaths(sourcePaths, 50)) {
+          const id = memory.frontmatter.id;
+          if (!allContents.has(id)) {
+            allContents.set(id, deps.storedContentIdentityCandidates(memory.content));
+          }
+        }
+        return allContents;
+      })());
+      const allContents = await fill;
+      if (ledgerMtimeMs !== revision) continue;
+      for (const id of requested) {
+        const content = allContents.get(id);
+        if (content !== undefined) contents.set(id, content);
+      }
+      return contents;
+    }
+  };
+}

package/src/storage.ts

@@ -26,6 +26,7 @@ import { resolveSafeStoragePath } from "./storage-paths.js";
 import { log } from "./logger.js";
 import { createMemorySnapshot } from "./memory-snapshot.js";
 import { assertMemoryFrontmatterId, warnProjectionFallback } from "./storage-guards.js";
+import { createTombstoneMigrationSourceContents } from "./storage/tombstone-migration-sources.js";
 import { MemoryReadStore, readWindowedMemories, type WindowedMemoryReadOptions, type WindowedMemoryReadResult } from "./storage/memory-read-store.js";
 import { hasSupersessionAudit } from "./storage/supersession-audit.js";
 import { runCommittedInvalidation } from "./storage/committed-invalidation.js";
@@ -3016,46 +3017,24 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   }
 
   private buildTombstoneStore(): TombstoneStore {
-    let sourcePathsPromise: Promise<string[]> | undefined;
-    let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
     const options: TombstoneStoreOptions = {
       enabled: this.tombstonesConfig.enabled,
       semanticMatch: this.tombstonesConfig.semanticMatch,
       semanticThreshold: this.tombstonesConfig.semanticThreshold,
       hashContent: ContentHashIndex.computeHash,
       normalizeText: ContentHashIndex.normalizeContent,
-      sourceContentsForMemoryIds: async (sourceMemoryIds) => {
-        const requested = new Set(sourceMemoryIds);
-        const sourcePaths = await (sourcePathsPromise ??=
-          this.memoryReadStore.collectTombstoneMigrationPaths());
-        const directPaths = sourcePaths.filter((filePath) =>
-          requested.has(path.basename(filePath, ".md"))
-        );
-        const contents = new Map<string, readonly string[]>();
-        for (const memory of await this.readParsedMemoriesFromPaths(directPaths, 50)) {
-          const id = memory.frontmatter.id;
-          if (requested.has(id) && !contents.has(id)) {
-            contents.set(id, this.storedContentIdentityCandidates(memory.content));
-          }
-        }
-        if (contents.size === requested.size) return contents;
-        sourceContentsByIdPromise ??= (async () => {
-          const allContents = new Map<string, readonly string[]>();
-          for (const memory of await this.readParsedMemoriesFromPaths(sourcePaths, 50)) {
-            const id = memory.frontmatter.id;
-            if (!allContents.has(id)) {
-              allContents.set(id, this.storedContentIdentityCandidates(memory.content));
-            }
-          }
-          return allContents;
-        })();
-        const allContents = await sourceContentsByIdPromise;
-        for (const id of requested) {
-          const content = allContents.get(id);
-          if (content !== undefined) contents.set(id, content);
-        }
-        return contents;
-      },
+      // Corpus-path + source-content caches are scoped to one ledger revision
+      // so a peer-appended legacy row is verified on the next staleness reload
+      // (issue #2367). Implementation lives in storage/tombstone-migration-sources.ts.
+      sourceContentsForMemoryIds: createTombstoneMigrationSourceContents({
+        tombstonesPath: () => this.tombstonesPath,
+        collectTombstoneMigrationPaths: () =>
+          this.memoryReadStore.collectTombstoneMigrationPaths(),
+        readParsedMemoriesFromPaths: (filePaths, batchSize) =>
+          this.readParsedMemoriesFromPaths(filePaths, batchSize),
+        storedContentIdentityCandidates: (content) =>
+          this.storedContentIdentityCandidates(content),
+      }),
     };
     const io: TombstoneFileIo = {
       read: (filePath) => this.readStorageSecureFile(filePath),

package/src/types.ts

@@ -631,7 +631,6 @@ export interface DependencyPropagationConfig {
   dryRun: boolean;
 }
 
-
 export interface HeartbeatConfig {
   enabled: boolean;
   journalPath: string;
@@ -1408,6 +1407,7 @@ export interface PluginConfig extends BoundedJsonlStateConfig, SecurityConfig {
   harmonicRetrievalEnabled: boolean;
   abstractionAnchorsEnabled: boolean;
   abstractionNodeStoreDir: string;
+  episodicContextEnabled: boolean;
   // Episodic/semantic split foundation
   verifiedRecallEnabled: boolean;
   semanticRulePromotionEnabled: boolean;
@@ -1870,6 +1870,8 @@ export interface PluginConfig extends BoundedJsonlStateConfig, SecurityConfig {
   oramaEnabled: boolean;
   oramaDbPath?: string;
   oramaEmbeddingDimension?: number;
+  /** Segment space-free scripts (CJK/Thai) in the Orama lexical index (issue #2187). Default `true`. */
+  oramaCjkSegmentation?: boolean;
 
   // QMD daemon mode
   qmdDaemonEnabled: boolean;
@@ -2908,12 +2910,10 @@ export interface MemoryFrontmatter extends SourceConnectorProvenance, OriginMeta
   /** Last time this memory was accessed (ISO 8601) */
   lastAccessed?: string;
   // Memory Worth counters (issue #560)
-  //
   // Per-fact outcome counters used to derive a dynamic utility score —
   // `p(success | retrieved)` — as a complement to the static `importance`
   // field. Absent on legacy memories written before #560; readers must treat
   // `undefined` as zero observations (uniform Beta(1,1) prior).
-  //
   // Both values must be non-negative integers on write. PR 1 wires only the
   // schema + storage round-trip — no increments, scoring, or filtering yet.
   /** Number of sessions where this memory was retrieved and the outcome was judged a success. */

package/src/user-message-cleaning.ts

@@ -0,0 +1,139 @@
+/**
+ * Host-agnostic user-message cleaning (issue #2331).
+ *
+ * Owns the generic part of the cleaner the OpenClaw wiring has always used
+ * (`src/user-message-cleaning.ts` in the root host bundle): stripping the
+ * structured host-injected memory wrappers and the leading memory-context
+ * preamble from a user message. The host wrapper keeps only the
+ * platform-header pattern (channel envelopes are an OpenClaw concept) and
+ * delegates here, so core consumers — the episodic-context recall section
+ * re-rendering archived user turns — share ONE implementation instead of a
+ * second cleaner drifting beside it.
+ */
+
+/** Never-matching header pattern for callers with no platform envelope. */
+const NO_PLATFORM_HEADER = /$a/;
+
+const WRAPPER_OPEN = "<supermemory-context";
+const WRAPPER_CLOSE = "</supermemory-context>";
+const WRAPPER_MAX_ATTRIBUTE_CHARS = 256;
+
+function isWhitespaceCode(code: number): boolean {
+  return code === 32 || (code >= 9 && code <= 13);
+}
+
+/**
+ * Remove `<supermemory-context …>…</supermemory-context>` wrappers. A
+ * single left-to-right scan with string searches — no regex on uncontrolled
+ * input — so adversarial repeats of the tag literal stay linear (CodeQL
+ * redos rule, issue #2331 review). Malformed or oversized tag heads are
+ * treated as plain text, and an unterminated wrapper preserves the rest.
+ */
+function stripMemoryWrappers(content: string): string {
+  let out = "";
+  let cursor = 0;
+  for (;;) {
+    const open = content.indexOf(WRAPPER_OPEN, cursor);
+    if (open === -1) return out + content.slice(cursor);
+    const headEnd = content.indexOf(">", open);
+    if (headEnd === -1) return out + content.slice(cursor);
+    const head = content.slice(open + WRAPPER_OPEN.length, headEnd);
+    const headIsPlain =
+      head.length > 0 &&
+      (head.length > WRAPPER_MAX_ATTRIBUTE_CHARS + 1 || headIsNotAttributeRun(head));
+    if (head !== "" && headIsPlain) {
+      // Not a wrapper open tag: keep the literal and continue after it.
+      out += content.slice(cursor, open + WRAPPER_OPEN.length);
+      cursor = open + WRAPPER_OPEN.length;
+      continue;
+    }
+    const close = content.indexOf(WRAPPER_CLOSE, headEnd);
+    if (close === -1) return out + content.slice(cursor);
+    let after = close + WRAPPER_CLOSE.length;
+    while (after < content.length && isWhitespaceCode(content.charCodeAt(after))) {
+      after += 1;
+    }
+    // Keep everything before the wrapper; only the wrapper (plus its
+    // trailing whitespace) is removed.
+    out += content.slice(cursor, open);
+    cursor = after;
+  }
+}
+
+/**
+ * Remove a trailing `[message_id: …]` marker (with surrounding whitespace)
+ * using a single right-anchored string scan — no regex on uncontrolled
+ * input (CodeQL redos rule). Returns null when no marker terminates the
+ * content, leaving the caller's string untouched.
+ */
+function stripTrailingMessageId(content: string): string | null {
+  const marker = "[message_id:";
+  const start = content.lastIndexOf(marker);
+  if (start === -1) return null;
+  let index = start + marker.length;
+  while (index < content.length && isWhitespaceCode(content.charCodeAt(index))) {
+    index += 1;
+  }
+  const idStart = index;
+  while (index < content.length && content.charCodeAt(index) !== 93 /* ] */) {
+    index += 1;
+  }
+  if (index === idStart || index === content.length) return null;
+  index += 1; // past the closing bracket
+  while (index < content.length && isWhitespaceCode(content.charCodeAt(index))) {
+    index += 1;
+  }
+  if (index !== content.length) return null;
+  let boundary = start;
+  while (boundary > 0 && isWhitespaceCode(content.charCodeAt(boundary - 1))) {
+    boundary -= 1;
+  }
+  return content.slice(0, boundary);
+}
+
+/** True when the tag-head text cannot be a whitespace-led attribute run. */
+function headIsNotAttributeRun(head: string): boolean {
+  if (head.charCodeAt(0) !== 32 && !(head.charCodeAt(0) >= 9 && head.charCodeAt(0) <= 13)) {
+    return true;
+  }
+  for (let index = 0; index < head.length; index += 1) {
+    const code = head.charCodeAt(index);
+    if (code === 60 || code === 62) return true;
+  }
+  return false;
+}
+
+export function cleanUserMessageWithPattern(
+  content: string,
+  platformHeaderPattern: RegExp,
+): string {
+  let cleaned = stripMemoryWrappers(content);
+  const platformHeader = cleaned.match(platformHeaderPattern);
+  const hasPlatformHeader = platformHeader !== null;
+  if (platformHeader) {
+    cleaned = cleaned.slice(platformHeader[0].length);
+  }
+
+  // Remove markdown memory context only when it is a leading preamble. If a
+  // user writes a section with this title later in their message, preserve it.
+  cleaned = cleaned.replace(
+    /^\s*## Memory Context \((?:Engram|Remnic)\)[\s\S]*?(?=\n## |\n$)/i,
+    "",
+  );
+
+  if (hasPlatformHeader) {
+    cleaned = stripTrailingMessageId(cleaned) ?? cleaned;
+  }
+  return cleaned.trim();
+}
+
+/**
+ * Clean an archived user turn for re-injection (issue #2331): strips the
+ * memory wrappers this system itself injects, so an archived echo of a
+ * previous recall output cannot re-enter the context through the
+ * episodic-context section. No platform envelope stripping — core has no
+ * host channel-envelope concept.
+ */
+export function cleanArchivedUserMessage(content: string): string {
+  return cleanUserMessageWithPattern(content, NO_PLATFORM_HEADER);
+}

package/src/whitespace.test.ts

@@ -0,0 +1,178 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  collapseWhitespace,
+  displayWidth,
+  graphemeUnits,
+  padEndDisplay,
+  truncateCodePointSafe,
+  truncateGraphemeSafe,
+} from "./whitespace.js";
+
+function hasLoneSurrogate(value: string): boolean {
+  for (let index = 0; index < value.length; index += 1) {
+    const code = value.charCodeAt(index);
+    const isHigh = code >= 0xd800 && code <= 0xdbff;
+    const isLow = code >= 0xdc00 && code <= 0xdfff;
+    if (isHigh) {
+      const next = value.charCodeAt(index + 1);
+      if (next < 0xdc00 || next > 0xdfff) return true;
+      index += 1;
+    } else if (isLow) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function isGraphemePrefix(prefix: string, value: string): boolean {
+  let accumulated = "";
+  for (const unit of graphemeUnits(value)) {
+    if (accumulated === prefix) return true;
+    accumulated += unit;
+  }
+  return accumulated === prefix;
+}
+
+// Deterministic pseudo-random source so failures replay exactly.
+function lcg(seed: number): () => number {
+  let state = seed >>> 0;
+  return () => {
+    state = (state * 1664525 + 1013904223) >>> 0;
+    return state / 0x1_0000_0000;
+  };
+}
+
+// Every entry is one or more whole grapheme clusters covering the cases named
+// in issue #2195: astral emoji, ZWJ sequences, skin-tone modifiers,
+// regional-indicator flags, Hangul (precomposed and jamo), combining marks,
+// and astral kanji.
+const GRAPHEME_PIECES = [
+  "a",
+  "b",
+  " ",
+  "7",
+  "😀",
+  "👨‍👩‍👧‍👦",
+  "👍🏽",
+  "🇯🇵",
+  "🇺🇸",
+  "🇰🇷",
+  "한",
+  "한",
+  "e\u0301",
+  "k\u0331\u0301",
+  "क़",
+  "𠮷",
+  "日",
+  "本",
+  "語",
+] as const;
+
+test("truncateGraphemeSafe: property - valid Unicode, grapheme-aligned, within budget", () => {
+  const random = lcg(20260816);
+  for (let iteration = 0; iteration < 500; iteration += 1) {
+    let input = "";
+    const pieceCount = 1 + Math.floor(random() * 12);
+    for (let piece = 0; piece < pieceCount; piece += 1) {
+      input += GRAPHEME_PIECES[Math.floor(random() * GRAPHEME_PIECES.length)]!;
+    }
+    const maxChars = Math.floor(random() * (input.length + 2));
+    const output = truncateGraphemeSafe(input, maxChars);
+    assert.ok(
+      output.length <= maxChars,
+      `budget exceeded for ${JSON.stringify(input)} @ ${maxChars}: ${JSON.stringify(output)}`,
+    );
+    assert.ok(
+      !hasLoneSurrogate(output),
+      `lone surrogate for ${JSON.stringify(input)} @ ${maxChars}: ${JSON.stringify(output)}`,
+    );
+    assert.ok(
+      isGraphemePrefix(output, input),
+      `split grapheme for ${JSON.stringify(input)} @ ${maxChars}: ${JSON.stringify(output)}`,
+    );
+    if (input.length <= maxChars) {
+      assert.equal(output, input, `fits must pass through @ ${maxChars}`);
+    }
+  }
+});
+
+test("truncateGraphemeSafe: emoji, ZWJ, flags, jamo, and combining marks stay whole", () => {
+  assert.equal(truncateGraphemeSafe("hello world", 5), "hello");
+  assert.equal(truncateGraphemeSafe("hello", 50), "hello");
+  assert.equal(truncateGraphemeSafe("hello", 0), "");
+  assert.equal(truncateGraphemeSafe("👍👍", 2), "👍");
+  assert.equal(truncateGraphemeSafe("👍", 1), "", "a split emoji is dropped, never half-cut");
+  assert.equal(truncateGraphemeSafe("a👍b", 2), "a");
+  assert.equal(truncateGraphemeSafe("🇯🇵🇺🇸", 4), "🇯🇵");
+  assert.equal(truncateGraphemeSafe("🇯🇵🇺🇸", 3), "", "a split flag is dropped, never half-cut");
+  const family = "👨‍👩‍👧‍👦";
+  assert.equal(truncateGraphemeSafe(`${family}!`, family.length + 1), `${family}!`);
+  assert.equal(truncateGraphemeSafe(`${family}!`, family.length), family);
+  assert.equal(truncateGraphemeSafe(family, family.length - 1), "", "ZWJ sequence is atomic");
+  assert.equal(truncateGraphemeSafe("한글", 3), "한", "jamo run is one cluster");
+  assert.equal(truncateGraphemeSafe("e\u0301x", 2), "e\u0301", "base plus mark is one cluster");
+  assert.equal(truncateGraphemeSafe("e\u0301x", 1), "");
+});
+
+test("truncateCodePointSafe and collapseWhitespace keep prior behavior", () => {
+  assert.equal(collapseWhitespace(" a \n b "), "a b");
+  assert.equal(truncateCodePointSafe("abcd", 2), "ab");
+  assert.equal(truncateCodePointSafe("ab", 9), "ab");
+  assert.equal(truncateCodePointSafe("abc", 0), "");
+});
+
+test("displayWidth: CJK wide, fullwidth, halfwidth, marks, and emoji", () => {
+  assert.equal(displayWidth(""), 0);
+  assert.equal(displayWidth("abc"), 3);
+  assert.equal(displayWidth("日本語"), 6);
+  assert.equal(displayWidth("ハングル"), 8);
+  assert.equal(displayWidth("한국어"), 6);
+  assert.equal(displayWidth("ｱ"), 1, "halfwidth katakana is narrow");
+  assert.equal(displayWidth("ｆｕｌｌ"), 8, "fullwidth ASCII is wide");
+  assert.equal(displayWidth("e\u0301"), 1, "combining mark is zero width");
+  assert.equal(displayWidth("😀"), 2);
+  assert.equal(displayWidth("中日 memo"), 9, "2+2+1+4");
+});
+
+test("displayWidth: emoji sequences measure as one two-cell grapheme", () => {
+  assert.equal(displayWidth("👨‍👩‍👧‍👦"), 2, "ZWJ family emoji is one cluster");
+  assert.equal(displayWidth("🇯🇵"), 2, "regional-indicator flag is one cluster");
+  assert.equal(displayWidth("👍🏽"), 2, "skin-tone modifier joins the cluster");
+  assert.equal(displayWidth("❤️"), 2, "VS16 requests emoji presentation");
+  assert.equal(displayWidth("1️⃣"), 2, "keycap sequence is one cluster");
+  assert.equal(displayWidth("👨‍👩‍👧‍👦 文"), 5, "clusters compose: 2 + 1 + 2");
+  assert.equal(
+    padEndDisplay("👨‍👩‍👧‍👦", 6),
+    "👨‍👩‍👧‍👦" + " ".repeat(4),
+    "emoji rows pad by display width, not code units",
+  );
+});
+
+test("padEndDisplay: CJK rows align to one computed width", () => {
+  assert.equal(padEndDisplay("abc", 6), "abc   ");
+  assert.equal(padEndDisplay("abc", 2), "abc", "narrow target never truncates");
+  assert.equal(padEndDisplay("日本", 10), "日本" + " ".repeat(6));
+  const rows = ["機械学習", "nlp", "한국어", "x"];
+  const padded = rows.map((row) => padEndDisplay(row, 12));
+  for (const row of padded) {
+    assert.equal(displayWidth(row), 12);
+  }
+  assert.deepEqual(padded, [
+    "機械学習    ",
+    "nlp         ",
+    "한국어      ",
+    "x           ",
+  ]);
+  // Snapshot-style: the CLI topics table shape keeps its score column aligned
+  // for CJK rows, which plain padEnd misaligns.
+  const table = rows.slice(0, 3).map((term) => `  ${padEndDisplay(term, 12)} 0.123`);
+  assert.deepEqual(table, [
+    "  機械学習     0.123",
+    "  nlp          0.123",
+    "  한국어       0.123",
+  ]);
+  const tableWidth = displayWidth(table[0]!);
+  assert.ok(table.every((line) => displayWidth(line) === tableWidth));
+});