@remnic/core 9.62.0 → 9.62.2

package/src/orchestration/recall-section-coordinator.ts

@@ -19,6 +19,7 @@
 
 import type { LastRecallBudgetSummary } from "../recall-state.js";
 import { estimateTokenCount } from "../token-estimate.js";
+import { graphemeUnits, truncateGraphemeSafe } from "../whitespace.js";
 import type { PluginConfig, RecallSectionConfig } from "../types.js";
 
 export interface RecallSectionAppendOptions {
@@ -145,7 +146,7 @@ export class RecallSectionCoordinator {
       finalContent =
         sectionId === "profile"
           ? this.truncateProfileToBoundary(finalContent, maxChars)
-          : `${finalContent.slice(0, maxChars)}\n\n...(trimmed)\n`;
+          : `${truncateGraphemeSafe(finalContent, maxChars)}\n\n...(trimmed)\n`;
     }
     if (finalContent.length === 0) return false;
     const existing = sectionBuckets.get(sectionId) ?? [];
@@ -174,11 +175,16 @@ export class RecallSectionCoordinator {
     const contentLimit = maxChars - suffix.length;
     if (contentLimit <= 0) {
       const boundary = content.lastIndexOf("\n", maxChars);
-      return content.slice(0, boundary > 0 ? boundary : maxChars).trimEnd();
+      const clipped = boundary > 0
+        ? content.slice(0, boundary)
+        : truncateGraphemeSafe(content, maxChars);
+      return clipped.trimEnd();
     }
     const boundary = content.lastIndexOf("\n", contentLimit);
-    const end = boundary > 0 ? boundary : contentLimit;
-    return `${content.slice(0, end).trimEnd()}${suffix}`;
+    const clipped = boundary > 0
+      ? content.slice(0, boundary)
+      : truncateGraphemeSafe(content, contentLimit);
+    return `${clipped.trimEnd()}${suffix}`;
   }
   truncateRecallSectionToBudget(
     content: string,
@@ -191,20 +197,20 @@ export class RecallSectionCoordinator {
       content.length <= maxChars
         ? content
         : maxChars <= suffix.length
-          ? content.slice(0, maxChars)
-          : `${content.slice(0, maxChars - suffix.length)}${suffix}`;
+          ? truncateGraphemeSafe(content, maxChars)
+          : `${truncateGraphemeSafe(content, maxChars - suffix.length)}${suffix}`;
     if (maxTokens === undefined || estimateTokenCount(charBounded) <= maxTokens) {
       return charBounded;
     }
     const markerFits =
       maxChars > suffix.length && estimateTokenCount(suffix) <= maxTokens;
-    const tokenSource = content.length > maxChars ? content.slice(0, maxChars) : content;
-    const codePoints = [...tokenSource];
+    const tokenSource = content.length > maxChars ? truncateGraphemeSafe(content, maxChars) : content;
+    const units = graphemeUnits(tokenSource);
     let low = 0;
-    let high = codePoints.length;
+    let high = units.length;
     while (low < high) {
       const mid = Math.ceil((low + high) / 2);
-      const prefix = codePoints.slice(0, mid).join("");
+      const prefix = units.slice(0, mid).join("");
       const candidate = markerFits ? `${prefix}${suffix}` : prefix;
       if (candidate.length <= maxChars && estimateTokenCount(candidate) <= maxTokens) {
         low = mid;
@@ -214,19 +220,19 @@ export class RecallSectionCoordinator {
     }
     if (markerFits && low === 0) {
       low = 0;
-      high = codePoints.length;
+      high = units.length;
       while (low < high) {
         const mid = Math.ceil((low + high) / 2);
-        const prefix = codePoints.slice(0, mid).join("");
+        const prefix = units.slice(0, mid).join("");
         if (prefix.length <= maxChars && estimateTokenCount(prefix) <= maxTokens) {
           low = mid;
         } else {
           high = mid - 1;
         }
       }
-      return codePoints.slice(0, low).join("");
+      return units.slice(0, low).join("");
     }
-    const prefix = codePoints.slice(0, low).join("");
+    const prefix = units.slice(0, low).join("");
     return markerFits ? `${prefix}${suffix}` : prefix;
   }
 

package/src/recall-tokenization.ts

@@ -4,7 +4,7 @@ export interface NormalizeRecallTokenOptions {
 
 const DEFAULT_RECALL_STOP_WORDS = ["the", "and", "for", "with", "from", "into", "that", "this", "why", "did"];
 
-function isUnsegmentableRecallChar(char: string): boolean {
+export function isUnsegmentableRecallChar(char: string): boolean {
   if (char === "ー" || char === "ｰ") return true;
   return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(char);
 }
@@ -24,21 +24,27 @@ function shouldKeepRecallToken(token: string, minTokenLength: number, stopWords:
   return token.length >= 2 && hasNonAsciiCodepoint && /\p{L}/u.test(token);
 }
 
-function addUnsegmentableRecallSegment(tokens: Set<string>, segment: string, stopWords: Set<string>): void {
+export function expandUnsegmentableRecallNGrams(segment: string): string[] {
   const chars = [...segment].filter((ch) => /[\p{L}\p{N}\p{M}]/u.test(ch) || isUnsegmentableRecallChar(ch));
+  const tokens: string[] = [];
   for (const ch of chars) {
-    if (!stopWords.has(ch)) tokens.add(ch);
+    tokens.push(ch);
   }
   for (const size of [2, 3, 4]) {
     if (chars.length < size) continue;
     for (let index = 0; index <= chars.length - size; index += 1) {
-      const token = chars.slice(index, index + size).join("");
-      if (!stopWords.has(token)) tokens.add(token);
+      tokens.push(chars.slice(index, index + size).join(""));
     }
   }
-  const whole = chars.join("");
-  if (whole.length > 3 && !stopWords.has(whole)) {
-    tokens.add(whole);
+  if (chars.length > 3) {
+    tokens.push(chars.join(""));
+  }
+  return tokens;
+}
+
+function addUnsegmentableRecallSegment(tokens: Set<string>, segment: string, stopWords: Set<string>): void {
+  for (const token of expandUnsegmentableRecallNGrams(segment)) {
+    if (!stopWords.has(token)) tokens.add(token);
   }
 }
 

package/src/reconcile/manifest.test.ts

@@ -50,6 +50,7 @@ test("reconcile manifest keeps file identity separate from canonical semantic id
     category: "fact",
     contentHash: semanticHash,
     normalizerVersion: 4,
+    identityResolutionVersion: 2,
     status: "active",
   });
 });
@@ -100,11 +101,52 @@ test("reconcile manifest reparses cached pre-version identities after Unicode mi
   });
 
   assert.equal(reads, 1);
+  // The café body is lossy under the legacy normalizer, so the persisted
+  // legacy hash stays primary and the recovered current identity rides along
+  // as an alias (issue #2367).
   assert.equal(
     manifest.files[0]?.memory?.contentHash,
-    ContentHashIndex.computeHash(content),
+    legacyHash,
   );
-  assert.equal("contentHashAliases" in (manifest.files[0]?.memory ?? {}), false);
+  assert.deepEqual(manifest.files[0]?.memory?.contentHashAliases, [
+    ContentHashIndex.computeHash(content),
+  ]);
+});
+
+test("reconcile manifest reparses cached identities lacking the identity-resolution version", async () => {
+  const content = "The user prefers café.";
+  const legacyHash = computeLegacyContentHash(content);
+  const serialized = memoryFile({ id: "legacy-identity-version", content, contentHash: legacyHash });
+  const file = { path: "facts/legacy-identity-version.md", sha256: fileHash(serialized) };
+  let reads = 0;
+
+  const manifest = await buildReconcileManifest({
+    files: [file],
+    parseMemory: parseFrontmatter,
+    readFile: async () => {
+      reads += 1;
+      return serialized;
+    },
+    // Same normalizerVersion, but written before identity-resolution
+    // versioning existed: the entry carries no aliases and must NOT be
+    // reused (issue #2367 review round 2).
+    cachedFiles: [{
+      ...file,
+      memory: {
+        id: "legacy-identity-version",
+        category: "fact",
+        contentHash: legacyHash,
+        normalizerVersion: 4,
+        status: "active",
+      },
+    }],
+  });
+
+  assert.equal(reads, 1);
+  assert.equal(manifest.files[0]?.memory?.identityResolutionVersion, 2);
+  assert.deepEqual(manifest.files[0]?.memory?.contentHashAliases, [
+    ContentHashIndex.computeHash(content),
+  ]);
 });
 
 test("reconcile manifest replaces a pure-CJK legacy identity with its current hash", async () => {
@@ -146,7 +188,10 @@ test("reconcile manifest recovers a raw hash source beneath citation and attribu
     readFile: async () => serialized,
   });
 
-  assert.equal(manifest.files[0]?.memory?.contentHash, ContentHashIndex.computeHash(content));
+  assert.equal(manifest.files[0]?.memory?.contentHash, computeLegacyContentHash(content));
+  assert.deepEqual(manifest.files[0]?.memory?.contentHashAliases, [
+    ContentHashIndex.computeHash(content),
+  ]);
 });
 
 test("reconcile manifest recovers raw identity beneath a configured citation", async () => {
@@ -169,7 +214,91 @@ test("reconcile manifest recovers raw identity beneath a configured citation", a
     citationTemplate,
   });
 
-  assert.equal(manifest.files[0]?.memory?.contentHash, ContentHashIndex.computeHash(content));
+  assert.equal(manifest.files[0]?.memory?.contentHash, computeLegacyContentHash(content));
+  assert.deepEqual(manifest.files[0]?.memory?.contentHashAliases, [
+    ContentHashIndex.computeHash(content),
+  ]);
+});
+
+test("reconcile manifest prefers a persisted explicit hash over a colliding legacy body hash (issue #2367)", async () => {
+  // Source `caf` with body `café`: hash("caf") numerically equals the legacy
+  // hash of the stored body, but it is the fact's explicit contentHashSource
+  // identity written by the current normalizer. The legacy candidate match
+  // must not replace it.
+  const source = "caf";
+  const body = "café";
+  const persistedHash = ContentHashIndex.computeHash(source);
+  assert.equal(persistedHash, computeLegacyContentHash(body));
+  const serialized = memoryFile({ id: "explicit-source", content: body, contentHash: persistedHash });
+  const manifest = await buildReconcileManifest({
+    files: [{ path: "facts/explicit-source.md", sha256: fileHash(serialized) }],
+    parseMemory: parseFrontmatter,
+    readFile: async () => serialized,
+  });
+
+  assert.equal(manifest.files[0]?.memory?.contentHash, persistedHash);
+  assert.notEqual(manifest.files[0]?.memory?.contentHash, ContentHashIndex.computeHash(body));
+  assert.deepEqual(manifest.files[0]?.memory?.contentHashAliases, [
+    ContentHashIndex.computeHash(body),
+  ]);
+});
+
+test("reconcile collapse matches an explicit-source replica through the persisted hash (issue #2367)", async () => {
+  const source = "caf";
+  const persistedHash = ContentHashIndex.computeHash(source);
+  const localSerialized = memoryFile({ id: "explicit-local", content: "café", contentHash: persistedHash });
+  const peerSerialized = memoryFile({ id: "explicit-peer", content: source, contentHash: persistedHash });
+  const localFile = { path: "facts/explicit-local.md", sha256: fileHash(localSerialized) };
+  const peerFile = { path: "facts/explicit-peer.md", sha256: fileHash(peerSerialized) };
+  const build = async (file: typeof localFile, raw: string) =>
+    await buildReconcileManifest({
+      files: [file],
+      parseMemory: parseFrontmatter,
+      readFile: async () => raw,
+    });
+  const local = await build(localFile, localSerialized);
+  const peer = await build(peerFile, peerSerialized);
+  const plan = planReconciliation([{ namespace: "default", local: [localFile], peer: [peerFile] }]);
+
+  const collapsed = collapseActiveFactDuplicates(
+    plan,
+    new Map([["default", local]]),
+    new Map([["default", peer]]),
+  );
+
+  assert.equal(collapsed.converged, true);
+  assert.equal(collapsed.entries.length, 1);
+  assert.equal(collapsed.entries[0]?.reason, "semantic_duplicate");
+});
+
+test("reconcile manifest does not collapse distinct pure-CJK legacy facts through the empty skeleton (issue #2367)", async () => {
+  const first = "利用者は紅茶を好む。";
+  const second = "利用者は珈琲を好む。";
+  assert.equal(computeLegacyContentHash(first), computeLegacyContentHash(second));
+  const build = async (id: string, content: string) => {
+    const serialized = memoryFile({ id, content, contentHash: computeLegacyContentHash(content) });
+    const file = { path: `facts/${id}.md`, sha256: fileHash(serialized) };
+    const manifest = await buildReconcileManifest({
+      files: [file],
+      parseMemory: parseFrontmatter,
+      readFile: async () => serialized,
+    });
+    return { file, manifest };
+  };
+  const a = await build("legacy-cjk-first", first);
+  const b = await build("legacy-cjk-second", second);
+  const plan = planReconciliation([
+    { namespace: "default", local: [a.file], peer: [b.file] },
+  ]);
+
+  const collapsed = collapseActiveFactDuplicates(
+    plan,
+    new Map([["default", a.manifest]]),
+    new Map([["default", b.manifest]]),
+  );
+
+  assert.equal(collapsed.converged, false);
+  assert.equal(collapsed.entries.length, 2);
 });
 
 test("reconcile manifest does not add a legacy alias to a current Unicode identity", async () => {

package/src/reconcile/manifest.ts

@@ -18,12 +18,30 @@ import {
 export const RECONCILE_MANIFEST_FORMAT = "remnic-reconcile-manifest";
 export const RECONCILE_MANIFEST_SCHEMA_VERSION = 1;
 const CONTENT_HASH_NORMALIZER_VERSION = 4;
+/**
+ * Version of the identity-resolution LOGIC (alias recovery, candidate
+ * ordering), independent of the content normalizer: a cached manifest entry
+ * written before an identity-resolution change carries an older value and is
+ * reparsed even when its normalizerVersion still matches (issue #2367
+ * review round 2). Bump when parsedMemoryIdentity's output shape or
+ * semantics change.
+ */
+const IDENTITY_RESOLUTION_VERSION = 2;
 
 export interface ReconcileMemoryIdentity {
   id: string;
   category: string;
   contentHash: string;
+  /**
+   * Recovered current-normalizer identities for a record whose persisted hash
+   * is ambiguous under the legacy normalizer (issue #2367). The persisted
+   * hash stays primary; these participate in duplicate bucketing so a
+   * replica carrying the current form of the same fact still collapses.
+   */
+  contentHashAliases?: readonly string[];
+  /** See IDENTITY_RESOLUTION_VERSION; absent on pre-versioning cache entries. */
   normalizerVersion?: number;
+  identityResolutionVersion?: number;
   status: MemoryStatus;
 }
 
@@ -88,17 +106,34 @@ function parsedMemoryIdentity(
   const legacyMatch = [...candidates].reverse().find(
     (content) => computeLegacyContentHash(content) === storedHash
   );
-  const contentHash = currentMatch
-    ? ContentHashIndex.computeHash(currentMatch)
-    : legacyMatch
-      ? ContentHashIndex.computeHash(legacyMatch)
-      : storedHash ?? bodyHash;
+  let contentHash: string;
+  let contentHashAliases: string[] | undefined;
+  if (currentMatch) {
+    contentHash = ContentHashIndex.computeHash(currentMatch);
+  } else if (legacyMatch && normalizeLegacyContent(legacyMatch).length > 0) {
+    // A non-empty legacy equality is ambiguous (issue #2367): the persisted
+    // hash may be an explicit contentHashSource identity that merely collides
+    // with the body's ASCII skeleton under the lossy legacy normalizer, and
+    // nothing in the record proves which it is. Keep the persisted hash as
+    // the primary identity and expose the recovered current identity as an
+    // alias so replicas carrying either form still collapse.
+    contentHash = storedHash ?? bodyHash;
+    const recovered = ContentHashIndex.computeHash(legacyMatch);
+    if (recovered !== contentHash) contentHashAliases = [recovered];
+  } else {
+    // An EMPTY legacy skeleton identifies nothing (every non-ASCII body maps
+    // to it), so preserving it would collapse unrelated records. Recovering
+    // the current identity here is repair, not replacement.
+    contentHash = legacyMatch ? ContentHashIndex.computeHash(legacyMatch) : storedHash ?? bodyHash;
+  }
 
   return {
     id: parsed.frontmatter.id,
     category: parsed.frontmatter.category,
     contentHash,
+    ...(contentHashAliases ? { contentHashAliases } : {}),
     normalizerVersion: CONTENT_HASH_NORMALIZER_VERSION,
+    identityResolutionVersion: IDENTITY_RESOLUTION_VERSION,
     status: inferMemoryStatus(parsed.frontmatter, filePath),
   };
 }
@@ -136,7 +171,8 @@ export async function buildReconcileManifest(options: BuildReconcileManifestOpti
     if (
       cached?.sha256.toLowerCase() === file.sha256.toLowerCase() &&
       (cached.memory === undefined ||
-        cached.memory.normalizerVersion === CONTENT_HASH_NORMALIZER_VERSION)
+        (cached.memory.normalizerVersion === CONTENT_HASH_NORMALIZER_VERSION &&
+          cached.memory.identityResolutionVersion === IDENTITY_RESOLUTION_VERSION))
     ) {
       files.push({ ...file, ...(cached.memory ? { memory: cached.memory } : {}) });
       continue;
@@ -168,7 +204,7 @@ function activeFactByPath(manifest: ReconcileManifest | undefined): Map<string,
 }
 
 function memoryIdentityHashes(memory: ReconcileMemoryIdentity): string[] {
-  return [memory.contentHash];
+  return [memory.contentHash, ...(memory.contentHashAliases ?? [])];
 }
 
 function contentHashRows(

package/src/search/factory.ts

@@ -79,6 +79,7 @@ function resolveNonQmdBackend(config: PluginConfig): SearchBackend | undefined {
       embedHelper,
       memoryDir: config.memoryDir,
       embeddingDimension: config.oramaEmbeddingDimension!,
+      cjkSegmentationEnabled: config.oramaCjkSegmentation !== false,
     });
   }
 

package/src/search/lancedb-backend.ts

@@ -221,12 +221,8 @@ export class LanceDbBackend implements SearchBackend {
         rows.length > 0 && rows.every((row) => row.vectorProvider === embeddingProviderIdentity),
       );
       if (isSearchAborted(execution)) return;
-      // Create FTS index on content column
-      try {
-        await table.createIndex("content", { config: this.lanceIndex.fts() });
-      } catch {
-        // FTS index creation may fail on some platforms — degrade gracefully
-      }
+      // FTS index on the content column.
+      await this.createFtsIndex(table);
       if (collection === this.collection) this.table = table;
     } catch (err) {
       log.debug(`LanceDbBackend update failed: ${err}`);
@@ -365,11 +361,7 @@ export class LanceDbBackend implements SearchBackend {
       vectorProvider: "",
     };
     const newTable = await db.createTable(collection, [emptyRow]);
-    try {
-      await newTable.createIndex("content", { config: this.lanceIndex.fts() });
-    } catch {
-      // FTS index creation may fail — degrade gracefully
-    }
+    await this.createFtsIndex(newTable);
     try {
       await newTable.delete("docid = '__placeholder__'");
     } catch {
@@ -424,11 +416,7 @@ export class LanceDbBackend implements SearchBackend {
     };
     this.table = await db.createTable(this.collection, [emptyRow]);
     // Create FTS index on content column
-    try {
-      await this.table.createIndex("content", { config: this.lanceIndex.fts() });
-    } catch {
-      // FTS index creation may fail — degrade gracefully
-    }
+    await this.createFtsIndex(this.table);
     // Remove placeholder row
     try {
       await this.table.delete("docid = '__placeholder__'");
@@ -438,6 +426,26 @@ export class LanceDbBackend implements SearchBackend {
     return this.table;
   }
 
+  /**
+   * Create the FTS index on the content column with the default tokenizer.
+   *
+   * Known limitation (issue #2187): tantivy's default tokenizer does not
+   * segment space-free scripts (CJK/Thai), so lexical recall over those
+   * scripts is near zero here. LanceDB's `baseTokenizer: "ngram"` option is
+   * not script-aware — it would n-gram English content too, degrading
+   * precision and bloating the index — so no CJK-capable configuration is
+   * available in the current @lancedb/lancedb API. Non-English lexical recall
+   * on this backend rides on the vector tier instead. See
+   * docs/search-backends.md ("Non-English content").
+   */
+  private async createFtsIndex(table: any): Promise<void> {
+    try {
+      await table.createIndex("content", { config: this.lanceIndex.fts() });
+    } catch {
+      // FTS index creation may fail on some platforms — degrade gracefully
+    }
+  }
+
   private async searchTable(
     table: any,
     query: string,