@remnic/core 9.52.2 → 9.53.0

package/src/dedup/index.test.ts

@@ -1,5 +1,5 @@
 import assert from "node:assert/strict";
-import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import test from "node:test";
@@ -14,25 +14,13 @@ test("findContradictions detects can versus cannot statements", async () => {
     await mkdir(factsDir);
     await writeFile(
       path.join(factsDir, "can.md"),
-      [
-        "---",
-        "id: mem-can",
-        "category: fact",
-        "---",
-        "The user can access production.",
-      ].join("\n"),
-      "utf8",
+      ["---", "id: mem-can", "category: fact", "---", "The user can access production."].join("\n"),
+      "utf8"
     );
     await writeFile(
       path.join(factsDir, "cannot.md"),
-      [
-        "---",
-        "id: mem-cannot",
-        "category: fact",
-        "---",
-        "The user cannot access production.",
-      ].join("\n"),
-      "utf8",
+      ["---", "id: mem-cannot", "category: fact", "---", "The user cannot access production."].join("\n"),
+      "utf8"
     );
 
     const result = findContradictions({ memoryDir, categories: ["facts"] });
@@ -58,14 +46,8 @@ test("dedup public numeric options are normalized before scanning", async () =>
     ] as const) {
       await writeFile(
         path.join(factsDir, fileName),
-        [
-          "---",
-          `id: ${id}`,
-          "category: fact",
-          "---",
-          content,
-        ].join("\n"),
-        "utf8",
+        ["---", `id: ${id}`, "category: fact", "---", content].join("\n"),
+        "utf8"
       );
     }
 
@@ -95,6 +77,51 @@ test("dedup public numeric options are normalized before scanning", async () =>
   }
 });
 
+test("dedup keeps duplicate and distinct pairs across Unicode scripts", async () => {
+  const memoryDir = await mkdtemp(path.join(tmpdir(), "remnic-dedup-unicode-"));
+
+  try {
+    const factsDir = path.join(memoryDir, "facts");
+    await mkdir(factsDir);
+    const entries = [
+      ["english-a.md", "mem-english-a", "The user prefers tea."],
+      ["english-b.md", "mem-english-b", "The user prefers tea"],
+      ["english-c.md", "mem-english-c", "The user prefers coffee."],
+      ["japanese-a.md", "mem-japanese-a", "利用者は紅茶を好む。"],
+      ["japanese-b.md", "mem-japanese-b", "利用者は紅茶を好む。"],
+      ["japanese-c.md", "mem-japanese-c", "利用者は珈琲を好む。"],
+      ["korean-a.md", "mem-korean-a", "사용자는홍차를좋아한다"],
+      ["korean-b.md", "mem-korean-b", "사용자는홍차를좋아한다"],
+      ["korean-c.md", "mem-korean-c", "사용자는커피를좋아한다"],
+      ["korean-punctuation.md", "mem-korean-punctuation", "홍차・커피"],
+      ["korean-hyphen.md", "mem-korean-hyphen", "홍차-커피"],
+      ["korean-joined.md", "mem-korean-joined", "홍차커피"],
+      ["japanese-boundary-a.md", "mem-japanese-boundary-a", "甲・乙丙"],
+      ["japanese-boundary-b.md", "mem-japanese-boundary-b", "甲乙・丙"],
+    ] as const;
+
+    for (const [fileName, id, content] of entries) {
+      await writeFile(
+        path.join(factsDir, fileName),
+        ["---", `id: ${id}`, "category: fact", "---", content].join("\n"),
+        "utf8"
+      );
+    }
+
+    const result = findDuplicates({ memoryDir, categories: ["facts"] });
+
+    assert.equal(result.scanned, entries.length);
+    assert.equal(result.duplicates.length, 3);
+    assert.deepEqual(result.duplicates.map(({ left, right }) => [left.id, right.id]).sort(), [
+      ["mem-english-a", "mem-english-b"],
+      ["mem-japanese-a", "mem-japanese-b"],
+      ["mem-korean-a", "mem-korean-b"],
+    ]);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("dedup scans reject symlinked category directories outside memoryDir", async (t) => {
   if (process.platform === "win32") {
     t.skip("directory symlink setup is platform-specific");
@@ -107,24 +134,15 @@ test("dedup scans reject symlinked category directories outside memoryDir", asyn
   try {
     await writeFile(
       path.join(outsideDir, "outside.md"),
-      [
-        "---",
-        "id: outside",
-        "category: fact",
-        "---",
-        "Outside memory should not be scanned.",
-      ].join("\n"),
-      "utf8",
+      ["---", "id: outside", "category: fact", "---", "Outside memory should not be scanned."].join("\n"),
+      "utf8"
     );
     await symlink(outsideDir, path.join(memoryDir, "facts"), "dir");
 
-    assert.throws(
-      () => findDuplicates({ memoryDir, categories: ["facts"] }),
-      /symlinked memory category directory/,
-    );
+    assert.throws(() => findDuplicates({ memoryDir, categories: ["facts"] }), /symlinked memory category directory/);
     assert.throws(
       () => findContradictions({ memoryDir, categories: ["facts"] }),
-      /symlinked memory category directory/,
+      /symlinked memory category directory/
     );
   } finally {
     await rm(memoryDir, { recursive: true, force: true });

package/src/dedup/index.ts

@@ -5,9 +5,10 @@
  * against existing memories. Can be used standalone or via curation.
  */
 
+import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import crypto from "node:crypto";
+import { normalizeRecallTokenSet } from "../recall-tokenization.js";
 import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
 
 // ── Types ────────────────────────────────────────────────────────────────────
@@ -89,20 +90,11 @@ const DEFAULT_DEDUP_THRESHOLD = 0.85;
 const DEFAULT_MAX_LOAD = 10000;
 
 function normalizeThreshold(value: number | undefined, defaultValue: number): number {
-  return typeof value === "number" &&
-    Number.isFinite(value) &&
-    value >= 0 &&
-    value <= 1
-    ? value
-    : defaultValue;
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1 ? value : defaultValue;
 }
 
 function normalizeMaxLoad(value: number | undefined, defaultValue: number): number {
-  return typeof value === "number" &&
-    Number.isInteger(value) &&
-    value >= 0
-    ? value
-    : defaultValue;
+  return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : defaultValue;
 }
 
 export function findDuplicates(options: DedupOptions): DedupResult {
@@ -163,26 +155,25 @@ export function findContradictions(options: ContradictionOptions): Contradiction
 // ── Similarity computation ───────────────────────────────────────────────────
 
 function computeSimilarity(a: string, b: string): number {
-  // Normalize
   const normA = normalize(a);
   const normB = normalize(b);
+  const punctuationBoundariesA = embeddedPunctuationSignature(a);
+  const punctuationBoundariesB = embeddedPunctuationSignature(b);
+
+  // Compare punctuation and positions before normalization's exact/hash paths.
+  if (punctuationBoundariesA !== punctuationBoundariesB) return 0;
 
-  // Exact match
   if (normA === normB) return 1;
 
-  // Hash-based exact match
   if (hashContent(normA) === hashContent(normB)) return 0.99;
-
-  // Substring containment
   if (normA.length > 50 && normB.length > 50) {
     if (normA.includes(normB.slice(0, 40)) || normB.includes(normA.slice(0, 40))) {
       return 0.9;
     }
   }
 
-  // Word overlap (Jaccard)
-  const wordsA = new Set(normA.split(/\s+/));
-  const wordsB = new Set(normB.split(/\s+/));
+  const wordsA = tokenizeForSimilarity(normA);
+  const wordsB = tokenizeForSimilarity(normB);
   const intersection = new Set([...wordsA].filter((w) => wordsB.has(w)));
   const union = new Set([...wordsA, ...wordsB]);
 
@@ -190,25 +181,89 @@ function computeSimilarity(a: string, b: string): number {
   return intersection.size / union.size;
 }
 
+function embeddedPunctuationSignature(text: string): string {
+  const chars = [...text.normalize("NFC")];
+  const significant = /[\p{L}\p{M}\p{N}]/u;
+  const ignored = /[\s']/u;
+  const boundaries: string[] = [];
+  let position = 0;
+  for (let index = 0; index < chars.length;) {
+    const char = chars[index];
+    if (char === undefined) break;
+    if (significant.test(char)) {
+      position++;
+      index++;
+      continue;
+    }
+    if (ignored.test(char)) {
+      index++;
+      continue;
+    }
+    const start = index;
+    while (index < chars.length) {
+      const next = chars[index];
+      if (next === undefined || significant.test(next) || ignored.test(next)) break;
+      index++;
+    }
+    const before = chars[start - 1];
+    const after = chars[index];
+    if (before && after && significant.test(before) && significant.test(after)) {
+      boundaries.push(`${position}:${chars.slice(start, index).join("")}`);
+    }
+  }
+  return boundaries.join("\u0000");
+}
+
 function normalize(text: string): string {
   return text
+    .normalize("NFC")
     .toLowerCase()
-    .replace(/[^\w\s']/g, "")
+    .replace(/[^\p{ASCII}\p{L}\p{M}\p{N}\s']/gu, " ")
+    .replace(/[^\p{L}\p{M}\p{N}\s']/gu, "")
     .replace(/\s+/g, " ")
     .trim();
 }
 
+function tokenizeForSimilarity(text: string): Set<string> {
+  if (/\p{Script=Hangul}/u.test(text)) {
+    const tokens = normalizeRecallTokenSet(text);
+    for (const segment of text.match(/\p{Script=Hangul}+/gu) ?? []) {
+      const chars = [...segment];
+      for (const size of [1, 2, 3, 4]) {
+        for (let index = 0; index <= chars.length - size; index += 1) {
+          tokens.add(chars.slice(index, index + size).join(""));
+        }
+      }
+    }
+    return tokens;
+  }
+  if (/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(text)) {
+    return normalizeRecallTokenSet(text);
+  }
+  return text.length === 0 ? new Set() : new Set(text.split(/\s+/));
+}
+
 // ── Contradiction detection ──────────────────────────────────────────────────
 
 const NEGATION_WORDS = new Set([
-  "not", "don't", "doesn't", "isn't", "aren't", "won't", "can't", "cannot",
-  "never", "no", "none", "neither", "nor", "nothing", "nowhere",
+  "not",
+  "don't",
+  "doesn't",
+  "isn't",
+  "aren't",
+  "won't",
+  "can't",
+  "cannot",
+  "never",
+  "no",
+  "none",
+  "neither",
+  "nor",
+  "nothing",
+  "nowhere",
 ]);
 
-function detectContradiction(
-  a: MemoryEntry,
-  b: MemoryEntry,
-): ContradictionPair | null {
+function detectContradiction(a: MemoryEntry, b: MemoryEntry): ContradictionPair | null {
   const normA = normalize(a.content);
   const normB = normalize(b.content);
 
@@ -271,11 +326,7 @@ function stripNegation(text: string): string {
 
 // ── Memory loading ───────────────────────────────────────────────────────────
 
-function loadMemories(
-  memoryDir: string,
-  categories?: string[],
-  maxLoad = 10000,
-): MemoryEntry[] {
+function loadMemories(memoryDir: string, categories?: string[], maxLoad = 10000): MemoryEntry[] {
   const result: MemoryEntry[] = [];
   const allCategories = categories ?? ALL_CATEGORY_DIRS;
   if (!fs.existsSync(memoryDir)) return result;
@@ -322,11 +373,7 @@ function hashContent(content: string): string {
   return crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
 }
 
-function readFileSafe(
-  filePath: string,
-  memoryRootReal: string,
-  categoryRootReal: string,
-): string | null {
+function readFileSafe(filePath: string, memoryRootReal: string, categoryRootReal: string): string | null {
   try {
     const fileStat = fs.lstatSync(filePath);
     if (fileStat.isSymbolicLink()) {
@@ -373,7 +420,7 @@ function walkMdFiles(
   dir: string,
   memoryRootReal: string,
   categoryRootReal: string,
-  callback: (filePath: string) => void,
+  callback: (filePath: string) => void
 ): void {
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     const fullPath = path.join(dir, entry.name);