@remnic/core 9.52.2 → 9.54.0

package/src/storage.ts

@@ -13,7 +13,11 @@ import {
 } from "node:fs/promises";
 import { appendFileSync, createReadStream, mkdirSync, readFileSync, statSync } from "node:fs";
 import { createHash } from "node:crypto";
-import { normalizeContent, computeContentHash } from "./content-hash.js";
+import {
+  computeContentHash,
+  computeLegacyContentHash,
+  normalizeContent,
+} from "./content-hash.js";
 import { raceAbort } from "./abort-error.js";
 import { checkCorpusReadAbort, type CorpusReadOptions } from "./corpus-read-cancellation.js";
 import { selectArtifactMatches, type ArtifactSearchOptions } from "./artifact-search.js";
@@ -115,7 +119,6 @@ import {
   sortStructuredSectionsBySchema,
 } from "./entity-schema.js";
 import {
-  hasCitation,
   hasCitationForTemplate,
   stripCitationForTemplate,
   DEFAULT_CITATION_FORMAT,
@@ -1058,7 +1061,7 @@ const FACT_HASH_INDEX_REBUILD_RETRY_BASE_MS = 50;
 // the coding surfaces + wearable service). Imported here so internal callers
 // (snapshotBeforeWrite, snapshotForProvenance) resolve, and re-exported to
 // keep the public storage API stable for existing callers (wearables, dist).
-import { assemblePersistedBody, stripAttributesSuffix } from "./structured-attributes.js";
+import { assemblePersistedBody, storedContentIdentityCandidates, stripAttributesSuffix } from "./structured-attributes.js";
 export { stripAttributesSuffix };
 
 // `normalizeAttributePairs` moved to ./structured-attributes.ts (issue #1989
@@ -3009,22 +3012,51 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   }
 
   private buildTombstoneStore(): TombstoneStore {
+    let sourcePathsPromise: Promise<string[]> | undefined;
+    let sourceContentsByIdPromise: Promise<Map<string, readonly string[]>> | undefined;
     const options: TombstoneStoreOptions = {
       enabled: this.tombstonesConfig.enabled,
       semanticMatch: this.tombstonesConfig.semanticMatch,
       semanticThreshold: this.tombstonesConfig.semanticThreshold,
-      // Wire the SAME helpers the dedup index uses (rule 23 / checklist §13):
-      // hashContent = ContentHashIndex.computeHash, normalizeText =
-      // ContentHashIndex.normalizeContent. Importing them here (not copying)
-      // guarantees the tombstone tiers can never drift from dedup.
       hashContent: ContentHashIndex.computeHash,
       normalizeText: ContentHashIndex.normalizeContent,
+      sourceContentsForMemoryIds: async (sourceMemoryIds) => {
+        const requested = new Set(sourceMemoryIds);
+        const sourcePaths = await (sourcePathsPromise ??=
+          this.memoryReadStore.collectTombstoneMigrationPaths());
+        const directPaths = sourcePaths.filter((filePath) =>
+          requested.has(path.basename(filePath, ".md"))
+        );
+        const contents = new Map<string, readonly string[]>();
+        for (const memory of await this.readParsedMemoriesFromPaths(directPaths, 50)) {
+          const id = memory.frontmatter.id;
+          if (requested.has(id) && !contents.has(id)) {
+            contents.set(id, this.storedContentIdentityCandidates(memory.content));
+          }
+        }
+        if (contents.size === requested.size) return contents;
+        sourceContentsByIdPromise ??= (async () => {
+          const allContents = new Map<string, readonly string[]>();
+          for (const memory of await this.readParsedMemoriesFromPaths(sourcePaths, 50)) {
+            const id = memory.frontmatter.id;
+            if (!allContents.has(id)) {
+              allContents.set(id, this.storedContentIdentityCandidates(memory.content));
+            }
+          }
+          return allContents;
+        })();
+        const allContents = await sourceContentsByIdPromise;
+        for (const id of requested) {
+          const content = allContents.get(id);
+          if (content !== undefined) contents.set(id, content);
+        }
+        return contents;
+      },
     };
     const io: TombstoneFileIo = {
       read: (filePath) => this.readStorageSecureFile(filePath),
       append: (filePath, content) => this.appendStorageSecureFile(filePath, content),
       write: (filePath, content) => this.writeStorageSecureFile(filePath, content),
-      // stat lets the store own its cross-process staleness probe (#1579).
       stat: (filePath) => statSync(filePath),
     };
     return new TombstoneStore(this.tombstonesPath, this.tombstonesConfig.namespace, options, io);
@@ -3263,13 +3295,14 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     }
     this.factHashIndexAuthoritative = false;
     this.factHashIndexAuthoritativePromise = (async () => {
-      // Round 11: ALWAYS rebuild the fact-hash index from the durable corpus on
-      // first use per process — no on-disk "ready" marker is written or trusted,
-      // so a deferred write, crash, or multi-process interleave can never leave a
-      // stale index trusted. The scan AND publish run under the SAME per-index
-      // cross-process lock the reconciling saves use (rebuildUnderLock), so the
-      // rebuild can never overwrite a peer's newer lock-merged or deferred
-      // additions with an unlocked overwrite.
+      // ALWAYS rebuild the index from the durable corpus on first use per
+      // process, preserving the #2016 crash/restart invariant. Markerless
+      // legacy indexes additionally require body-derived hashes because their
+      // frontmatter.contentHash values may use the old normalizer.
+      //
+      // The scan and publish run under the SAME per-index cross-process lock
+      // used by reconcile saves, so migration cannot overwrite a peer's newer
+      // additions with an unlocked write.
       //
       // PR #2016 (findings 1-2): bounded-retry the LOCKED rebuild so transient
       // contention (a peer mid reconcile-save) clears within a short budget
@@ -3284,7 +3317,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       const baseMs = this.factHashIndexLockOptions.retryBaseMs ?? FACT_HASH_INDEX_REBUILD_RETRY_BASE_MS;
       let published = false;
       for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
-        published = await factHashIndex.rebuildUnderLock(() => this.rebuildFactHashIndexFromCorpus(factHashIndex));
+        published = await factHashIndex.rebuildUnderLock(() =>
+          this.rebuildFactHashIndexFromCorpus(factHashIndex)
+        );
         if (published || attempt === maxAttempts - 1) break;
         const wait = Math.min(baseMs * 2 ** attempt, CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS);
         await new Promise<void>((resolve) => setTimeout(resolve, wait));
@@ -3346,17 +3381,16 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       // citation-strip reconstruction below, hashing the stored persist body
       // (title + steps) exactly as buildProcedurePersistBody registered it.
       if (inferMemoryStatus(memory.frontmatter, memory.path) !== "active") continue;
-      const hash = this.corpusRegisteredHash(memory);
-      if (hash === null) {
-        // Body carries a citation from an unknown/custom template we cannot
-        // safely strip — skip rather than register a wrong hash. A
-        // false-negative miss beats a wrong entry that would permanently
-        // suppress legitimate duplicate writes (see corpusRegisteredHash).
+      const hashes = this.corpusRegisteredHashes(memory);
+      if (hashes.length === 0) {
+        // Skip bodies whose configured citation cannot be stripped safely.
         legacyRecovered++;
         continue;
       }
-      factHashIndex.addByHash(hash);
-      if (memory.frontmatter.category === "fact") this.factOnlyHashes.add(hash);
+      for (const hash of hashes) {
+        factHashIndex.addByHash(hash);
+        if (memory.frontmatter.category === "fact") this.factOnlyHashes.add(hash);
+      }
     }
     if (legacyRecovered > 0) {
       log.info(
@@ -3364,39 +3398,28 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       );
     }
   }
-  /**
-   * The content-hash the corpus rebuild registers for `memory`, or null when the
-   * body carries a citation from an unknown/custom template that cannot be
-   * safely stripped. Shared by the authoritative rebuild and the fact-only
-   * corpus confirmation (`hasFactContentHash`) so both derive the identical
-   * hash for a given stored body.
-   *
-   * Preference order:
-   *  1. frontmatter.contentHash — the raw pre-citation hash writeMemory records
-   *     for facts (issue #369 round 8); matches hasFactContentHash(rawFact).
-   *  2. Reconstruct from the stored body: strip the "[Attributes: …]" suffix
-   *     writeMemory appends for structuredAttributes (registration hashed the
-   *     raw canonical content WITHOUT it — a no-op when absent), then strip a
-   *     recognised citation and hash the bare body, or hash a citation-free
-   *     body as-is.
-   *  3. A body with a citation from an unknown/custom template → null: a
-   *     false-negative miss beats a wrong hash that would permanently suppress
-   *     legitimate duplicate writes.
-   */
-  private corpusRegisteredHash(memory: MemoryFile): string | null {
-    if (memory.frontmatter.contentHash) {
-      return memory.frontmatter.contentHash;
-    }
-    const content = stripAttributesSuffix(memory.content);
-    const stripped = stripCitationForTemplate(content, this.citationTemplate);
-    if (stripped !== content) {
-      return ContentHashIndex.computeHash(sanitizeMemoryContent(stripped).text);
-    }
-    if (!hasCitation(content)) {
-      return ContentHashIndex.computeHash(sanitizeMemoryContent(content).text);
-    }
-    return null;
+  private storedContentIdentityCandidates(content: string): string[] {
+    return storedContentIdentityCandidates(content, (value) =>
+      hasCitationForTemplate(value, this.citationTemplate) ? stripCitationForTemplate(value, this.citationTemplate) : value
+    );
+  }
+
+  /** Returns the current body identity while preserving explicit external identities. */
+  private corpusRegisteredHashes(memory: MemoryFile): string[] {
+    const persistedHash = memory.frontmatter.contentHash;
+    const candidates = this.storedContentIdentityCandidates(memory.content)
+      .map((content) => sanitizeMemoryContent(content).text);
+    const currentHashes = candidates.map((content) => ContentHashIndex.computeHash(content));
+    if (persistedHash && currentHashes.includes(persistedHash)) return [persistedHash];
+    if (!persistedHash) return [currentHashes[0] ?? ContentHashIndex.computeHash("")];
+    const legacyMatches = candidates.filter(
+      (content) => computeLegacyContentHash(content) === persistedHash
+    );
+    return legacyMatches.length > 0
+      ? [...new Set(legacyMatches.map((content) => ContentHashIndex.computeHash(content)))]
+      : [persistedHash];
   }
+
   private get questionsDir(): string {
     return path.join(this.baseDir, "questions");
   }
@@ -3787,10 +3810,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     let tombstoneBlocked = false;
     let gateRef: string | undefined;
     let statusBeforeBlock: MemoryFrontmatter["status"];
-    // Closure so the post-persist repair can RE-RUN it when a journal move
-    // changed the final entityRef: a verdict is only valid for the ref it was
-    // computed under, so reset and re-evaluate (parked mappings fall BACK to
-    // the legacy claimant; entity-independent tiers re-block in the lookup).
+    // Re-run after journal moves that change the final entityRef.
     const applyTombstoneGate = async (): Promise<void> => {
       if (tombstoneBlocked) {
         if (fm.entityRef === gateRef) return;
@@ -3799,11 +3819,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
         delete fm.blockedBy;
         delete fm.tombstoneBlockTier;
       }
-      // Status semantics (thread ObteQ: pending_review candidates included,
-      // terminal statuses respected) live in applyTombstoneResurrectionGate.
       if (category !== "fact" || factHashSourceForTombstone === null) return;
       const statusBefore = fm.status;
-      const match = await this.entityRefRepair.gate(fm, factHashSourceForTombstone, options.structuredAttributes);
+      const match = await this.entityRefRepair.gate(fm, sanitized.text, options.structuredAttributes);
       if (match) {
         gateRef = fm.entityRef;
         statusBeforeBlock = statusBefore;
@@ -3980,34 +3998,28 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     for (const memory of existing) {
       if (memory.frontmatter.category !== "fact") continue;
       if (inferMemoryStatus(memory.frontmatter, memory.path) !== "active") continue;
-      if (this.corpusRegisteredHash(memory) === targetHash) return true;
+      if (this.corpusRegisteredHashes(memory).includes(targetHash)) return true;
     }
     return false;
   }
 
-  private factContentHashForRemoval(memory: MemoryFile): string | null {
-    if (memory.frontmatter.category !== "fact") return null;
-    if (typeof memory.frontmatter.contentHash === "string" && memory.frontmatter.contentHash.length > 0) {
-      return memory.frontmatter.contentHash;
-    }
-    const configuredHashSource = stripCitationMarkersForHashRemoval(memory.content, this.citationTemplate);
-    const hashSource =
-      configuredHashSource !== memory.content
-        ? configuredHashSource
-        : stripDefaultCitationMarkersWithoutRegex(memory.content);
-    return ContentHashIndex.computeHash(sanitizeMemoryContent(hashSource).text);
+  private factContentHashesForRemoval(memory: MemoryFile): string[] {
+    if (memory.frontmatter.category !== "fact") return [];
+    return this.corpusRegisteredHashes(memory);
   }
 
   private async addActiveFactContentHash(memory: MemoryFile): Promise<void> {
     if (memory.frontmatter.category !== "fact") return;
     if (inferMemoryStatus(memory.frontmatter, memory.path) !== "active") return;
-    const hash = this.factContentHashForRemoval(memory);
-    if (!hash) return;
+    const hashes = this.factContentHashesForRemoval(memory);
+    if (hashes.length === 0) return;
 
     await this.ensureFactHashIndexAuthoritative();
     const factHashIndex = await this.getFactHashIndex();
-    factHashIndex.addByHash(hash);
-    this.factOnlyHashes.add(hash);
+    for (const hash of hashes) {
+      factHashIndex.addByHash(hash);
+      this.factOnlyHashes.add(hash);
+    }
     // PR #2016 thread SDzOP: flush through the SAME cross-process locked
     // reconcile the write/batch/rebuild paths use, never the unlocked whole-file
     // save() — an unlocked overwrite drops a concurrent extraction's appended
@@ -4045,16 +4057,19 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   private async syncFactHashIndexAfterRewrite(before: MemoryFile, after: MemoryFile): Promise<void> {
     if (before.frontmatter.category !== "fact" && after.frontmatter.category !== "fact") return;
 
-    const beforeHash = this.factContentHashForRemoval(before);
-    const afterHash = this.factContentHashForRemoval(after);
+    const beforeHashes = this.factContentHashesForRemoval(before);
+    const afterHashes = this.factContentHashesForRemoval(after);
     const beforeStatus = inferMemoryStatus(before.frontmatter, before.path);
     const afterStatus = inferMemoryStatus(after.frontmatter, after.path);
-    if (beforeHash === afterHash && beforeStatus === afterStatus) return;
+    const hashesChanged =
+      beforeHashes.length !== afterHashes.length ||
+      beforeHashes.some((hash, index) => hash !== afterHashes[index]);
+    if (!hashesChanged && beforeStatus === afterStatus) return;
 
-    if (beforeStatus === "active" && beforeHash && (beforeHash !== afterHash || afterStatus !== "active")) {
+    if (beforeStatus === "active" && beforeHashes.length > 0 && (hashesChanged || afterStatus !== "active")) {
       await this.removeFactContentHashesForMemories([before]);
     }
-    if (afterStatus === "active" && afterHash && (beforeHash !== afterHash || beforeStatus !== "active")) {
+    if (afterStatus === "active" && afterHashes.length > 0 && (hashesChanged || beforeStatus !== "active")) {
       await this.addActiveFactContentHash(after);
     }
   }
@@ -4067,11 +4082,10 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
         .map((memory) => memory.frontmatter.id)
         .filter((id): id is string => typeof id === "string" && id.length > 0)
     );
-    const removedHashes = new Map<MemoryFile, string>();
+    const removedHashes = new Set<string>();
     for (const memory of memories) {
-      const hash = this.factContentHashForRemoval(memory);
-      if (hash) {
-        removedHashes.set(memory, hash);
+      for (const hash of this.factContentHashesForRemoval(memory)) {
+        removedHashes.add(hash);
       }
     }
     if (removedHashes.size === 0) return;
@@ -4082,13 +4096,12 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       if (memory.frontmatter.category !== "fact") continue;
       if (removedIds.has(memory.frontmatter.id)) continue;
       if (inferMemoryStatus(memory.frontmatter, memory.path) !== "active") continue;
-      const hash = this.factContentHashForRemoval(memory);
-      if (hash) {
+      for (const hash of this.factContentHashesForRemoval(memory)) {
         remainingActiveHashes.add(hash);
       }
     }
 
-    for (const hash of removedHashes.values()) {
+    for (const hash of removedHashes) {
       if (!remainingActiveHashes.has(hash)) {
         factHashIndex.removeByHash(hash);
         this.factOnlyHashes.delete(hash);
@@ -4117,8 +4130,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
    */
   removeFactOnlyHashForMemory(memory: MemoryFile): void {
     if (memory.frontmatter.category !== "fact") return;
-    const hash = this.factContentHashForRemoval(memory);
-    if (hash) this.factOnlyHashes.delete(hash);
+    for (const hash of this.factContentHashesForRemoval(memory)) {
+      this.factOnlyHashes.delete(hash);
+    }
   }
 
   async isFactContentHashAuthoritative(): Promise<boolean> {

package/src/structured-attributes.ts

@@ -39,6 +39,17 @@ export function stripAttributesSuffix(content: string): string {
   if (inner.includes("]") || inner.includes("\n")) return content.trim();
   return trimmed.slice(0, markerIndex).trim();
 }
+
+/** Return stored-body identity forms, from raw semantic content to full body. */
+export function storedContentIdentityCandidates(
+  content: string,
+  stripCitation: (value: string) => string,
+): string[] {
+  const fullBody = content.trim();
+  const bodyWithoutAttributes = stripAttributesSuffix(fullBody);
+  const rawBody = stripCitation(bodyWithoutAttributes);
+  return [...new Set([rawBody, bodyWithoutAttributes, fullBody])];
+}
 // ---------------------------------------------------------------------------
 // Persisted-body assembly (issue #1989 PR2)
 // ---------------------------------------------------------------------------

package/src/wearables/cleanup.test.ts

@@ -188,3 +188,74 @@ test("preserveUtteranceBoundaries default (off) still collapses generic labels",
   assert.equal(result.conversation.segments.length, 1);
   assert.equal(result.conversation.segments[0].text, "First. Second.");
 });
+
+test("strips Japanese filler tokens that carry no word spaces", () => {
+  assert.equal(
+    stripFillerTokens("えーと、明日の会議は三時です"),
+    "明日の会議は三時です",
+  );
+  assert.equal(stripFillerTokens("あのー来週で"), "来週で");
+});
+
+test("keeps a meaning-bearing Japanese demonstrative", () => {
+  assert.equal(stripFillerTokens("あの会議は長かった"), "あの会議は長かった");
+});
+
+test("strips Korean and Cyrillic filler tokens as whole tokens", () => {
+  assert.equal(stripFillerTokens("음 회의는 세시입니다"), "회의는 세시입니다");
+  assert.equal(stripFillerTokens("эм встреча в три"), "встреча в три");
+});
+
+test("never strips a filler lookalike inside a longer token", () => {
+  assert.equal(stripFillerTokens("the umbrella is uhh wet"), "the umbrella is wet");
+  // "음" also occurs inside ordinary Korean words; only the whole token goes.
+  assert.equal(stripFillerTokens("다음 회의"), "다음 회의");
+});
+
+test("operator filler tokens apply to any script", () => {
+  assert.equal(stripFillerTokens("bueno vamos a empezar", ["bueno"]), "vamos a empezar");
+  assert.equal(stripFillerTokens("那个我们开始吧", ["那个"]), "我们开始吧");
+});
+
+test("cleanConversation strips configured filler tokens", () => {
+  const result = cleanConversation(
+    conversation([
+      { speakerKey: "a", text: "えーと、予算を確認します" },
+      { speakerKey: "b", text: "bueno, de acuerdo" },
+    ]),
+    ALL_ON,
+    ["bueno"],
+  );
+  assert.deepEqual(
+    result.conversation.segments.map((segment) => segment.text),
+    ["予算を確認します", "de acuerdo"],
+  );
+});
+
+test("an Arabic filler followed by Arabic punctuation is stripped", () => {
+  assert.equal(stripFillerTokens("اه، نعم"), "نعم");
+});
+
+test("a longer operator token wins over a built-in prefix", () => {
+  assert.equal(stripFillerTokens("あのーさあ始めよう", ["あのーさあ"]), "始めよう");
+});
+
+test("matcher cache keys distinguish differently-shaped token lists", () => {
+  assert.equal(stripFillerTokens("uh,um is here", ["uh,um"]), "is here");
+  assert.equal(stripFillerTokens("uh,um is here", ["uh", "um"]), "uh,um is here");
+});
+
+test("a Hebrew operator token is a whole token, never a substring", () => {
+  // Hebrew spaces its words, so an inline match would strip the suffix of
+  // an ordinary word and corrupt the transcript.
+  assert.equal(stripFillerTokens("נראה טוב", ["אה"]), "נראה טוב");
+  assert.equal(stripFillerTokens("אה טוב", ["אה"]), "טוב");
+});
+
+test("a decomposed accented filler is stripped without rewriting the rest", () => {
+  const decomposed = "a\u0308hm weiter mit dem cafe\u0301";
+  const cleaned = stripFillerTokens(decomposed);
+  assert.equal(cleaned, "weiter mit dem cafe\u0301");
+  // The surviving text keeps the ASR's own bytes: no NFC rewrite.
+  assert.ok(cleaned.includes("\u0301"));
+});

package/src/wearables/cleanup.ts

@@ -7,6 +7,7 @@
  * everything here is conservative and reversible by re-syncing.
  */
 
+import { detectScriptHints, type ScriptHint } from "./text-language.js";
 import type {
   WearableCleanupSettings,
   WearableConversation,
@@ -33,24 +34,155 @@ const MERGE_GAP_MS = 30_000;
 const GENERIC_SPEAKER_PATTERN = /^unknown$/i;
 
 /**
- * Standalone filler tokens stripped when `stripFillers` is on. Matched
- * case-insensitively on word boundaries, only as whole tokens — "um"
- * inside "umbrella" is never touched. Deliberately short, low-risk
- * list; meaning-bearing hedges ("like", "well") are NOT stripped.
+ * Standalone filler tokens stripped when `stripFillers` is on, for
+ * scripts that separate words with spaces. Matched case-insensitively
+ * as whole tokens — "um" inside "umbrella" is never touched.
+ * Deliberately short and low-risk: meaning-bearing hedges ("like",
+ * "well", Spanish "este", Arabic "يعني") are NOT stripped, and neither
+ * is "mm", which is also a unit of length.
  */
-const FILLER_TOKENS = ["um", "uh", "uhm", "umm", "uhh", "erm", "hmm", "mhm"];
+const TOKEN_FILLERS: Readonly<Partial<Record<ScriptHint, readonly string[]>>> = {
+  latin: [
+    "um",
+    "uh",
+    "uhm",
+    "umm",
+    "uhh",
+    "erm",
+    "hmm",
+    "mhm",
+    "ähm",
+    "äh",
+    "ehm",
+    "euh",
+    "mmm",
+  ],
+  korean: ["음", "으음", "엄", "어어"],
+  arabic: ["امم", "اممم", "اه"],
+  cyrillic: ["эм", "ээ", "ммм"],
+};
 
-const FILLER_PATTERN = new RegExp(
-  // Leading/trailing punctuation around the filler collapses with it so
-  // "Um, so we should" -> "so we should" rather than ", so we should".
-  `(?:^|\\s)(?:${FILLER_TOKENS.join("|")})[,.]?(?=\\s|$)`,
-  "gi",
-);
+/**
+ * Filler tokens for scripts without word spaces. These are removed
+ * inline, so each entry must be unambiguous on its own: the elongated
+ * Japanese hesitation forms and the two Chinese hesitation particles
+ * qualify, while a bare 「あの」 ("that") does not (issue #2196).
+ */
+const INLINE_FILLERS: Readonly<Partial<Record<ScriptHint, readonly string[]>>> = {
+  japanese: ["えーと", "えっと", "ええと", "あのー", "あのう", "うーん"],
+  han: ["呃", "嗯"],
+};
+
+/**
+ * Punctuation that trails a filler and collapses with it, so
+ * 「えーと、では」 becomes 「では」 rather than 「、では」. Arabic sentence
+ * punctuation is included: an Arabic filler is normally followed by
+ * `،`, and without it the token survives the pass.
+ */
+const TRAILING_FILLER_PUNCTUATION = "[,.、。،؛؟!?]?";
+
+interface FillerMatchers {
+  token: RegExp | null;
+  inline: RegExp | null;
+}
+
+/**
+ * Compiled matchers are cached per (script hints + extra tokens) key.
+ * A day of transcripts is thousands of segments over a handful of
+ * scripts, so this compiles a few regexes instead of one per segment.
+ */
+const FILLER_MATCHER_CACHE = new Map<string, FillerMatchers>();
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Build one alternation from `tokens`, or `null` when none remain.
+ *
+ * Each token contributes BOTH its composed and its decomposed spelling,
+ * so `ähm` also matches an ASR that emits `a\u0308hm`. Matching both
+ * forms keeps the surrounding transcript byte-identical, which
+ * normalizing the text before stripping would not.
+ *
+ * JavaScript alternation is first-match, not longest-match, so the
+ * tokens are sorted longest-first. Without that a built-in prefix such
+ * as 「あのー」 would match ahead of a longer operator token that starts
+ * with it and leave the tail in the transcript. Ties break on the token
+ * itself so the pattern is stable across runs.
+ */
+function buildAlternation(
+  tokens: readonly string[],
+  shape: (pattern: string) => { source: string; flags: string },
+): RegExp | null {
+  const forms = new Set<string>();
+  for (const token of tokens) {
+    forms.add(token.normalize("NFC"));
+    forms.add(token.normalize("NFD"));
+  }
+  const unique = [...forms].sort(
+    (left, right) => right.length - left.length || (left < right ? -1 : left > right ? 1 : 0),
+  );
+  if (unique.length === 0) return null;
+  const { source, flags } = shape(unique.map((token) => escapeRegExp(token)).join("|"));
+  return new RegExp(source, flags);
+}
+
+function buildFillerMatchers(
+  hints: readonly ScriptHint[],
+  extraTokens: readonly string[],
+): FillerMatchers {
+  // Structural key: comma-joining collapses `["a,b"]` and `["a","b"]`
+  // onto one entry, and the first caller's regex would then clean the
+  // second caller's transcripts.
+  const key = JSON.stringify([hints, extraTokens]);
+  const cached = FILLER_MATCHER_CACHE.get(key);
+  if (cached) return cached;
+
+  const tokens: string[] = [];
+  const inline: string[] = [];
+  for (const hint of hints) {
+    tokens.push(...(TOKEN_FILLERS[hint] ?? []));
+    inline.push(...(INLINE_FILLERS[hint] ?? []));
+  }
+  // Operator tokens apply to every script: the operator knows their own
+  // language, and routing them by script hint would silently drop them.
+  for (const extra of extraTokens) {
+    const trimmed = extra.trim();
+    if (trimmed.length === 0) continue;
+    if (SPACE_DELIMITED_TOKEN.test(trimmed)) tokens.push(trimmed);
+    else inline.push(trimmed);
+  }
+
+  const matchers: FillerMatchers = {
+    token: buildAlternation(tokens, (pattern) => ({
+      source: `(?:^|\\s)(?:${pattern})${TRAILING_FILLER_PUNCTUATION}(?=\\s|$)`,
+      flags: "giu",
+    })),
+    inline: buildAlternation(inline, (pattern) => ({
+      source: `(?:${pattern})${TRAILING_FILLER_PUNCTUATION}`,
+      flags: "gu",
+    })),
+  };
+  FILLER_MATCHER_CACHE.set(key, matchers);
+  return matchers;
+}
+
+/**
+ * A token whose first character belongs to a space-delimited script.
+ *
+ * Anything outside this set is removed INLINE, so a script that does
+ * space its words must appear here. Hebrew `אה` routed to the inline
+ * matcher would strip the suffix of `נראה` and corrupt the transcript.
+ */
+const SPACE_DELIMITED_TOKEN =
+  /^[\p{Script=Latin}\p{Script=Cyrillic}\p{Script=Greek}\p{Script=Hangul}\p{Script=Arabic}\p{Script=Hebrew}\p{N}]/u;
 
 /** Apply configured cleanup passes to one conversation. */
 export function cleanConversation(
   conversation: WearableConversation,
   settings: WearableCleanupSettings,
+  extraFillerTokens: readonly string[] = [],
 ): CleanupResult {
   let segments = conversation.segments.map((segment) => ({ ...segment }));
   let droppedSegments = 0;
@@ -61,7 +193,7 @@ export function cleanConversation(
 
   if (settings.stripFillers) {
     for (const segment of segments) {
-      segment.text = stripFillerTokens(segment.text);
+      segment.text = stripFillerTokens(segment.text, extraFillerTokens);
     }
   }
 
@@ -130,15 +262,30 @@ function canMerge(
     const label = previous.speakerKey.trim();
     if (label.length === 0 || GENERIC_SPEAKER_PATTERN.test(label)) return false;
   }
-  const previousEnd = previous.endIso ? Date.parse(previous.endIso) : NaN;
-  const nextStart = next.startIso ? Date.parse(next.startIso) : NaN;
+  const previousEnd = previous.endIso
+    ? Date.parse(previous.endIso)
+    : Number.NaN;
+  const nextStart = next.startIso ? Date.parse(next.startIso) : Number.NaN;
   // Without timestamps, adjacency is the only signal — still merge.
   if (Number.isNaN(previousEnd) || Number.isNaN(nextStart)) return true;
   return nextStart - previousEnd <= MERGE_GAP_MS;
 }
 
-export function stripFillerTokens(text: string): string {
-  return normalizeWhitespace(text.replace(FILLER_PATTERN, " "));
+/**
+ * Remove filler tokens from `text`. The built-in token set is chosen
+ * from the scripts present in the text, so a Japanese or Korean segment
+ * is cleaned by its own list instead of silently keeping every filler
+ * (issue #2196). Operator tokens apply to every script.
+ */
+export function stripFillerTokens(
+  text: string,
+  extraTokens: readonly string[] = [],
+): string {
+  const matchers = buildFillerMatchers(detectScriptHints(text), extraTokens);
+  let result = text;
+  if (matchers.token) result = result.replace(matchers.token, " ");
+  if (matchers.inline) result = result.replace(matchers.inline, "");
+  return normalizeWhitespace(result);
 }
 
 /**