@remnic/core 9.62.1 → 9.62.2

package/src/reconcile/manifest.ts

@@ -18,12 +18,30 @@ import {
 export const RECONCILE_MANIFEST_FORMAT = "remnic-reconcile-manifest";
 export const RECONCILE_MANIFEST_SCHEMA_VERSION = 1;
 const CONTENT_HASH_NORMALIZER_VERSION = 4;
+/**
+ * Version of the identity-resolution LOGIC (alias recovery, candidate
+ * ordering), independent of the content normalizer: a cached manifest entry
+ * written before an identity-resolution change carries an older value and is
+ * reparsed even when its normalizerVersion still matches (issue #2367
+ * review round 2). Bump when parsedMemoryIdentity's output shape or
+ * semantics change.
+ */
+const IDENTITY_RESOLUTION_VERSION = 2;
 
 export interface ReconcileMemoryIdentity {
   id: string;
   category: string;
   contentHash: string;
+  /**
+   * Recovered current-normalizer identities for a record whose persisted hash
+   * is ambiguous under the legacy normalizer (issue #2367). The persisted
+   * hash stays primary; these participate in duplicate bucketing so a
+   * replica carrying the current form of the same fact still collapses.
+   */
+  contentHashAliases?: readonly string[];
+  /** See IDENTITY_RESOLUTION_VERSION; absent on pre-versioning cache entries. */
   normalizerVersion?: number;
+  identityResolutionVersion?: number;
   status: MemoryStatus;
 }
 
@@ -88,17 +106,34 @@ function parsedMemoryIdentity(
   const legacyMatch = [...candidates].reverse().find(
     (content) => computeLegacyContentHash(content) === storedHash
   );
-  const contentHash = currentMatch
-    ? ContentHashIndex.computeHash(currentMatch)
-    : legacyMatch
-      ? ContentHashIndex.computeHash(legacyMatch)
-      : storedHash ?? bodyHash;
+  let contentHash: string;
+  let contentHashAliases: string[] | undefined;
+  if (currentMatch) {
+    contentHash = ContentHashIndex.computeHash(currentMatch);
+  } else if (legacyMatch && normalizeLegacyContent(legacyMatch).length > 0) {
+    // A non-empty legacy equality is ambiguous (issue #2367): the persisted
+    // hash may be an explicit contentHashSource identity that merely collides
+    // with the body's ASCII skeleton under the lossy legacy normalizer, and
+    // nothing in the record proves which it is. Keep the persisted hash as
+    // the primary identity and expose the recovered current identity as an
+    // alias so replicas carrying either form still collapse.
+    contentHash = storedHash ?? bodyHash;
+    const recovered = ContentHashIndex.computeHash(legacyMatch);
+    if (recovered !== contentHash) contentHashAliases = [recovered];
+  } else {
+    // An EMPTY legacy skeleton identifies nothing (every non-ASCII body maps
+    // to it), so preserving it would collapse unrelated records. Recovering
+    // the current identity here is repair, not replacement.
+    contentHash = legacyMatch ? ContentHashIndex.computeHash(legacyMatch) : storedHash ?? bodyHash;
+  }
 
   return {
     id: parsed.frontmatter.id,
     category: parsed.frontmatter.category,
     contentHash,
+    ...(contentHashAliases ? { contentHashAliases } : {}),
     normalizerVersion: CONTENT_HASH_NORMALIZER_VERSION,
+    identityResolutionVersion: IDENTITY_RESOLUTION_VERSION,
     status: inferMemoryStatus(parsed.frontmatter, filePath),
   };
 }
@@ -136,7 +171,8 @@ export async function buildReconcileManifest(options: BuildReconcileManifestOpti
     if (
       cached?.sha256.toLowerCase() === file.sha256.toLowerCase() &&
       (cached.memory === undefined ||
-        cached.memory.normalizerVersion === CONTENT_HASH_NORMALIZER_VERSION)
+        (cached.memory.normalizerVersion === CONTENT_HASH_NORMALIZER_VERSION &&
+          cached.memory.identityResolutionVersion === IDENTITY_RESOLUTION_VERSION))
     ) {
       files.push({ ...file, ...(cached.memory ? { memory: cached.memory } : {}) });
       continue;
@@ -168,7 +204,7 @@ function activeFactByPath(manifest: ReconcileManifest | undefined): Map<string,
 }
 
 function memoryIdentityHashes(memory: ReconcileMemoryIdentity): string[] {
-  return [memory.contentHash];
+  return [memory.contentHash, ...(memory.contentHashAliases ?? [])];
 }
 
 function contentHashRows(

package/src/search/factory.ts

@@ -79,6 +79,7 @@ function resolveNonQmdBackend(config: PluginConfig): SearchBackend | undefined {
       embedHelper,
       memoryDir: config.memoryDir,
       embeddingDimension: config.oramaEmbeddingDimension!,
+      cjkSegmentationEnabled: config.oramaCjkSegmentation !== false,
     });
   }
 

package/src/search/lancedb-backend.ts

@@ -221,12 +221,8 @@ export class LanceDbBackend implements SearchBackend {
         rows.length > 0 && rows.every((row) => row.vectorProvider === embeddingProviderIdentity),
       );
       if (isSearchAborted(execution)) return;
-      // Create FTS index on content column
-      try {
-        await table.createIndex("content", { config: this.lanceIndex.fts() });
-      } catch {
-        // FTS index creation may fail on some platforms — degrade gracefully
-      }
+      // FTS index on the content column.
+      await this.createFtsIndex(table);
       if (collection === this.collection) this.table = table;
     } catch (err) {
       log.debug(`LanceDbBackend update failed: ${err}`);
@@ -365,11 +361,7 @@ export class LanceDbBackend implements SearchBackend {
       vectorProvider: "",
     };
     const newTable = await db.createTable(collection, [emptyRow]);
-    try {
-      await newTable.createIndex("content", { config: this.lanceIndex.fts() });
-    } catch {
-      // FTS index creation may fail — degrade gracefully
-    }
+    await this.createFtsIndex(newTable);
     try {
       await newTable.delete("docid = '__placeholder__'");
     } catch {
@@ -424,11 +416,7 @@ export class LanceDbBackend implements SearchBackend {
     };
     this.table = await db.createTable(this.collection, [emptyRow]);
     // Create FTS index on content column
-    try {
-      await this.table.createIndex("content", { config: this.lanceIndex.fts() });
-    } catch {
-      // FTS index creation may fail — degrade gracefully
-    }
+    await this.createFtsIndex(this.table);
     // Remove placeholder row
     try {
       await this.table.delete("docid = '__placeholder__'");
@@ -438,6 +426,26 @@ export class LanceDbBackend implements SearchBackend {
     return this.table;
   }
 
+  /**
+   * Create the FTS index on the content column with the default tokenizer.
+   *
+   * Known limitation (issue #2187): tantivy's default tokenizer does not
+   * segment space-free scripts (CJK/Thai), so lexical recall over those
+   * scripts is near zero here. LanceDB's `baseTokenizer: "ngram"` option is
+   * not script-aware — it would n-gram English content too, degrading
+   * precision and bloating the index — so no CJK-capable configuration is
+   * available in the current @lancedb/lancedb API. Non-English lexical recall
+   * on this backend rides on the vector tier instead. See
+   * docs/search-backends.md ("Non-English content").
+   */
+  private async createFtsIndex(table: any): Promise<void> {
+    try {
+      await table.createIndex("content", { config: this.lanceIndex.fts() });
+    } catch {
+      // FTS index creation may fail on some platforms — degrade gracefully
+    }
+  }
+
   private async searchTable(
     table: any,
     query: string,

package/src/search/orama-backend.test.ts

@@ -1,8 +1,46 @@
 import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
 import path from "node:path";
 import test from "node:test";
 
-import { resolveOramaCollectionDbFilePath } from "./orama-backend.js";
+import { EmbedHelper } from "./embed-helper.js";
+import { OramaBackend, resolveOramaCollectionDbFilePath } from "./orama-backend.js";
+import type { PluginConfig } from "../types.js";
+
+function noEmbedHelper(): EmbedHelper {
+  return new EmbedHelper({} as PluginConfig);
+}
+
+async function writeMemoryFact(
+  memoryDir: string,
+  fileName: string,
+  content: string,
+): Promise<void> {
+  await mkdir(path.join(memoryDir, "facts"), { recursive: true });
+  await writeFile(path.join(memoryDir, "facts", fileName), content, "utf8");
+}
+
+async function createBackend(
+  memoryDir: string,
+  dbPath: string,
+  cjkSegmentationEnabled?: boolean,
+  options: { update?: boolean } = {},
+): Promise<OramaBackend> {
+  const backend = new OramaBackend({
+    dbPath,
+    collection: "openclaw-engram",
+    embedHelper: noEmbedHelper(),
+    memoryDir,
+    embeddingDimension: 4,
+    cjkSegmentationEnabled,
+  });
+  assert.equal(await backend.probe(), true);
+  if (options.update !== false) {
+    await backend.update();
+  }
+  return backend;
+}
 
 test("Orama collection filenames cannot escape dbPath", () => {
   const dbPath = path.join("/tmp", "remnic-orama-db");
@@ -25,3 +63,172 @@ test("Orama collection filenames cannot escape dbPath", () => {
     );
   }
 });
+
+test("Orama lexical search finds a Japanese fact with no embeddings, at parity with English (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-cjk-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "japanese-fact.md",
+      "---\nid: mem-japanese-fact\n---\n東京都庁の所在地は新宿区にある。最寄り駅は都庁前駅だ。",
+    );
+    await writeMemoryFact(
+      root,
+      "english-fact.md",
+      "---\nid: mem-english-fact\n---\nThe Tokyo Metropolitan Government Office is located in Shinjuku. The nearest station is Tochomae.",
+    );
+
+    const backend = await createBackend(root, path.join(root, "orama-db"));
+
+    const japanese = await backend.bm25Search("東京都庁の所在地");
+    const english = await backend.bm25Search("Tokyo Metropolitan Government Office location");
+
+    assert.equal(japanese.some((r) => r.docid === "mem-japanese-fact"), true);
+    assert.equal(english.some((r) => r.docid === "mem-english-fact"), true);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama CJK tokenizer rebuilds stale pre-CJK indexes on first open (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-rebuild-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "japanese-fact.md",
+      "---\nid: mem-japanese-fact\n---\n東京都庁の所在地は新宿区にある。",
+    );
+    await writeMemoryFact(
+      root,
+      "russian-fact.md",
+      "---\nid: mem-russian-fact\n---\nПравительство Токио находится в Синдзюку.",
+    );
+    await writeMemoryFact(
+      root,
+      "english-fact.md",
+      "---\nid: mem-english-fact\n---\nNebula-472 deploy finished in production.",
+    );
+
+    // Phase 1: index with the pre-CJK stock tokenizer (marker "english").
+    const legacyBackend = await createBackend(root, path.join(root, "orama-db"), false);
+    const legacyJapanese = await legacyBackend.bm25Search("東京都庁の所在地");
+    assert.equal(
+      legacyJapanese.some((r) => r.docid === "mem-japanese-fact"),
+      false,
+      "stock tokenizer must not match Japanese phrases",
+    );
+    const legacyEnglish = await legacyBackend.bm25Search("Nebula-472 deploy");
+    assert.equal(legacyEnglish.some((r) => r.docid === "mem-english-fact"), true);
+
+    // Phase 2: reopen with CJK segmentation on and NO update() call — the
+    // stale marker alone must trigger the in-place rebuild during probe().
+    const rebuiltBackend = await createBackend(root, path.join(root, "orama-db"), true, {
+      update: false,
+    });
+    const rebuiltJapanese = await rebuiltBackend.bm25Search("東京都庁の所在地");
+    assert.equal(rebuiltJapanese.some((r) => r.docid === "mem-japanese-fact"), true);
+
+    // Non-CJK non-Latin scripts are re-indexed too (whole-word terms), not
+    // just CJK/Thai.
+    const rebuiltRussian = await rebuiltBackend.bm25Search("Правительство Токио");
+    assert.equal(rebuiltRussian.some((r) => r.docid === "mem-russian-fact"), true);
+
+    // English-corpus behavior is unchanged by the rebuild.
+    const rebuiltEnglish = await rebuiltBackend.bm25Search("Nebula-472 deploy");
+    assert.equal(rebuiltEnglish.some((r) => r.docid === "mem-english-fact"), true);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama English-only corpora are not re-indexed when the tokenizer version advances (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-english-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "english-fact.md",
+      "---\nid: mem-english-fact\n---\nNebula-472 deploy finished in production.",
+    );
+
+    const legacyBackend = await createBackend(root, path.join(root, "orama-db"), false);
+    assert.equal(
+      (await legacyBackend.bm25Search("Nebula-472 deploy")).some((r) => r.docid === "mem-english-fact"),
+      true,
+    );
+
+    const upgradedBackend = await createBackend(root, path.join(root, "orama-db"), true);
+    assert.equal(
+      (await upgradedBackend.bm25Search("Nebula-472 deploy")).some((r) => r.docid === "mem-english-fact"),
+      true,
+      "English recall must survive the tokenizer upgrade",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama downgrades persisted CJK indexes when segmentation is disabled (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-downgrade-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "japanese-fact.md",
+      "---\nid: mem-japanese-fact\n---\n東京都庁の所在地は新宿区にある。",
+    );
+
+    // Phase 1: index with CJK segmentation on (marker "english+cjk-v1").
+    const cjkBackend = await createBackend(root, path.join(root, "orama-db"), true);
+    assert.equal(
+      (await cjkBackend.bm25Search("東京都庁の所在地")).some((r) => r.docid === "mem-japanese-fact"),
+      true,
+      "CJK tokenizer must match Japanese phrases before the downgrade",
+    );
+
+    // Phase 2: reopen with segmentation disabled and NO update() call — the
+    // persisted CJK marker alone must trigger the stock re-index.
+    const downgraded = await createBackend(root, path.join(root, "orama-db"), false, {
+      update: false,
+    });
+    assert.equal(
+      (await downgraded.bm25Search("東京都庁の所在地")).some((r) => r.docid === "mem-japanese-fact"),
+      false,
+      "stock tokenizer must not match Japanese phrases after the downgrade",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama rebuild gate inspects every indexed string field, not only content (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-pathfield-"));
+  try {
+    // The only non-Latin text is the FILENAME, which lands in the indexed
+    // path field; the body is pure English.
+    await writeMemoryFact(
+      root,
+      "東京-fact.md",
+      "---\nid: mem-path-only\n---\nAn English body with no non-Latin characters.",
+    );
+
+    // Phase 1: pre-CJK stock index — the CJK filename never becomes a term.
+    const legacyBackend = await createBackend(root, path.join(root, "orama-db"), false);
+    assert.equal(
+      (await legacyBackend.bm25Search("東京")).some((r) => r.docid === "mem-path-only"),
+      false,
+      "stock tokenizer must not match the CJK filename",
+    );
+
+    // Phase 2: reopen with segmentation on and NO update() call — the
+    // path field alone must gate the in-place re-index.
+    const rebuilt = await createBackend(root, path.join(root, "orama-db"), true, {
+      update: false,
+    });
+    assert.equal(
+      (await rebuilt.bm25Search("東京")).some((r) => r.docid === "mem-path-only"),
+      true,
+      "non-Latin text in the path field must gate the rebuild",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/src/search/orama-backend.ts

@@ -9,17 +9,28 @@ import {
   type SearchQueryOptions,
   type SearchResult,
 } from "./port.js";
-import type { EmbedHelper, EmbedProviderIdentity, EmbedWithProviderResult } from "./embed-helper.js";
-import { scanMemoryDir } from "./document-scanner.js";
-import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
-
-export interface OramaBackendOptions {
-  dbPath: string;
-  collection: string;
-  embedHelper: EmbedHelper;
-  memoryDir: string;
-  embeddingDimension: number;
-}
+ import type { EmbedHelper, EmbedProviderIdentity, EmbedWithProviderResult } from "./embed-helper.js";
+ import { scanMemoryDir } from "./document-scanner.js";
+ import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
+import {
+  containsNonLegacyTokenizerChars,
+  createCjkCapableTokenizer,
+  ORAMA_CJK_TOKENIZER_LANGUAGE,
+} from "./orama-cjk-tokenizer.js";
+ 
+ export interface OramaBackendOptions {
+   dbPath: string;
+   collection: string;
+   embedHelper: EmbedHelper;
+   memoryDir: string;
+   embeddingDimension: number;
+  /**
+   * Segment space-free scripts (CJK/Thai) in the lexical index. Default
+   * `true`; `false` restores the stock English-only tokenizer (issue #2187).
+   */
+  cjkSegmentationEnabled?: boolean;
+ }
+
 
 const ORAMA_COLLECTION_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
 
@@ -57,6 +68,7 @@ export class OramaBackend implements SearchBackend {
   private readonly embedHelper: EmbedHelper;
   private readonly memoryDir: string;
   private readonly embeddingDimension: number;
+  private readonly cjkSegmentationEnabled: boolean;
   private available = false;
   private db: any = null;
   private oramaModule: any = null;
@@ -72,6 +84,7 @@ export class OramaBackend implements SearchBackend {
     this.embedHelper = opts.embedHelper;
     this.memoryDir = opts.memoryDir;
     this.embeddingDimension = opts.embeddingDimension;
+    this.cjkSegmentationEnabled = opts.cjkSegmentationEnabled !== false;
   }
 
   async probe(): Promise<boolean> {
@@ -127,6 +140,11 @@ export class OramaBackend implements SearchBackend {
     }
   }
 
+  /**
+   * Full-text (FTS) search only. With the stock tokenizer this cannot match
+   * space-free CJK/Thai text lexically — hybrid or vector search provides
+   * the fallback for those scripts (issue #2187).
+   */
   async bm25Search(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
     if (isSearchAborted(execution)) return [];
     const db = await this.ensureDbForCollection(collection ?? this.collection);
@@ -403,8 +421,11 @@ export class OramaBackend implements SearchBackend {
       return this.db;
     }
 
-    this.db = await this.migrateLegacyVectorProviderSchema(
-      await this.persistModule.restore("json", raw),
+    this.db = await this.applyTokenizerVersion(
+      await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        this.collection,
+      ),
       this.collection,
     );
     return this.db;
@@ -426,8 +447,11 @@ export class OramaBackend implements SearchBackend {
       return await this.createDb();
     }
 
-    return await this.migrateLegacyVectorProviderSchema(
-      await this.persistModule.restore("json", raw),
+    return await this.applyTokenizerVersion(
+      await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        collection,
+      ),
       collection,
     );
   }
@@ -442,7 +466,95 @@ export class OramaBackend implements SearchBackend {
       vectorProvider: "string",
       vector: `vector[${this.embeddingDimension}]`,
     };
-    return await create({ schema });
+    if (!this.cjkSegmentationEnabled) {
+      return await create({ schema });
+    }
+    return await create({
+      schema,
+      components: { tokenizer: createCjkCapableTokenizer(this.oramaModule) },
+    });
+  }
+
+  /**
+   * Attach the CJK-capable tokenizer to a restored index and rebuild the
+   * full-text index when it predates CJK tokenization. Orama persists
+   * `tokenizer.language` with the index, so a marker other than
+   * `ORAMA_CJK_TOKENIZER_LANGUAGE` identifies an index whose terms were
+   * produced by the stock English tokenizer. Corpora whose content the stock
+   * tokenizer and this tokenizer treat identically (pure legacy Latin) need
+   * no re-indexing — only the version marker is rewritten for them.
+   */
+  private async applyTokenizerVersion(db: any, collection: string): Promise<any> {
+    const persistedLanguage =
+      db?.tokenizer?.language && typeof db.tokenizer.language === "string"
+        ? db.tokenizer.language
+        : "";
+    if (this.cjkSegmentationEnabled) {
+      db.tokenizer = createCjkCapableTokenizer(this.oramaModule);
+      if (persistedLanguage === ORAMA_CJK_TOKENIZER_LANGUAGE) return db;
+    } else {
+      // Segmentation disabled: a database persisted under the CJK tokenizer
+      // keeps its CJK terms unless it is re-indexed under the stock one, so
+      // downgrade it instead of returning it unchanged (issue #2187 round 2).
+      if (persistedLanguage !== ORAMA_CJK_TOKENIZER_LANGUAGE) return db;
+      db.tokenizer = this.oramaModule.components.tokenizer.createTokenizer({ language: "english" });
+    }
+
+
+    try {
+      const { search: oramaSearch, count, update: oramaUpdate } = this.oramaModule;
+      const existingCount = await count(db);
+      if (existingCount === 0) {
+        await this.persistDbForCollection(db, collection);
+        return db;
+      }
+      const allHits = await oramaSearch(db, { term: "", limit: existingCount + 100 });
+      const hits = allHits.hits ?? [];
+      const hasNonLegacyContent = hits.some((hit: any) => {
+        const doc = this.getStoredDocument(db, hit);
+        // Orama full-text queries every string field by default, so every
+        // indexed string field gates the rebuild — non-Latin text that only
+        // rides in snippet/path/id would otherwise keep stale terms
+        // (issue #2187 review round 2).
+        for (const field of [doc.content, doc.snippet, doc.path, doc.id, doc.vectorProvider]) {
+          if (typeof field === "string" && containsNonLegacyTokenizerChars(field)) return true;
+        }
+        return false;
+      });
+      if (hasNonLegacyContent) {
+        // Re-index every document in place (Orama update is remove+insert,
+        // so content is re-tokenized; vectors are carried through). A single
+        // failing document must not block the rest of the corpus: it keeps
+        // its pre-upgrade terms, which is no worse than before the upgrade.
+        for (const hit of hits) {
+          try {
+            const doc = this.getStoredDocument(db, hit);
+            const vector = this.getStoredVector(db, hit, doc);
+            await oramaUpdate(db, hit.id, {
+              id: typeof doc.id === "string" && doc.id.length > 0 ? doc.id : String(hit.id),
+              path: typeof doc.path === "string" ? doc.path : "",
+              content: typeof doc.content === "string" ? doc.content : "",
+              snippet:
+                typeof doc.snippet === "string"
+                  ? doc.snippet
+                  : typeof doc.content === "string"
+                    ? doc.content.slice(0, 200)
+                    : "",
+              vectorProvider: typeof doc.vectorProvider === "string" ? doc.vectorProvider : "",
+              vector: vector ?? this.zeroVector(),
+            });
+          } catch (docErr) {
+            log.debug(
+              `OramaBackend tokenizer-version re-index skipped document ${hit.id} in ${collection}: ${docErr}`,
+            );
+          }
+        }
+      }
+      await this.persistDbForCollection(db, collection);
+    } catch (err) {
+      log.debug(`OramaBackend tokenizer-version rebuild for ${collection} failed: ${err}`);
+    }
+    return db;
   }
 
   private async migrateLegacyVectorProviderSchema(db: any, collection: string): Promise<any> {
@@ -526,8 +638,11 @@ export class OramaBackend implements SearchBackend {
       await this.ensureModules();
       const raw = await readFile(filePath, "utf-8");
       const collection = path.basename(filePath, ".msp");
-      return await this.migrateLegacyVectorProviderSchema(
-        await this.persistModule.restore("json", raw),
+      return await this.applyTokenizerVersion(
+        await this.migrateLegacyVectorProviderSchema(
+          await this.persistModule.restore("json", raw),
+          collection,
+        ),
         collection,
       );
     } catch (err) {