@remnic/core 9.62.1 → 9.62.2

package/dist/{chunk-YLHM5BQQ.js → chunk-DCUMFQYM.js}

@@ -9,6 +9,10 @@ import {
   reportSearchDegradation,
   resolveEnsureCollectionArgs
 } from "./chunk-URS5GSM7.js";
+import {
+  expandUnsegmentableRecallNGrams,
+  isUnsegmentableRecallChar
+} from "./chunk-DJ7HMNSL.js";
 import {
   log
 } from "./chunk-3WCASNQZ.js";
@@ -16,6 +20,83 @@ import {
 // src/search/orama-backend.ts
 import path from "path";
 import { mkdir, readdir, readFile, rename, rm, writeFile } from "fs/promises";
+
+// src/search/orama-cjk-tokenizer.ts
+var ORAMA_CJK_TOKENIZER_LANGUAGE = "english+cjk-v1";
+var THAI_SCRIPT_CHAR = /[\p{Script=Thai}]/u;
+var WORD_CHAR = /[\p{L}\p{N}\p{M}]/u;
+var COMBINING_MARK = /\p{M}/u;
+var WHITESPACE_CHAR = /\s/u;
+var LEGACY_TOKENIZER_CHAR = /[A-Za-zàèéìòóù0-9_'-]/u;
+function isSpaceFreeScriptChar(char) {
+  return isUnsegmentableRecallChar(char) || THAI_SCRIPT_CHAR.test(char);
+}
+function containsNonLegacyTokenizerChars(value) {
+  for (const ch of value) {
+    if (LEGACY_TOKENIZER_CHAR.test(ch) || WHITESPACE_CHAR.test(ch)) continue;
+    if (WORD_CHAR.test(ch) || COMBINING_MARK.test(ch) || isSpaceFreeScriptChar(ch)) {
+      return true;
+    }
+  }
+  return false;
+}
+function createCjkCapableTokenizer(oramaModule) {
+  const base = oramaModule.components.tokenizer.createTokenizer({ language: "english" });
+  const tokenize = (raw, _language, prop, withCache) => {
+    if (typeof raw !== "string") return [];
+    const normalized = raw.normalize("NFC");
+    if (!containsNonLegacyTokenizerChars(normalized)) {
+      return base.tokenize(normalized, "english", prop, withCache);
+    }
+    const tokens = base.tokenize(normalized, "english", prop, withCache);
+    const seen = new Set(tokens);
+    const pushToken = (token) => {
+      if (token && !seen.has(token)) {
+        seen.add(token);
+        tokens.push(token);
+      }
+    };
+    let spaceFreeRun = "";
+    let wordRun = "";
+    const flushSpaceFreeRun = () => {
+      if (!spaceFreeRun) return;
+      for (const token of expandUnsegmentableRecallNGrams(spaceFreeRun)) {
+        pushToken(token);
+      }
+      spaceFreeRun = "";
+    };
+    const flushWordRun = () => {
+      if (!wordRun) return;
+      pushToken(wordRun.toLowerCase());
+      wordRun = "";
+    };
+    for (const ch of normalized) {
+      if (isSpaceFreeScriptChar(ch)) {
+        flushWordRun();
+        spaceFreeRun += ch;
+      } else if (COMBINING_MARK.test(ch)) {
+        if (spaceFreeRun) spaceFreeRun += ch;
+        else if (wordRun) wordRun += ch;
+      } else if (WORD_CHAR.test(ch) && !LEGACY_TOKENIZER_CHAR.test(ch)) {
+        flushSpaceFreeRun();
+        wordRun += ch;
+      } else {
+        flushSpaceFreeRun();
+        flushWordRun();
+      }
+    }
+    flushSpaceFreeRun();
+    flushWordRun();
+    return tokens;
+  };
+  return {
+    language: ORAMA_CJK_TOKENIZER_LANGUAGE,
+    normalizationCache: base.normalizationCache,
+    tokenize
+  };
+}
+
+// src/search/orama-backend.ts
 var ORAMA_COLLECTION_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
 function pathIsInside(parent, child) {
   const relative = path.relative(parent, child);
@@ -42,6 +123,7 @@ var OramaBackend = class {
   embedHelper;
   memoryDir;
   embeddingDimension;
+  cjkSegmentationEnabled;
   available = false;
   db = null;
   oramaModule = null;
@@ -53,6 +135,7 @@ var OramaBackend = class {
     this.embedHelper = opts.embedHelper;
     this.memoryDir = opts.memoryDir;
     this.embeddingDimension = opts.embeddingDimension;
+    this.cjkSegmentationEnabled = opts.cjkSegmentationEnabled !== false;
   }
   async probe() {
     try {
@@ -96,6 +179,11 @@ var OramaBackend = class {
       return [];
     }
   }
+  /**
+   * Full-text (FTS) search only. With the stock tokenizer this cannot match
+   * space-free CJK/Thai text lexically — hybrid or vector search provides
+   * the fallback for those scripts (issue #2187).
+   */
   async bm25Search(query, collection, maxResults, execution) {
     if (isSearchAborted(execution)) return [];
     const db = await this.ensureDbForCollection(collection ?? this.collection);
@@ -313,8 +401,11 @@ var OramaBackend = class {
       this.db = await this.createDb();
       return this.db;
     }
-    this.db = await this.migrateLegacyVectorProviderSchema(
-      await this.persistModule.restore("json", raw),
+    this.db = await this.applyTokenizerVersion(
+      await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        this.collection
+      ),
       this.collection
     );
     return this.db;
@@ -330,8 +421,11 @@ var OramaBackend = class {
     } catch {
       return await this.createDb();
     }
-    return await this.migrateLegacyVectorProviderSchema(
-      await this.persistModule.restore("json", raw),
+    return await this.applyTokenizerVersion(
+      await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        collection
+      ),
       collection
     );
   }
@@ -345,7 +439,73 @@ var OramaBackend = class {
       vectorProvider: "string",
       vector: `vector[${this.embeddingDimension}]`
     };
-    return await create({ schema });
+    if (!this.cjkSegmentationEnabled) {
+      return await create({ schema });
+    }
+    return await create({
+      schema,
+      components: { tokenizer: createCjkCapableTokenizer(this.oramaModule) }
+    });
+  }
+  /**
+   * Attach the CJK-capable tokenizer to a restored index and rebuild the
+   * full-text index when it predates CJK tokenization. Orama persists
+   * `tokenizer.language` with the index, so a marker other than
+   * `ORAMA_CJK_TOKENIZER_LANGUAGE` identifies an index whose terms were
+   * produced by the stock English tokenizer. Corpora whose content the stock
+   * tokenizer and this tokenizer treat identically (pure legacy Latin) need
+   * no re-indexing — only the version marker is rewritten for them.
+   */
+  async applyTokenizerVersion(db, collection) {
+    const persistedLanguage = db?.tokenizer?.language && typeof db.tokenizer.language === "string" ? db.tokenizer.language : "";
+    if (this.cjkSegmentationEnabled) {
+      db.tokenizer = createCjkCapableTokenizer(this.oramaModule);
+      if (persistedLanguage === ORAMA_CJK_TOKENIZER_LANGUAGE) return db;
+    } else {
+      if (persistedLanguage !== ORAMA_CJK_TOKENIZER_LANGUAGE) return db;
+      db.tokenizer = this.oramaModule.components.tokenizer.createTokenizer({ language: "english" });
+    }
+    try {
+      const { search: oramaSearch, count, update: oramaUpdate } = this.oramaModule;
+      const existingCount = await count(db);
+      if (existingCount === 0) {
+        await this.persistDbForCollection(db, collection);
+        return db;
+      }
+      const allHits = await oramaSearch(db, { term: "", limit: existingCount + 100 });
+      const hits = allHits.hits ?? [];
+      const hasNonLegacyContent = hits.some((hit) => {
+        const doc = this.getStoredDocument(db, hit);
+        for (const field of [doc.content, doc.snippet, doc.path, doc.id, doc.vectorProvider]) {
+          if (typeof field === "string" && containsNonLegacyTokenizerChars(field)) return true;
+        }
+        return false;
+      });
+      if (hasNonLegacyContent) {
+        for (const hit of hits) {
+          try {
+            const doc = this.getStoredDocument(db, hit);
+            const vector = this.getStoredVector(db, hit, doc);
+            await oramaUpdate(db, hit.id, {
+              id: typeof doc.id === "string" && doc.id.length > 0 ? doc.id : String(hit.id),
+              path: typeof doc.path === "string" ? doc.path : "",
+              content: typeof doc.content === "string" ? doc.content : "",
+              snippet: typeof doc.snippet === "string" ? doc.snippet : typeof doc.content === "string" ? doc.content.slice(0, 200) : "",
+              vectorProvider: typeof doc.vectorProvider === "string" ? doc.vectorProvider : "",
+              vector: vector ?? this.zeroVector()
+            });
+          } catch (docErr) {
+            log.debug(
+              `OramaBackend tokenizer-version re-index skipped document ${hit.id} in ${collection}: ${docErr}`
+            );
+          }
+        }
+      }
+      await this.persistDbForCollection(db, collection);
+    } catch (err) {
+      log.debug(`OramaBackend tokenizer-version rebuild for ${collection} failed: ${err}`);
+    }
+    return db;
   }
   async migrateLegacyVectorProviderSchema(db, collection) {
     const { search: oramaSearch, count, insert } = this.oramaModule;
@@ -414,8 +574,11 @@ var OramaBackend = class {
       await this.ensureModules();
       const raw = await readFile(filePath, "utf-8");
       const collection = path.basename(filePath, ".msp");
-      return await this.migrateLegacyVectorProviderSchema(
-        await this.persistModule.restore("json", raw),
+      return await this.applyTokenizerVersion(
+        await this.migrateLegacyVectorProviderSchema(
+          await this.persistModule.restore("json", raw),
+          collection
+        ),
         collection
       );
     } catch (err) {
@@ -613,4 +776,4 @@ export {
   resolveOramaCollectionDbFilePath,
   OramaBackend
 };
-//# sourceMappingURL=chunk-YLHM5BQQ.js.map
+//# sourceMappingURL=chunk-DCUMFQYM.js.map

package/dist/chunk-DCUMFQYM.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/search/orama-backend.ts","../src/search/orama-cjk-tokenizer.ts"],"sourcesContent":["import path from \"node:path\";\nimport { mkdir, readdir, readFile, rename, rm, writeFile } from \"node:fs/promises\";\nimport { log } from \"../logger.js\";\nimport {\n  reportSearchDegradation,\n  resolveEnsureCollectionArgs,\n  type SearchBackend,\n  type SearchExecutionOptions,\n  type SearchQueryOptions,\n  type SearchResult,\n} from \"./port.js\";\n import type { EmbedHelper, EmbedProviderIdentity, EmbedWithProviderResult } from \"./embed-helper.js\";\n import { scanMemoryDir } from \"./document-scanner.js\";\n import { isSearchAborted, throwIfSearchAborted } from \"./abort.js\";\nimport {\n  containsNonLegacyTokenizerChars,\n  createCjkCapableTokenizer,\n  ORAMA_CJK_TOKENIZER_LANGUAGE,\n} from \"./orama-cjk-tokenizer.js\";\n \n export interface OramaBackendOptions {\n   dbPath: string;\n   collection: string;\n   embedHelper: EmbedHelper;\n   memoryDir: string;\n   embeddingDimension: number;\n  /**\n   * Segment space-free scripts (CJK/Thai) in the lexical index. Default\n   * `true`; `false` restores the stock English-only tokenizer (issue #2187).\n   */\n  cjkSegmentationEnabled?: boolean;\n }\n\n\nconst ORAMA_COLLECTION_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;\n\nfunction pathIsInside(parent: string, child: string): boolean {\n  const relative = path.relative(parent, child);\n  return relative === \"\" || (!relative.startsWith(\"..\") && !path.isAbsolute(relative));\n}\n\nexport function resolveOramaCollectionDbFilePath(dbPath: string, collection: string): string {\n  if (!ORAMA_COLLECTION_FILENAME_PATTERN.test(collection)) {\n    throw new Error(\n      `Invalid Orama collection name ${JSON.stringify(collection)}. ` +\n        \"Collection names must match [A-Za-z0-9][A-Za-z0-9._-]*.\",\n    );\n  }\n  const resolvedDbPath = path.resolve(dbPath);\n  const filePath = path.resolve(resolvedDbPath, `${collection}.msp`);\n  if (!pathIsInside(resolvedDbPath, filePath)) {\n    throw new Error(\n      `Invalid Orama collection path for ${JSON.stringify(collection)}: resolved outside dbPath.`,\n    );\n  }\n  return filePath;\n}\n\n/**\n * Orama search backend — embedded hybrid FTS+vector, pure JS.\n *\n * Uses @orama/orama for full-text search with optional vector support.\n * Persists data to JSON files via @orama/plugin-data-persistence.\n */\nexport class OramaBackend implements SearchBackend {\n  private readonly dbPath: string;\n  private readonly collection: string;\n  private readonly embedHelper: EmbedHelper;\n  private readonly memoryDir: string;\n  private readonly embeddingDimension: number;\n  private readonly cjkSegmentationEnabled: boolean;\n  private available = false;\n  private db: any = null;\n  private oramaModule: any = null;\n  private persistModule: any = null;\n  private readonly vectorProviderCompatibility = new WeakMap<\n    object,\n    { providerIdentity: EmbedProviderIdentity; compatible: boolean }\n  >();\n\n  constructor(opts: OramaBackendOptions) {\n    this.dbPath = opts.dbPath;\n    this.collection = opts.collection;\n    this.embedHelper = opts.embedHelper;\n    this.memoryDir = opts.memoryDir;\n    this.embeddingDimension = opts.embeddingDimension;\n    this.cjkSegmentationEnabled = opts.cjkSegmentationEnabled !== false;\n  }\n\n  async probe(): Promise<boolean> {\n    try {\n      await this.ensureModules();\n      await this.ensureDb();\n      this.available = true;\n      return true;\n    } catch (err) {\n      log.debug(`OramaBackend probe failed: ${err}`);\n      this.available = false;\n      return false;\n    }\n  }\n\n  isAvailable(): boolean {\n    return this.available;\n  }\n\n  debugStatus(): string {\n    return `backend=orama available=${this.available} dbPath=${this.dbPath}`;\n  }\n\n  async search(\n    query: string,\n    _collection?: string,\n    maxResults?: number,\n    _options?: SearchQueryOptions,\n    execution?: SearchExecutionOptions,\n  ): Promise<SearchResult[]> {\n    return this.hybridSearch(query, _collection, maxResults, execution);\n  }\n\n  async searchGlobal(query: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {\n    const limit = maxResults ?? 10;\n    if (!this.available) return [];\n    try {\n      throwIfSearchAborted(execution, \"OramaBackend global search aborted\");\n      const files = await this.listDbFiles();\n      const allResults: SearchResult[] = [];\n      for (const file of files) {\n        throwIfSearchAborted(execution, \"OramaBackend global search aborted\");\n        const db = await this.loadDbFromFile(file);\n        if (!db) continue;\n        const results = await this.searchDb(db, query, \"hybrid\", limit, execution);\n        allResults.push(...results);\n      }\n      allResults.sort((a, b) => b.score - a.score);\n      return allResults.slice(0, limit);\n    } catch (err) {\n      log.debug(`OramaBackend searchGlobal failed: ${err}`);\n      return [];\n    }\n  }\n\n  /**\n   * Full-text (FTS) search only. With the stock tokenizer this cannot match\n   * space-free CJK/Thai text lexically — hybrid or vector search provides\n   * the fallback for those scripts (issue #2187).\n   */\n  async bm25Search(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {\n    if (isSearchAborted(execution)) return [];\n    const db = await this.ensureDbForCollection(collection ?? this.collection);\n    if (isSearchAborted(execution)) return [];\n    if (!db) return [];\n    return this.searchDb(db, query, \"fulltext\", maxResults ?? 10, execution);\n  }\n\n  async vectorSearch(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {\n    if (isSearchAborted(execution)) return [];\n    const db = await this.ensureDbForCollection(collection ?? this.collection);\n    if (isSearchAborted(execution)) return [];\n    if (!db) return [];\n    return this.searchDb(db, query, \"vector\", maxResults ?? 10, execution);\n  }\n\n  async hybridSearch(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {\n    if (isSearchAborted(execution)) return [];\n    const db = await this.ensureDbForCollection(collection ?? this.collection);\n    if (isSearchAborted(execution)) return [];\n    if (!db) return [];\n    return this.searchDb(db, query, \"hybrid\", maxResults ?? 10, execution);\n  }\n\n  async update(execution?: SearchExecutionOptions): Promise<void> {\n    await this.updateCollection(this.collection, execution);\n  }\n\n  async updateCollection(collection: string, execution?: SearchExecutionOptions): Promise<void> {\n    await this.updateCollectionFromDir(collection, this.memoryDir, execution);\n  }\n\n  async updateCollectionFromDir(collection: string, memoryDir: string, execution?: SearchExecutionOptions): Promise<void> {\n    if (isSearchAborted(execution)) return;\n    const db = await this.ensureDbForCollection(collection);\n    if (isSearchAborted(execution)) return;\n    if (!db) return;\n    const { search: oramaSearch, insert, remove, count, getByID } = this.oramaModule;\n\n    const docs = await scanMemoryDir(memoryDir);\n    if (isSearchAborted(execution)) return;\n    const docMap = new Map(docs.map((d) => [d.docid, d]));\n    const { update: oramaUpdate } = this.oramaModule;\n\n    const embeddingProviderIdentity = this.embedHelper.getProviderIdentity();\n    let allRowsCompatible = !!embeddingProviderIdentity && docs.length > 0;\n    // Get existing docs to diff — map user doc ID → { internalId, vector }\n    const existingDocs = new Map<string, {\n      internalId: string;\n      vector?: number[];\n      vectorProvider?: string;\n    }>();\n    const existingCount = await count(db);\n    if (existingCount > 0) {\n      const allHits = await oramaSearch(db, {\n        term: \"\",\n        limit: existingCount + 100,\n      });\n      for (const hit of allHits.hits) {\n        if (isSearchAborted(execution)) return;\n        const storedDocument =\n          typeof getByID === \"function\"\n            ? await getByID(db, hit.id)\n            : hit.document;\n        const document = storedDocument ?? hit.document ?? {};\n        if (!docMap.has(document.id)) {\n          await remove(db, hit.id);\n        } else {\n          existingDocs.set(document.id, {\n            internalId: hit.id,\n            vector: this.normalizeStoredVector(document.vector) ?? undefined,\n            vectorProvider:\n              typeof document.vectorProvider === \"string\"\n                ? document.vectorProvider\n                : undefined,\n          });\n        }\n      }\n    }\n\n    // Insert new docs, update existing ones (preserving vectors since update is remove+insert)\n    for (const doc of docs) {\n      if (isSearchAborted(execution)) return;\n      const existing = existingDocs.get(doc.docid);\n      if (existing) {\n        const payload: Record<string, unknown> = {\n          id: doc.docid,\n          path: doc.path,\n          content: doc.content,\n          snippet: doc.snippet,\n        };\n        const preservesCompatibleProvider =\n          !!embeddingProviderIdentity &&\n          existing.vectorProvider === embeddingProviderIdentity;\n        if (preservesCompatibleProvider) {\n          if (this.isCompatibleStoredVector(existing.vector)) {\n            payload.vector = existing.vector;\n            payload.vectorProvider = existing.vectorProvider ?? \"\";\n          } else {\n            payload.vector = this.zeroVector();\n            payload.vectorProvider = \"\";\n            allRowsCompatible = false;\n          }\n        } else if (!embeddingProviderIdentity && this.isCompatibleStoredVector(existing.vector)) {\n          payload.vector = existing.vector;\n          payload.vectorProvider = existing.vectorProvider ?? \"\";\n          allRowsCompatible = false;\n        } else {\n          payload.vector = this.zeroVector();\n          payload.vectorProvider = \"\";\n          allRowsCompatible = false;\n        }\n        try {\n          await oramaUpdate(db, existing.internalId, payload);\n        } catch {\n          allRowsCompatible = false;\n          // Update failed — skip and continue with remaining docs\n        }\n      } else {\n        allRowsCompatible = false;\n        try {\n          await insert(db, {\n            id: doc.docid,\n            path: doc.path,\n            content: doc.content,\n            snippet: doc.snippet,\n            vector: this.zeroVector(),\n            vectorProvider: \"\",\n          });\n        } catch {\n          allRowsCompatible = false;\n          // Duplicate id edge case — skip\n        }\n      }\n    }\n\n    if (isSearchAborted(execution)) return;\n    await this.persistDbForCollection(db, collection);\n    this.rememberVectorProviderCompatibility(\n      db,\n      embeddingProviderIdentity,\n      allRowsCompatible,\n    );\n  }\n\n  async embed(): Promise<void> {\n    await this.embedCollection(this.collection);\n  }\n\n  async embedCollection(collection: string): Promise<void> {\n    if (!this.embedHelper.isAvailable()) return;\n\n    const db = await this.ensureDbForCollection(collection);\n    if (!db) return;\n    const { search: oramaSearch, update: oramaUpdate, count } = this.oramaModule;\n\n    const existingCount = await count(db);\n    if (existingCount === 0) return;\n\n    const embeddingProviderIdentity = this.embedHelper.getProviderIdentity();\n    // Find docs without vectors or with vectors from a different provider.\n    const allHits = await oramaSearch(db, { term: \"\", limit: existingCount + 100 });\n    const needsEmbed = allHits.hits.filter((h: any) => {\n      const vector = this.normalizeStoredVector(h.document?.vector);\n      return (\n        (embeddingProviderIdentity &&\n          h.document?.vectorProvider !== embeddingProviderIdentity) ||\n        !this.isCompatibleStoredVector(vector)\n      );\n    });\n\n    if (needsEmbed.length === 0) {\n      this.rememberVectorProviderCompatibility(db, embeddingProviderIdentity, true);\n      return;\n    }\n\n    let rowsToEmbed = needsEmbed;\n    let embedResult = await this.embedHelper.embedBatchWithProvider(\n      rowsToEmbed.map((h: any) => h.document.content as string),\n    );\n    if (!embedResult) return;\n    if (\n      embeddingProviderIdentity &&\n      embedResult.providerIdentity !== embeddingProviderIdentity\n    ) {\n      const effectiveProviderIdentity = embedResult.providerIdentity;\n      const originalIds = new Set(rowsToEmbed.map((h: any) => h.id));\n      const effectiveNeedsEmbed = allHits.hits.filter((h: any) => {\n        const vector = this.normalizeStoredVector(h.document?.vector);\n        return (\n          h.document?.vectorProvider !== effectiveProviderIdentity ||\n          !this.isCompatibleStoredVector(vector)\n        );\n      });\n      const sameRows =\n        effectiveNeedsEmbed.length === rowsToEmbed.length &&\n        effectiveNeedsEmbed.every((h: any) => originalIds.has(h.id));\n      if (!sameRows) {\n        const effectiveTexts = effectiveNeedsEmbed.map((h: any) => h.document.content as string);\n        const effectiveEmbedResult = await this.embedHelper.embedBatchWithProvider(effectiveTexts);\n        if (effectiveEmbedResult) {\n          rowsToEmbed = effectiveNeedsEmbed;\n          embedResult = effectiveEmbedResult;\n        }\n      }\n    }\n    const { vectors, providerIdentity } = embedResult;\n\n    let allEmbedded = true;\n    for (let i = 0; i < rowsToEmbed.length; i++) {\n      const vec = vectors[i];\n      if (!this.isExpectedDimensionVector(vec)) {\n        allEmbedded = false;\n        continue;\n      }\n      // Orama update is remove+insert — must include all fields to avoid data loss\n      const doc = rowsToEmbed[i].document;\n      await oramaUpdate(db, rowsToEmbed[i].id, {\n        id: doc.id,\n        path: doc.path,\n        content: doc.content,\n        snippet: doc.snippet,\n        vector: vec,\n        vectorProvider: providerIdentity,\n      });\n    }\n\n    await this.persistDbForCollection(db, collection);\n    if (allEmbedded) {\n      this.rememberVectorProviderCompatibility(db, providerIdentity, true);\n    } else {\n      this.rememberVectorProviderCompatibility(db, providerIdentity, false);\n    }\n  }\n\n  async ensureCollection(\n    _memoryDir: string,\n    collectionOrExecution?: string | SearchExecutionOptions,\n    execution?: SearchExecutionOptions,\n  ): Promise<\"present\" | \"missing\" | \"unknown\" | \"skipped\"> {\n    const { collection, execution: effectiveExecution } = resolveEnsureCollectionArgs(\n      collectionOrExecution,\n      execution,\n    );\n    if (isSearchAborted(effectiveExecution)) return \"skipped\";\n    try {\n      await this.ensureModules();\n      await this.ensureDbForCollection(collection ?? this.collection);\n      return \"present\";\n    } catch {\n      return \"missing\";\n    }\n  }\n\n  private async ensureModules(): Promise<void> {\n    if (this.oramaModule && this.persistModule) return;\n    this.oramaModule = await import(\"@orama/orama\");\n    this.persistModule = await import(\"@orama/plugin-data-persistence\");\n  }\n\n  private async ensureDb(): Promise<any> {\n    if (this.db) return this.db;\n    await this.ensureModules();\n\n    await mkdir(this.dbPath, { recursive: true });\n    const filePath = this.dbFilePath(this.collection);\n\n    let raw: string;\n    try {\n      raw = await readFile(filePath, \"utf-8\");\n    } catch {\n      // No existing DB — create fresh\n      this.db = await this.createDb();\n      return this.db;\n    }\n\n    this.db = await this.applyTokenizerVersion(\n      await this.migrateLegacyVectorProviderSchema(\n        await this.persistModule.restore(\"json\", raw),\n        this.collection,\n      ),\n      this.collection,\n    );\n    return this.db;\n  }\n\n  private async ensureDbForCollection(collection: string): Promise<any> {\n    // For the default collection, use the cached instance\n    if (collection === this.collection) return this.ensureDb();\n\n    await this.ensureModules();\n    await mkdir(this.dbPath, { recursive: true });\n    const filePath = this.dbFilePath(collection);\n\n    let raw: string;\n    try {\n      raw = await readFile(filePath, \"utf-8\");\n    } catch {\n      // No existing DB — create fresh\n      return await this.createDb();\n    }\n\n    return await this.applyTokenizerVersion(\n      await this.migrateLegacyVectorProviderSchema(\n        await this.persistModule.restore(\"json\", raw),\n        collection,\n      ),\n      collection,\n    );\n  }\n\n  private async createDb(): Promise<any> {\n    const { create } = this.oramaModule;\n    const schema: Record<string, string> = {\n      id: \"string\",\n      path: \"string\",\n      content: \"string\",\n      snippet: \"string\",\n      vectorProvider: \"string\",\n      vector: `vector[${this.embeddingDimension}]`,\n    };\n    if (!this.cjkSegmentationEnabled) {\n      return await create({ schema });\n    }\n    return await create({\n      schema,\n      components: { tokenizer: createCjkCapableTokenizer(this.oramaModule) },\n    });\n  }\n\n  /**\n   * Attach the CJK-capable tokenizer to a restored index and rebuild the\n   * full-text index when it predates CJK tokenization. Orama persists\n   * `tokenizer.language` with the index, so a marker other than\n   * `ORAMA_CJK_TOKENIZER_LANGUAGE` identifies an index whose terms were\n   * produced by the stock English tokenizer. Corpora whose content the stock\n   * tokenizer and this tokenizer treat identically (pure legacy Latin) need\n   * no re-indexing — only the version marker is rewritten for them.\n   */\n  private async applyTokenizerVersion(db: any, collection: string): Promise<any> {\n    const persistedLanguage =\n      db?.tokenizer?.language && typeof db.tokenizer.language === \"string\"\n        ? db.tokenizer.language\n        : \"\";\n    if (this.cjkSegmentationEnabled) {\n      db.tokenizer = createCjkCapableTokenizer(this.oramaModule);\n      if (persistedLanguage === ORAMA_CJK_TOKENIZER_LANGUAGE) return db;\n    } else {\n      // Segmentation disabled: a database persisted under the CJK tokenizer\n      // keeps its CJK terms unless it is re-indexed under the stock one, so\n      // downgrade it instead of returning it unchanged (issue #2187 round 2).\n      if (persistedLanguage !== ORAMA_CJK_TOKENIZER_LANGUAGE) return db;\n      db.tokenizer = this.oramaModule.components.tokenizer.createTokenizer({ language: \"english\" });\n    }\n\n\n    try {\n      const { search: oramaSearch, count, update: oramaUpdate } = this.oramaModule;\n      const existingCount = await count(db);\n      if (existingCount === 0) {\n        await this.persistDbForCollection(db, collection);\n        return db;\n      }\n      const allHits = await oramaSearch(db, { term: \"\", limit: existingCount + 100 });\n      const hits = allHits.hits ?? [];\n      const hasNonLegacyContent = hits.some((hit: any) => {\n        const doc = this.getStoredDocument(db, hit);\n        // Orama full-text queries every string field by default, so every\n        // indexed string field gates the rebuild — non-Latin text that only\n        // rides in snippet/path/id would otherwise keep stale terms\n        // (issue #2187 review round 2).\n        for (const field of [doc.content, doc.snippet, doc.path, doc.id, doc.vectorProvider]) {\n          if (typeof field === \"string\" && containsNonLegacyTokenizerChars(field)) return true;\n        }\n        return false;\n      });\n      if (hasNonLegacyContent) {\n        // Re-index every document in place (Orama update is remove+insert,\n        // so content is re-tokenized; vectors are carried through). A single\n        // failing document must not block the rest of the corpus: it keeps\n        // its pre-upgrade terms, which is no worse than before the upgrade.\n        for (const hit of hits) {\n          try {\n            const doc = this.getStoredDocument(db, hit);\n            const vector = this.getStoredVector(db, hit, doc);\n            await oramaUpdate(db, hit.id, {\n              id: typeof doc.id === \"string\" && doc.id.length > 0 ? doc.id : String(hit.id),\n              path: typeof doc.path === \"string\" ? doc.path : \"\",\n              content: typeof doc.content === \"string\" ? doc.content : \"\",\n              snippet:\n                typeof doc.snippet === \"string\"\n                  ? doc.snippet\n                  : typeof doc.content === \"string\"\n                    ? doc.content.slice(0, 200)\n                    : \"\",\n              vectorProvider: typeof doc.vectorProvider === \"string\" ? doc.vectorProvider : \"\",\n              vector: vector ?? this.zeroVector(),\n            });\n          } catch (docErr) {\n            log.debug(\n              `OramaBackend tokenizer-version re-index skipped document ${hit.id} in ${collection}: ${docErr}`,\n            );\n          }\n        }\n      }\n      await this.persistDbForCollection(db, collection);\n    } catch (err) {\n      log.debug(`OramaBackend tokenizer-version rebuild for ${collection} failed: ${err}`);\n    }\n    return db;\n  }\n\n  private async migrateLegacyVectorProviderSchema(db: any, collection: string): Promise<any> {\n    const { search: oramaSearch, count, insert } = this.oramaModule;\n    const existingCount = await count(db);\n    if (existingCount === 0) {\n      const migrated = await this.createDb();\n      await this.persistDbForCollection(migrated, collection);\n      return migrated;\n    }\n\n    const allHits = await oramaSearch(db, { term: \"\", limit: existingCount + 100 });\n    const hits = allHits.hits ?? [];\n    const needsMigration = hits.some((hit: any) =>\n      typeof hit.document?.vectorProvider !== \"string\"\n    );\n    if (!needsMigration) return db;\n\n    const migrated = await this.createDb();\n    for (const hit of hits) {\n      const doc = this.getStoredDocument(db, hit);\n      const vector = this.getStoredVector(db, hit, doc);\n      const payload: Record<string, unknown> = {\n        id: typeof doc.id === \"string\" && doc.id.length > 0 ? doc.id : String(hit.id),\n        path: typeof doc.path === \"string\" ? doc.path : \"\",\n        content: typeof doc.content === \"string\" ? doc.content : \"\",\n        snippet:\n          typeof doc.snippet === \"string\"\n            ? doc.snippet\n            : typeof doc.content === \"string\"\n              ? doc.content.slice(0, 200)\n              : \"\",\n        vectorProvider:\n          typeof doc.vectorProvider === \"string\" ? doc.vectorProvider : \"\",\n      };\n      if (vector) {\n        payload.vector = vector;\n      } else {\n        payload.vector = this.zeroVector();\n      }\n      await insert(migrated, payload);\n    }\n    await this.persistDbForCollection(migrated, collection);\n    return migrated;\n  }\n\n  private async persistDbForCollection(db: any, collection: string): Promise<void> {\n    const data = await this.persistModule.persist(db, \"json\");\n    const filePath = this.dbFilePath(collection);\n    await mkdir(path.dirname(filePath), { recursive: true });\n    const tempPath = path.join(\n      path.dirname(filePath),\n      `.${path.basename(filePath)}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`,\n    );\n    try {\n      await writeFile(tempPath, data, \"utf-8\");\n      await rename(tempPath, filePath);\n    } catch (err) {\n      await rm(tempPath, { force: true }).catch(() => undefined);\n      throw err;\n    }\n  }\n\n  private dbFilePath(collection: string): string {\n    return resolveOramaCollectionDbFilePath(this.dbPath, collection);\n  }\n\n  private async listDbFiles(): Promise<string[]> {\n    try {\n      const entries = await readdir(this.dbPath);\n      return entries\n        .filter((e) => e.endsWith(\".msp\"))\n        .map((e) => path.join(this.dbPath, e));\n    } catch {\n      return [];\n    }\n  }\n\n  private async loadDbFromFile(filePath: string): Promise<any> {\n    try {\n      await this.ensureModules();\n      const raw = await readFile(filePath, \"utf-8\");\n      const collection = path.basename(filePath, \".msp\");\n      return await this.applyTokenizerVersion(\n        await this.migrateLegacyVectorProviderSchema(\n          await this.persistModule.restore(\"json\", raw),\n          collection,\n        ),\n        collection,\n      );\n    } catch (err) {\n      log.debug(`OramaBackend failed to load ${filePath}: ${err}`);\n      return null;\n    }\n  }\n\n  private async searchDb(\n    db: any,\n    query: string,\n    mode: \"fulltext\" | \"vector\" | \"hybrid\",\n    limit: number,\n    execution?: SearchExecutionOptions,\n  ): Promise<SearchResult[]> {\n    const { search: oramaSearch } = this.oramaModule;\n\n    try {\n      throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);\n      let searchParams: any;\n\n      if (mode === \"fulltext\") {\n        searchParams = { term: query, limit };\n      } else if (mode === \"vector\") {\n        const embedResult = await this.resolveCompatibleQueryEmbedding(db, query, execution);\n        throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);\n        if (!embedResult) {\n          // Fall back to fulltext if no embeddings available\n          searchParams = { term: query, limit };\n        } else {\n          searchParams = { mode: \"vector\", vector: { value: embedResult.vector, property: \"vector\" }, limit };\n        }\n      } else {\n        // hybrid\n        const embedResult = await this.resolveCompatibleQueryEmbedding(db, query, execution);\n        throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);\n        if (!embedResult) {\n          searchParams = { term: query, limit };\n        } else {\n          searchParams = { mode: \"hybrid\", term: query, vector: { value: embedResult.vector, property: \"vector\" }, limit };\n        }\n      }\n\n      throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);\n      const result = await oramaSearch(db, searchParams);\n      throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);\n      return (result.hits ?? []).map((hit: any) => ({\n        docid: hit.document?.id ?? \"\",\n        path: hit.document?.path ?? \"\",\n        snippet: hit.document?.snippet ?? hit.document?.content?.slice(0, 200) ?? \"\",\n        score: hit.score ?? 0,\n      }));\n    } catch (err) {\n      log.debug(`OramaBackend search (${mode}) failed: ${err}`);\n      if (!isSearchAborted(execution)) {\n        reportSearchDegradation(execution, {\n          backend: \"orama\",\n          code: \"backend_error\",\n          detail: err instanceof Error ? err.name : typeof err,\n        });\n      }\n      return [];\n    }\n  }\n\n  private async resolveCompatibleQueryEmbedding(\n    db: any,\n    query: string,\n    execution?: SearchExecutionOptions,\n  ): Promise<EmbedWithProviderResult | null> {\n    const embedResult = await this.embedHelper.embedWithProvider(query, { signal: execution?.signal });\n    throwIfSearchAborted(execution, \"OramaBackend query embedding aborted\");\n    if (!embedResult || !this.isExpectedDimensionVector(embedResult.vector)) return null;\n\n    const storedProviderIdentity = await this.findCompatibleStoredVectorProvider(db, execution);\n    if (!storedProviderIdentity) {\n      this.rememberVectorProviderCompatibility(db, embedResult.providerIdentity, false);\n      return null;\n    }\n    if (storedProviderIdentity === embedResult.providerIdentity) return embedResult;\n\n    const fallbackEmbed = await this.embedQueryWithStoredFallbackProvider(query, storedProviderIdentity, execution);\n    throwIfSearchAborted(execution, \"OramaBackend fallback query embedding aborted\");\n    if (\n      fallbackEmbed &&\n      fallbackEmbed.providerIdentity === storedProviderIdentity &&\n      this.isExpectedDimensionVector(fallbackEmbed.vector)\n    ) {\n      return fallbackEmbed;\n    }\n\n    this.rememberVectorProviderCompatibility(db, embedResult.providerIdentity, false);\n    return null;\n  }\n\n  private async embedQueryWithStoredFallbackProvider(\n    query: string,\n    providerIdentity: EmbedProviderIdentity,\n    execution?: SearchExecutionOptions,\n  ): Promise<EmbedWithProviderResult | null> {\n    const embedWithIdentity = (this.embedHelper as unknown as {\n      embedWithFallbackProviderIdentity?: (\n        text: string,\n        identity: EmbedProviderIdentity,\n        options?: { signal?: AbortSignal },\n      ) => Promise<EmbedWithProviderResult | null>;\n    }).embedWithFallbackProviderIdentity;\n    if (typeof embedWithIdentity !== \"function\") return null;\n    return embedWithIdentity.call(this.embedHelper, query, providerIdentity, { signal: execution?.signal });\n  }\n\n  private async findCompatibleStoredVectorProvider(\n    db: any,\n    execution?: SearchExecutionOptions,\n  ): Promise<EmbedProviderIdentity | null> {\n    const { search: oramaSearch, count } = this.oramaModule;\n    try {\n      const cached = this.vectorProviderCompatibility.get(db);\n      if (cached?.compatible) return cached.providerIdentity;\n      const existingCount = await count(db);\n      if (existingCount === 0) return null;\n      const allHits = await oramaSearch(db, {\n        term: \"\",\n        limit: existingCount + 100,\n        properties: [\"vectorProvider\"],\n      });\n      let providerIdentity: EmbedProviderIdentity | null = null;\n      let compatible = (allHits.hits ?? []).length > 0;\n      for (const hit of allHits.hits ?? []) {\n        throwIfSearchAborted(execution, \"OramaBackend vector provider check aborted\");\n        const doc = this.getStoredDocument(db, hit);\n        if (\n          typeof doc.vectorProvider !== \"string\" ||\n          doc.vectorProvider.length === 0 ||\n          !this.isCompatibleStoredVector(this.getStoredVector(db, hit, doc))\n        ) {\n          compatible = false;\n          break;\n        }\n        if (providerIdentity && doc.vectorProvider !== providerIdentity) {\n          compatible = false;\n          break;\n        }\n        providerIdentity = doc.vectorProvider as EmbedProviderIdentity;\n      }\n      if (compatible && providerIdentity) {\n        this.vectorProviderCompatibility.set(db, {\n          providerIdentity,\n          compatible: true,\n        });\n        return providerIdentity;\n      }\n      return null;\n    } catch (err) {\n      if (isSearchAborted(execution)) throw err;\n      log.debug(`OramaBackend stored vector provider check failed: ${err}`);\n      return null;\n    }\n  }\n\n  private async dbHasCompatibleVectors(\n    db: any,\n    providerIdentity: EmbedProviderIdentity,\n    execution?: SearchExecutionOptions,\n  ): Promise<boolean> {\n    const { search: oramaSearch, count } = this.oramaModule;\n    try {\n      const cached = this.vectorProviderCompatibility.get(db);\n      if (cached?.providerIdentity === providerIdentity) return cached.compatible;\n      const existingCount = await count(db);\n      if (existingCount === 0) return false;\n      const allHits = await oramaSearch(db, {\n        term: \"\",\n        limit: existingCount + 100,\n        properties: [\"vectorProvider\"],\n      });\n      let compatible = (allHits.hits ?? []).length > 0;\n      for (const hit of allHits.hits ?? []) {\n        throwIfSearchAborted(execution, \"OramaBackend vector provider check aborted\");\n        const doc = this.getStoredDocument(db, hit);\n        if (\n          doc.vectorProvider !== providerIdentity ||\n          !this.isCompatibleStoredVector(this.getStoredVector(db, hit, doc))\n        ) {\n          compatible = false;\n          break;\n        }\n      }\n      this.vectorProviderCompatibility.set(db, { providerIdentity, compatible });\n      return compatible;\n    } catch (err) {\n      if (isSearchAborted(execution)) throw err;\n      log.debug(`OramaBackend vector provider check failed: ${err}`);\n      return false;\n    }\n  }\n\n  private rememberVectorProviderCompatibility(\n    db: unknown,\n    providerIdentity: EmbedProviderIdentity | null,\n    compatible: boolean,\n  ): void {\n    if (!db || typeof db !== \"object\") return;\n    if (!providerIdentity) {\n      this.vectorProviderCompatibility.delete(db);\n      return;\n    }\n    this.vectorProviderCompatibility.set(db, { providerIdentity, compatible });\n  }\n\n  private getStoredDocument(db: any, hit: any): Record<string, unknown> {\n    const internalId = this.getInternalDocumentId(db, hit);\n    const internalDoc =\n      internalId !== undefined && internalId !== null\n        ? db?.data?.docs?.docs?.[String(internalId)]\n        : undefined;\n    if (internalDoc && typeof internalDoc === \"object\") {\n      return internalDoc as Record<string, unknown>;\n    }\n    return hit?.document && typeof hit.document === \"object\"\n      ? hit.document as Record<string, unknown>\n      : {};\n  }\n\n  private getStoredVector(db: any, hit: any, doc: Record<string, unknown>): number[] | null {\n    const documentVector = this.normalizeStoredVector(doc.vector);\n    if (documentVector) return documentVector;\n    const internalId = this.getInternalDocumentId(db, hit);\n    if (internalId === undefined || internalId === null) return null;\n    const vectorEntry = db?.data?.index?.vectorIndexes?.vector?.node?.vectors?.get?.(internalId);\n    const vector = Array.isArray(vectorEntry) ? vectorEntry[1] : vectorEntry;\n    return this.normalizeStoredVector(vector);\n  }\n\n  private getInternalDocumentId(db: any, hit: any): unknown {\n    const publicId =\n      typeof hit?.id === \"string\"\n        ? hit.id\n        : typeof hit?.document?.id === \"string\"\n          ? hit.document.id\n          : undefined;\n    return publicId && typeof db?.internalDocumentIDStore?.idToInternalId?.get === \"function\"\n      ? db.internalDocumentIDStore.idToInternalId.get(publicId)\n      : undefined;\n  }\n\n  private isExpectedDimensionVector(vector: number[] | null | undefined): vector is number[] {\n    return Array.isArray(vector) && vector.length === this.embeddingDimension;\n  }\n\n  private isCompatibleStoredVector(vector: unknown): vector is number[] {\n    if (!vector || typeof vector !== \"object\") return false;\n    const arr = Array.from(vector as ArrayLike<number>);\n    return (\n      arr.length === this.embeddingDimension &&\n      arr.every((value) => Number.isFinite(value)) &&\n      arr.some((value) => value !== 0)\n    );\n  }\n\n  private zeroVector(): number[] {\n    return Array.from({ length: this.embeddingDimension }, () => 0);\n  }\n\n  private normalizeStoredVector(vector: unknown): number[] | null {\n    const values =\n      Array.isArray(vector)\n        ? vector\n        : ArrayBuffer.isView(vector) && !(vector instanceof DataView)\n          ? Array.from(vector as unknown as ArrayLike<unknown>)\n          : null;\n    if (!values || values.length !== this.embeddingDimension) return null;\n    const normalized = values.map((value) => Number(value));\n    return normalized.every((value) => Number.isFinite(value)) ? normalized : null;\n  }\n}\n","import { expandUnsegmentableRecallNGrams, isUnsegmentableRecallChar } from \"../recall-tokenization.js\";\n\n/**\n * Language marker persisted inside Orama index files (`.msp`). Orama persists\n * `tokenizer.language` with the index, so this doubles as the tokenization\n * version: `OramaBackend` detects any other marker on restore and rebuilds\n * the full-text index (issue #2187). Bump the suffix whenever tokenization\n * changes.\n */\nexport const ORAMA_CJK_TOKENIZER_LANGUAGE = \"english+cjk-v1\";\n\n/** Minimal structural surface of `@orama/orama` this module consumes. */\nexport interface OramaTokenizerComponents {\n  components: {\n    tokenizer: {\n      createTokenizer: (config: { language: string }) => OramaTokenizer;\n    };\n  };\n}\n\nexport interface OramaTokenizer {\n  language: string;\n  normalizationCache: Map<string, string>;\n  tokenize: (raw: string, language?: string, prop?: string, withCache?: boolean) => string[];\n}\n\nconst THAI_SCRIPT_CHAR = /[\\p{Script=Thai}]/u;\nconst WORD_CHAR = /[\\p{L}\\p{N}\\p{M}]/u;\nconst COMBINING_MARK = /\\p{M}/u;\nconst WHITESPACE_CHAR = /\\s/u;\n/**\n * Chars the stock English tokenizer keeps inside a token. Input made only of\n * these (plus whitespace) is delegated to the stock tokenizer byte-for-byte so\n * existing English indexes stay term-compatible.\n */\nconst LEGACY_TOKENIZER_CHAR = /[A-Za-zàèéìòóù0-9_'-]/u;\n\n/**\n * Space-free scripts: CJK per the shared recall segmentation strategy, plus\n * Thai. Runs in these scripts are indexed as character n-grams so phrase\n * queries match without word boundaries.\n */\nexport function isSpaceFreeScriptChar(char: string): boolean {\n  return isUnsegmentableRecallChar(char) || THAI_SCRIPT_CHAR.test(char);\n}\n\n/**\n * True when tokenizing the value differs from the stock English tokenizer —\n * i.e. the value contains characters outside the stock keep-set that this\n * tokenizer still indexes as token material: word characters (whole-word\n * non-ASCII scripts), combining marks (attached to runs), and space-free\n * scripts (CJK/Thai n-grams). Symbols and punctuation that are separators in\n * BOTH tokenizers do not count. `OramaBackend` uses this to decide whether a\n * stale pre-CJK index needs re-indexing on upgrade.\n */\nexport function containsNonLegacyTokenizerChars(value: string): boolean {\n  for (const ch of value) {\n    if (LEGACY_TOKENIZER_CHAR.test(ch) || WHITESPACE_CHAR.test(ch)) continue;\n    if (WORD_CHAR.test(ch) || COMBINING_MARK.test(ch) || isSpaceFreeScriptChar(ch)) {\n      return true;\n    }\n  }\n  return false;\n}\n\n/**\n * Orama tokenizer component that segments space-free scripts (CJK, Thai).\n *\n * - CJK/Thai runs expand to the same n-gram set the recall query-side\n *   tokenizer (`recall-tokenization.ts`) produces, so index-side and\n *   query-side tokens agree.\n * - Other non-Latin scripts (Hangul, Cyrillic, Greek, Arabic, ...) are kept\n *   as whole words instead of being dropped by the English splitter.\n * - Legacy Latin content tokenizes exactly like the stock English tokenizer;\n *   content with other characters keeps every stock token and additionally\n *   indexes non-ASCII words whole (e.g. \"über\" adds \"über\" beside \"ber\").\n */\nexport function createCjkCapableTokenizer(oramaModule: OramaTokenizerComponents): OramaTokenizer {\n  const base = oramaModule.components.tokenizer.createTokenizer({ language: \"english\" });\n\n  const tokenize = (raw: string, _language?: string, prop?: string, withCache?: boolean): string[] => {\n    if (typeof raw !== \"string\") return [];\n    const normalized = raw.normalize(\"NFC\");\n\n    if (!containsNonLegacyTokenizerChars(normalized)) {\n      return base.tokenize(normalized, \"english\", prop, withCache);\n    }\n\n    // The stock tokenizer treats every non-legacy char as a separator, so its\n    // output over the raw input already carries the legacy-Latin tokens\n    // (e.g. the \"nebula-472\" in \"Nebula-472 東京都庁\").\n    const tokens = base.tokenize(normalized, \"english\", prop, withCache);\n    const seen = new Set(tokens);\n\n    const pushToken = (token: string) => {\n      if (token && !seen.has(token)) {\n        seen.add(token);\n        tokens.push(token);\n      }\n    };\n\n    let spaceFreeRun = \"\";\n    let wordRun = \"\";\n    const flushSpaceFreeRun = () => {\n      if (!spaceFreeRun) return;\n      for (const token of expandUnsegmentableRecallNGrams(spaceFreeRun)) {\n        pushToken(token);\n      }\n      spaceFreeRun = \"\";\n    };\n    const flushWordRun = () => {\n      if (!wordRun) return;\n      pushToken(wordRun.toLowerCase());\n      wordRun = \"\";\n    };\n\n    for (const ch of normalized) {\n      if (isSpaceFreeScriptChar(ch)) {\n        flushWordRun();\n        spaceFreeRun += ch;\n      } else if (COMBINING_MARK.test(ch)) {\n        if (spaceFreeRun) spaceFreeRun += ch;\n        else if (wordRun) wordRun += ch;\n      } else if (WORD_CHAR.test(ch) && !LEGACY_TOKENIZER_CHAR.test(ch)) {\n        flushSpaceFreeRun();\n        wordRun += ch;\n      } else {\n        flushSpaceFreeRun();\n        flushWordRun();\n      }\n    }\n    flushSpaceFreeRun();\n    flushWordRun();\n\n    return tokens;\n  };\n\n  return {\n    language: ORAMA_CJK_TOKENIZER_LANGUAGE,\n    normalizationCache: base.normalizationCache,\n    tokenize,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,OAAO,UAAU;AACjB,SAAS,OAAO,SAAS,UAAU,QAAQ,IAAI,iBAAiB;;;ACQzD,IAAM,+BAA+B;AAiB5C,IAAM,mBAAmB;AACzB,IAAM,YAAY;AAClB,IAAM,iBAAiB;AACvB,IAAM,kBAAkB;AAMxB,IAAM,wBAAwB;AAOvB,SAAS,sBAAsB,MAAuB;AAC3D,SAAO,0BAA0B,IAAI,KAAK,iBAAiB,KAAK,IAAI;AACtE;AAWO,SAAS,gCAAgC,OAAwB;AACtE,aAAW,MAAM,OAAO;AACtB,QAAI,sBAAsB,KAAK,EAAE,KAAK,gBAAgB,KAAK,EAAE,EAAG;AAChE,QAAI,UAAU,KAAK,EAAE,KAAK,eAAe,KAAK,EAAE,KAAK,sBAAsB,EAAE,GAAG;AAC9E,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAcO,SAAS,0BAA0B,aAAuD;AAC/F,QAAM,OAAO,YAAY,WAAW,UAAU,gBAAgB,EAAE,UAAU,UAAU,CAAC;AAErF,QAAM,WAAW,CAAC,KAAa,WAAoB,MAAe,cAAkC;AAClG,QAAI,OAAO,QAAQ,SAAU,QAAO,CAAC;AACrC,UAAM,aAAa,IAAI,UAAU,KAAK;AAEtC,QAAI,CAAC,gCAAgC,UAAU,GAAG;AAChD,aAAO,KAAK,SAAS,YAAY,WAAW,MAAM,SAAS;AAAA,IAC7D;AAKA,UAAM,SAAS,KAAK,SAAS,YAAY,WAAW,MAAM,SAAS;AACnE,UAAM,OAAO,IAAI,IAAI,MAAM;AAE3B,UAAM,YAAY,CAAC,UAAkB;AACnC,UAAI,SAAS,CAAC,KAAK,IAAI,KAAK,GAAG;AAC7B,aAAK,IAAI,KAAK;AACd,eAAO,KAAK,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI,eAAe;AACnB,QAAI,UAAU;AACd,UAAM,oBAAoB,MAAM;AAC9B,UAAI,CAAC,aAAc;AACnB,iBAAW,SAAS,gCAAgC,YAAY,GAAG;AACjE,kBAAU,KAAK;AAAA,MACjB;AACA,qBAAe;AAAA,IACjB;AACA,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,QAAS;AACd,gBAAU,QAAQ,YAAY,CAAC;AAC/B,gBAAU;AAAA,IACZ;AAEA,eAAW,MAAM,YAAY;AAC3B,UAAI,sBAAsB,EAAE,GAAG;AAC7B,qBAAa;AACb,wBAAgB;AAAA,MAClB,WAAW,eAAe,KAAK,EAAE,GAAG;AAClC,YAAI,aAAc,iBAAgB;AAAA,iBACzB,QAAS,YAAW;AAAA,MAC/B,WAAW,UAAU,KAAK,EAAE,KAAK,CAAC,sBAAsB,KAAK,EAAE,GAAG;AAChE,0BAAkB;AAClB,mBAAW;AAAA,MACb,OAAO;AACL,0BAAkB;AAClB,qBAAa;AAAA,MACf;AAAA,IACF;AACA,sBAAkB;AAClB,iBAAa;AAEb,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,UAAU;AAAA,IACV,oBAAoB,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;AD5GA,IAAM,oCAAoC;AAE1C,SAAS,aAAa,QAAgB,OAAwB;AAC5D,QAAM,WAAW,KAAK,SAAS,QAAQ,KAAK;AAC5C,SAAO,aAAa,MAAO,CAAC,SAAS,WAAW,IAAI,KAAK,CAAC,KAAK,WAAW,QAAQ;AACpF;AAEO,SAAS,iCAAiC,QAAgB,YAA4B;AAC3F,MAAI,CAAC,kCAAkC,KAAK,UAAU,GAAG;AACvD,UAAM,IAAI;AAAA,MACR,iCAAiC,KAAK,UAAU,UAAU,CAAC;AAAA,IAE7D;AAAA,EACF;AACA,QAAM,iBAAiB,KAAK,QAAQ,MAAM;AAC1C,QAAM,WAAW,KAAK,QAAQ,gBAAgB,GAAG,UAAU,MAAM;AACjE,MAAI,CAAC,aAAa,gBAAgB,QAAQ,GAAG;AAC3C,UAAM,IAAI;AAAA,MACR,qCAAqC,KAAK,UAAU,UAAU,CAAC;AAAA,IACjE;AAAA,EACF;AACA,SAAO;AACT;AAQO,IAAM,eAAN,MAA4C;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT,YAAY;AAAA,EACZ,KAAU;AAAA,EACV,cAAmB;AAAA,EACnB,gBAAqB;AAAA,EACZ,8BAA8B,oBAAI,QAGjD;AAAA,EAEF,YAAY,MAA2B;AACrC,SAAK,SAAS,KAAK;AACnB,SAAK,aAAa,KAAK;AACvB,SAAK,cAAc,KAAK;AACxB,SAAK,YAAY,KAAK;AACtB,SAAK,qBAAqB,KAAK;AAC/B,SAAK,yBAAyB,KAAK,2BAA2B;AAAA,EAChE;AAAA,EAEA,MAAM,QAA0B;AAC9B,QAAI;AACF,YAAM,KAAK,cAAc;AACzB,YAAM,KAAK,SAAS;AACpB,WAAK,YAAY;AACjB,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,UAAI,MAAM,8BAA8B,GAAG,EAAE;AAC7C,WAAK,YAAY;AACjB,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,cAAsB;AACpB,WAAO,2BAA2B,KAAK,SAAS,WAAW,KAAK,MAAM;AAAA,EACxE;AAAA,EAEA,MAAM,OACJ,OACA,aACA,YACA,UACA,WACyB;AACzB,WAAO,KAAK,aAAa,OAAO,aAAa,YAAY,SAAS;AAAA,EACpE;AAAA,EAEA,MAAM,aAAa,OAAe,YAAqB,WAA6D;AAClH,UAAM,QAAQ,cAAc;AAC5B,QAAI,CAAC,KAAK,UAAW,QAAO,CAAC;AAC7B,QAAI;AACF,2BAAqB,WAAW,oCAAoC;AACpE,YAAM,QAAQ,MAAM,KAAK,YAAY;AACrC,YAAM,aAA6B,CAAC;AACpC,iBAAW,QAAQ,OAAO;AACxB,6BAAqB,WAAW,oCAAoC;AACpE,cAAM,KAAK,MAAM,KAAK,eAAe,IAAI;AACzC,YAAI,CAAC,GAAI;AACT,cAAM,UAAU,MAAM,KAAK,SAAS,IAAI,OAAO,UAAU,OAAO,SAAS;AACzE,mBAAW,KAAK,GAAG,OAAO;AAAA,MAC5B;AACA,iBAAW,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAC3C,aAAO,WAAW,MAAM,GAAG,KAAK;AAAA,IAClC,SAAS,KAAK;AACZ,UAAI,MAAM,qCAAqC,GAAG,EAAE;AACpD,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,WAAW,OAAe,YAAqB,YAAqB,WAA6D;AACrI,QAAI,gBAAgB,SAAS,EAAG,QAAO,CAAC;AACxC,UAAM,KAAK,MAAM,KAAK,sBAAsB,cAAc,KAAK,UAAU;AACzE,QAAI,gBAAgB,SAAS,EAAG,QAAO,CAAC;AACxC,QAAI,CAAC,GAAI,QAAO,CAAC;AACjB,WAAO,KAAK,SAAS,IAAI,OAAO,YAAY,cAAc,IAAI,SAAS;AAAA,EACzE;AAAA,EAEA,MAAM,aAAa,OAAe,YAAqB,YAAqB,WAA6D;AACvI,QAAI,gBAAgB,SAAS,EAAG,QAAO,CAAC;AACxC,UAAM,KAAK,MAAM,KAAK,sBAAsB,cAAc,KAAK,UAAU;AACzE,QAAI,gBAAgB,SAAS,EAAG,QAAO,CAAC;AACxC,QAAI,CAAC,GAAI,QAAO,CAAC;AACjB,WAAO,KAAK,SAAS,IAAI,OAAO,UAAU,cAAc,IAAI,SAAS;AAAA,EACvE;AAAA,EAEA,MAAM,aAAa,OAAe,YAAqB,YAAqB,WAA6D;AACvI,QAAI,gBAAgB,SAAS,EAAG,QAAO,CAAC;AACxC,UAAM,KAAK,MAAM,KAAK,sBAAsB,cAAc,KAAK,UAAU;AACzE,QAAI,gBAAgB,SAAS,EAAG,QAAO,CAAC;AACxC,QAAI,CAAC,GAAI,QAAO,CAAC;AACjB,WAAO,KAAK,SAAS,IAAI,OAAO,UAAU,cAAc,IAAI,SAAS;AAAA,EACvE;AAAA,EAEA,MAAM,OAAO,WAAmD;AAC9D,UAAM,KAAK,iBAAiB,KAAK,YAAY,SAAS;AAAA,EACxD;AAAA,EAEA,MAAM,iBAAiB,YAAoB,WAAmD;AAC5F,UAAM,KAAK,wBAAwB,YAAY,KAAK,WAAW,SAAS;AAAA,EAC1E;AAAA,EAEA,MAAM,wBAAwB,YAAoB,WAAmB,WAAmD;AACtH,QAAI,gBAAgB,SAAS,EAAG;AAChC,UAAM,KAAK,MAAM,KAAK,sBAAsB,UAAU;AACtD,QAAI,gBAAgB,SAAS,EAAG;AAChC,QAAI,CAAC,GAAI;AACT,UAAM,EAAE,QAAQ,aAAa,QAAQ,QAAQ,OAAO,QAAQ,IAAI,KAAK;AAErE,UAAM,OAAO,MAAM,cAAc,SAAS;AAC1C,QAAI,gBAAgB,SAAS,EAAG;AAChC,UAAM,SAAS,IAAI,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AACpD,UAAM,EAAE,QAAQ,YAAY,IAAI,KAAK;AAErC,UAAM,4BAA4B,KAAK,YAAY,oBAAoB;AACvE,QAAI,oBAAoB,CAAC,CAAC,6BAA6B,KAAK,SAAS;AAErE,UAAM,eAAe,oBAAI,IAItB;AACH,UAAM,gBAAgB,MAAM,MAAM,EAAE;AACpC,QAAI,gBAAgB,GAAG;AACrB,YAAM,UAAU,MAAM,YAAY,IAAI;AAAA,QACpC,MAAM;AAAA,QACN,OAAO,gBAAgB;AAAA,MACzB,CAAC;AACD,iBAAW,OAAO,QAAQ,MAAM;AAC9B,YAAI,gBAAgB,SAAS,EAAG;AAChC,cAAM,iBACJ,OAAO,YAAY,aACf,MAAM,QAAQ,IAAI,IAAI,EAAE,IACxB,IAAI;AACV,cAAM,WAAW,kBAAkB,IAAI,YAAY,CAAC;AACpD,YAAI,CAAC,OAAO,IAAI,SAAS,EAAE,GAAG;AAC5B,gBAAM,OAAO,IAAI,IAAI,EAAE;AAAA,QACzB,OAAO;AACL,uBAAa,IAAI,SAAS,IAAI;AAAA,YAC5B,YAAY,IAAI;AAAA,YAChB,QAAQ,KAAK,sBAAsB,SAAS,MAAM,KAAK;AAAA,YACvD,gBACE,OAAO,SAAS,mBAAmB,WAC/B,SAAS,iBACT;AAAA,UACR,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,eAAW,OAAO,MAAM;AACtB,UAAI,gBAAgB,SAAS,EAAG;AAChC,YAAM,WAAW,aAAa,IAAI,IAAI,KAAK;AAC3C,UAAI,UAAU;AACZ,cAAM,UAAmC;AAAA,UACvC,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV,SAAS,IAAI;AAAA,UACb,SAAS,IAAI;AAAA,QACf;AACA,cAAM,8BACJ,CAAC,CAAC,6BACF,SAAS,mBAAmB;AAC9B,YAAI,6BAA6B;AAC/B,cAAI,KAAK,yBAAyB,SAAS,MAAM,GAAG;AAClD,oBAAQ,SAAS,SAAS;AAC1B,oBAAQ,iBAAiB,SAAS,kBAAkB;AAAA,UACtD,OAAO;AACL,oBAAQ,SAAS,KAAK,WAAW;AACjC,oBAAQ,iBAAiB;AACzB,gCAAoB;AAAA,UACtB;AAAA,QACF,WAAW,CAAC,6BAA6B,KAAK,yBAAyB,SAAS,MAAM,GAAG;AACvF,kBAAQ,SAAS,SAAS;AAC1B,kBAAQ,iBAAiB,SAAS,kBAAkB;AACpD,8BAAoB;AAAA,QACtB,OAAO;AACL,kBAAQ,SAAS,KAAK,WAAW;AACjC,kBAAQ,iBAAiB;AACzB,8BAAoB;AAAA,QACtB;AACA,YAAI;AACF,gBAAM,YAAY,IAAI,SAAS,YAAY,OAAO;AAAA,QACpD,QAAQ;AACN,8BAAoB;AAAA,QAEtB;AAAA,MACF,OAAO;AACL,4BAAoB;AACpB,YAAI;AACF,gBAAM,OAAO,IAAI;AAAA,YACf,IAAI,IAAI;AAAA,YACR,MAAM,IAAI;AAAA,YACV,SAAS,IAAI;AAAA,YACb,SAAS,IAAI;AAAA,YACb,QAAQ,KAAK,WAAW;AAAA,YACxB,gBAAgB;AAAA,UAClB,CAAC;AAAA,QACH,QAAQ;AACN,8BAAoB;AAAA,QAEtB;AAAA,MACF;AAAA,IACF;AAEA,QAAI,gBAAgB,SAAS,EAAG;AAChC,UAAM,KAAK,uBAAuB,IAAI,UAAU;AAChD,SAAK;AAAA,MACH;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,gBAAgB,KAAK,UAAU;AAAA,EAC5C;AAAA,EAEA,MAAM,gBAAgB,YAAmC;AACvD,QAAI,CAAC,KAAK,YAAY,YAAY,EAAG;AAErC,UAAM,KAAK,MAAM,KAAK,sBAAsB,UAAU;AACtD,QAAI,CAAC,GAAI;AACT,UAAM,EAAE,QAAQ,aAAa,QAAQ,aAAa,MAAM,IAAI,KAAK;AAEjE,UAAM,gBAAgB,MAAM,MAAM,EAAE;AACpC,QAAI,kBAAkB,EAAG;AAEzB,UAAM,4BAA4B,KAAK,YAAY,oBAAoB;AAEvE,UAAM,UAAU,MAAM,YAAY,IAAI,EAAE,MAAM,IAAI,OAAO,gBAAgB,IAAI,CAAC;AAC9E,UAAM,aAAa,QAAQ,KAAK,OAAO,CAAC,MAAW;AACjD,YAAM,SAAS,KAAK,sBAAsB,EAAE,UAAU,MAAM;AAC5D,aACG,6BACC,EAAE,UAAU,mBAAmB,6BACjC,CAAC,KAAK,yBAAyB,MAAM;AAAA,IAEzC,CAAC;AAED,QAAI,WAAW,WAAW,GAAG;AAC3B,WAAK,oCAAoC,IAAI,2BAA2B,IAAI;AAC5E;AAAA,IACF;AAEA,QAAI,cAAc;AAClB,QAAI,cAAc,MAAM,KAAK,YAAY;AAAA,MACvC,YAAY,IAAI,CAAC,MAAW,EAAE,SAAS,OAAiB;AAAA,IAC1D;AACA,QAAI,CAAC,YAAa;AAClB,QACE,6BACA,YAAY,qBAAqB,2BACjC;AACA,YAAM,4BAA4B,YAAY;AAC9C,YAAM,cAAc,IAAI,IAAI,YAAY,IAAI,CAAC,MAAW,EAAE,EAAE,CAAC;AAC7D,YAAM,sBAAsB,QAAQ,KAAK,OAAO,CAAC,MAAW;AAC1D,cAAM,SAAS,KAAK,sBAAsB,EAAE,UAAU,MAAM;AAC5D,eACE,EAAE,UAAU,mBAAmB,6BAC/B,CAAC,KAAK,yBAAyB,MAAM;AAAA,MAEzC,CAAC;AACD,YAAM,WACJ,oBAAoB,WAAW,YAAY,UAC3C,oBAAoB,MAAM,CAAC,MAAW,YAAY,IAAI,EAAE,EAAE,CAAC;AAC7D,UAAI,CAAC,UAAU;AACb,cAAM,iBAAiB,oBAAoB,IAAI,CAAC,MAAW,EAAE,SAAS,OAAiB;AACvF,cAAM,uBAAuB,MAAM,KAAK,YAAY,uBAAuB,cAAc;AACzF,YAAI,sBAAsB;AACxB,wBAAc;AACd,wBAAc;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AACA,UAAM,EAAE,SAAS,iBAAiB,IAAI;AAEtC,QAAI,cAAc;AAClB,aAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,YAAM,MAAM,QAAQ,CAAC;AACrB,UAAI,CAAC,KAAK,0BAA0B,GAAG,GAAG;AACxC,sBAAc;AACd;AAAA,MACF;AAEA,YAAM,MAAM,YAAY,CAAC,EAAE;AAC3B,YAAM,YAAY,IAAI,YAAY,CAAC,EAAE,IAAI;AAAA,QACvC,IAAI,IAAI;AAAA,QACR,MAAM,IAAI;AAAA,QACV,SAAS,IAAI;AAAA,QACb,SAAS,IAAI;AAAA,QACb,QAAQ;AAAA,QACR,gBAAgB;AAAA,MAClB,CAAC;AAAA,IACH;AAEA,UAAM,KAAK,uBAAuB,IAAI,UAAU;AAChD,QAAI,aAAa;AACf,WAAK,oCAAoC,IAAI,kBAAkB,IAAI;AAAA,IACrE,OAAO;AACL,WAAK,oCAAoC,IAAI,kBAAkB,KAAK;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,MAAM,iBACJ,YACA,uBACA,WACwD;AACxD,UAAM,EAAE,YAAY,WAAW,mBAAmB,IAAI;AAAA,MACpD;AAAA,MACA;AAAA,IACF;AACA,QAAI,gBAAgB,kBAAkB,EAAG,QAAO;AAChD,QAAI;AACF,YAAM,KAAK,cAAc;AACzB,YAAM,KAAK,sBAAsB,cAAc,KAAK,UAAU;AAC9D,aAAO;AAAA,IACT,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAc,gBAA+B;AAC3C,QAAI,KAAK,eAAe,KAAK,cAAe;AAC5C,SAAK,cAAc,MAAM,OAAO,cAAc;AAC9C,SAAK,gBAAgB,MAAM,OAAO,gCAAgC;AAAA,EACpE;AAAA,EAEA,MAAc,WAAyB;AACrC,QAAI,KAAK,GAAI,QAAO,KAAK;AACzB,UAAM,KAAK,cAAc;AAEzB,UAAM,MAAM,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;AAC5C,UAAM,WAAW,KAAK,WAAW,KAAK,UAAU;AAEhD,QAAI;AACJ,QAAI;AACF,YAAM,MAAM,SAAS,UAAU,OAAO;AAAA,IACxC,QAAQ;AAEN,WAAK,KAAK,MAAM,KAAK,SAAS;AAC9B,aAAO,KAAK;AAAA,IACd;AAEA,SAAK,KAAK,MAAM,KAAK;AAAA,MACnB,MAAM,KAAK;AAAA,QACT,MAAM,KAAK,cAAc,QAAQ,QAAQ,GAAG;AAAA,QAC5C,KAAK;AAAA,MACP;AAAA,MACA,KAAK;AAAA,IACP;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,sBAAsB,YAAkC;AAEpE,QAAI,eAAe,KAAK,WAAY,QAAO,KAAK,SAAS;AAEzD,UAAM,KAAK,cAAc;AACzB,UAAM,MAAM,KAAK,QAAQ,EAAE,WAAW,KAAK,CAAC;AAC5C,UAAM,WAAW,KAAK,WAAW,UAAU;AAE3C,QAAI;AACJ,QAAI;AACF,YAAM,MAAM,SAAS,UAAU,OAAO;AAAA,IACxC,QAAQ;AAEN,aAAO,MAAM,KAAK,SAAS;AAAA,IAC7B;AAEA,WAAO,MAAM,KAAK;AAAA,MAChB,MAAM,KAAK;AAAA,QACT,MAAM,KAAK,cAAc,QAAQ,QAAQ,GAAG;AAAA,QAC5C;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,WAAyB;AACrC,UAAM,EAAE,OAAO,IAAI,KAAK;AACxB,UAAM,SAAiC;AAAA,MACrC,IAAI;AAAA,MACJ,MAAM;AAAA,MACN,SAAS;AAAA,MACT,SAAS;AAAA,MACT,gBAAgB;AAAA,MAChB,QAAQ,UAAU,KAAK,kBAAkB;AAAA,IAC3C;AACA,QAAI,CAAC,KAAK,wBAAwB;AAChC,aAAO,MAAM,OAAO,EAAE,OAAO,CAAC;AAAA,IAChC;AACA,WAAO,MAAM,OAAO;AAAA,MAClB;AAAA,MACA,YAAY,EAAE,WAAW,0BAA0B,KAAK,WAAW,EAAE;AAAA,IACvE,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAc,sBAAsB,IAAS,YAAkC;AAC7E,UAAM,oBACJ,IAAI,WAAW,YAAY,OAAO,GAAG,UAAU,aAAa,WACxD,GAAG,UAAU,WACb;AACN,QAAI,KAAK,wBAAwB;AAC/B,SAAG,YAAY,0BAA0B,KAAK,WAAW;AACzD,UAAI,sBAAsB,6BAA8B,QAAO;AAAA,IACjE,OAAO;AAIL,UAAI,sBAAsB,6BAA8B,QAAO;AAC/D,SAAG,YAAY,KAAK,YAAY,WAAW,UAAU,gBAAgB,EAAE,UAAU,UAAU,CAAC;AAAA,IAC9F;AAGA,QAAI;AACF,YAAM,EAAE,QAAQ,aAAa,OAAO,QAAQ,YAAY,IAAI,KAAK;AACjE,YAAM,gBAAgB,MAAM,MAAM,EAAE;AACpC,UAAI,kBAAkB,GAAG;AACvB,cAAM,KAAK,uBAAuB,IAAI,UAAU;AAChD,eAAO;AAAA,MACT;AACA,YAAM,UAAU,MAAM,YAAY,IAAI,EAAE,MAAM,IAAI,OAAO,gBAAgB,IAAI,CAAC;AAC9E,YAAM,OAAO,QAAQ,QAAQ,CAAC;AAC9B,YAAM,sBAAsB,KAAK,KAAK,CAAC,QAAa;AAClD,cAAM,MAAM,KAAK,kBAAkB,IAAI,GAAG;AAK1C,mBAAW,SAAS,CAAC,IAAI,SAAS,IAAI,SAAS,IAAI,MAAM,IAAI,IAAI,IAAI,cAAc,GAAG;AACpF,cAAI,OAAO,UAAU,YAAY,gCAAgC,KAAK,EAAG,QAAO;AAAA,QAClF;AACA,eAAO;AAAA,MACT,CAAC;AACD,UAAI,qBAAqB;AAKvB,mBAAW,OAAO,MAAM;AACtB,cAAI;AACF,kBAAM,MAAM,KAAK,kBAAkB,IAAI,GAAG;AAC1C,kBAAM,SAAS,KAAK,gBAAgB,IAAI,KAAK,GAAG;AAChD,kBAAM,YAAY,IAAI,IAAI,IAAI;AAAA,cAC5B,IAAI,OAAO,IAAI,OAAO,YAAY,IAAI,GAAG,SAAS,IAAI,IAAI,KAAK,OAAO,IAAI,EAAE;AAAA,cAC5E,MAAM,OAAO,IAAI,SAAS,WAAW,IAAI,OAAO;AAAA,cAChD,SAAS,OAAO,IAAI,YAAY,WAAW,IAAI,UAAU;AAAA,cACzD,SACE,OAAO,IAAI,YAAY,WACnB,IAAI,UACJ,OAAO,IAAI,YAAY,WACrB,IAAI,QAAQ,MAAM,GAAG,GAAG,IACxB;AAAA,cACR,gBAAgB,OAAO,IAAI,mBAAmB,WAAW,IAAI,iBAAiB;AAAA,cAC9E,QAAQ,UAAU,KAAK,WAAW;AAAA,YACpC,CAAC;AAAA,UACH,SAAS,QAAQ;AACf,gBAAI;AAAA,cACF,4DAA4D,IAAI,EAAE,OAAO,UAAU,KAAK,MAAM;AAAA,YAChG;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA,YAAM,KAAK,uBAAuB,IAAI,UAAU;AAAA,IAClD,SAAS,KAAK;AACZ,UAAI,MAAM,8CAA8C,UAAU,YAAY,GAAG,EAAE;AAAA,IACrF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,kCAAkC,IAAS,YAAkC;AACzF,UAAM,EAAE,QAAQ,aAAa,OAAO,OAAO,IAAI,KAAK;AACpD,UAAM,gBAAgB,MAAM,MAAM,EAAE;AACpC,QAAI,kBAAkB,GAAG;AACvB,YAAMA,YAAW,MAAM,KAAK,SAAS;AACrC,YAAM,KAAK,uBAAuBA,WAAU,UAAU;AACtD,aAAOA;AAAA,IACT;AAEA,UAAM,UAAU,MAAM,YAAY,IAAI,EAAE,MAAM,IAAI,OAAO,gBAAgB,IAAI,CAAC;AAC9E,UAAM,OAAO,QAAQ,QAAQ,CAAC;AAC9B,UAAM,iBAAiB,KAAK;AAAA,MAAK,CAAC,QAChC,OAAO,IAAI,UAAU,mBAAmB;AAAA,IAC1C;AACA,QAAI,CAAC,eAAgB,QAAO;AAE5B,UAAM,WAAW,MAAM,KAAK,SAAS;AACrC,eAAW,OAAO,MAAM;AACtB,YAAM,MAAM,KAAK,kBAAkB,IAAI,GAAG;AAC1C,YAAM,SAAS,KAAK,gBAAgB,IAAI,KAAK,GAAG;AAChD,YAAM,UAAmC;AAAA,QACvC,IAAI,OAAO,IAAI,OAAO,YAAY,IAAI,GAAG,SAAS,IAAI,IAAI,KAAK,OAAO,IAAI,EAAE;AAAA,QAC5E,MAAM,OAAO,IAAI,SAAS,WAAW,IAAI,OAAO;AAAA,QAChD,SAAS,OAAO,IAAI,YAAY,WAAW,IAAI,UAAU;AAAA,QACzD,SACE,OAAO,IAAI,YAAY,WACnB,IAAI,UACJ,OAAO,IAAI,YAAY,WACrB,IAAI,QAAQ,MAAM,GAAG,GAAG,IACxB;AAAA,QACR,gBACE,OAAO,IAAI,mBAAmB,WAAW,IAAI,iBAAiB;AAAA,MAClE;AACA,UAAI,QAAQ;AACV,gBAAQ,SAAS;AAAA,MACnB,OAAO;AACL,gBAAQ,SAAS,KAAK,WAAW;AAAA,MACnC;AACA,YAAM,OAAO,UAAU,OAAO;AAAA,IAChC;AACA,UAAM,KAAK,uBAAuB,UAAU,UAAU;AACtD,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,uBAAuB,IAAS,YAAmC;AAC/E,UAAM,OAAO,MAAM,KAAK,cAAc,QAAQ,IAAI,MAAM;AACxD,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,UAAM,MAAM,KAAK,QAAQ,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AACvD,UAAM,WAAW,KAAK;AAAA,MACpB,KAAK,QAAQ,QAAQ;AAAA,MACrB,IAAI,KAAK,SAAS,QAAQ,CAAC,IAAI,QAAQ,GAAG,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AAAA,IACjG;AACA,QAAI;AACF,YAAM,UAAU,UAAU,MAAM,OAAO;AACvC,YAAM,OAAO,UAAU,QAAQ;AAAA,IACjC,SAAS,KAAK;AACZ,YAAM,GAAG,UAAU,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM,MAAS;AACzD,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,WAAW,YAA4B;AAC7C,WAAO,iCAAiC,KAAK,QAAQ,UAAU;AAAA,EACjE;AAAA,EAEA,MAAc,cAAiC;AAC7C,QAAI;AACF,YAAM,UAAU,MAAM,QAAQ,KAAK,MAAM;AACzC,aAAO,QACJ,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM,CAAC,EAChC,IAAI,CAAC,MAAM,KAAK,KAAK,KAAK,QAAQ,CAAC,CAAC;AAAA,IACzC,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,MAAc,eAAe,UAAgC;AAC3D,QAAI;AACF,YAAM,KAAK,cAAc;AACzB,YAAM,MAAM,MAAM,SAAS,UAAU,OAAO;AAC5C,YAAM,aAAa,KAAK,SAAS,UAAU,MAAM;AACjD,aAAO,MAAM,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,UACT,MAAM,KAAK,cAAc,QAAQ,QAAQ,GAAG;AAAA,UAC5C;AAAA,QACF;AAAA,QACA;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,+BAA+B,QAAQ,KAAK,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAc,SACZ,IACA,OACA,MACA,OACA,WACyB;AACzB,UAAM,EAAE,QAAQ,YAAY,IAAI,KAAK;AAErC,QAAI;AACF,2BAAqB,WAAW,gBAAgB,IAAI,iBAAiB;AACrE,UAAI;AAEJ,UAAI,SAAS,YAAY;AACvB,uBAAe,EAAE,MAAM,OAAO,MAAM;AAAA,MACtC,WAAW,SAAS,UAAU;AAC5B,cAAM,cAAc,MAAM,KAAK,gCAAgC,IAAI,OAAO,SAAS;AACnF,6BAAqB,WAAW,gBAAgB,IAAI,iBAAiB;AACrE,YAAI,CAAC,aAAa;AAEhB,yBAAe,EAAE,MAAM,OAAO,MAAM;AAAA,QACtC,OAAO;AACL,yBAAe,EAAE,MAAM,UAAU,QAAQ,EAAE,OAAO,YAAY,QAAQ,UAAU,SAAS,GAAG,MAAM;AAAA,QACpG;AAAA,MACF,OAAO;AAEL,cAAM,cAAc,MAAM,KAAK,gCAAgC,IAAI,OAAO,SAAS;AACnF,6BAAqB,WAAW,gBAAgB,IAAI,iBAAiB;AACrE,YAAI,CAAC,aAAa;AAChB,yBAAe,EAAE,MAAM,OAAO,MAAM;AAAA,QACtC,OAAO;AACL,yBAAe,EAAE,MAAM,UAAU,MAAM,OAAO,QAAQ,EAAE,OAAO,YAAY,QAAQ,UAAU,SAAS,GAAG,MAAM;AAAA,QACjH;AAAA,MACF;AAEA,2BAAqB,WAAW,gBAAgB,IAAI,iBAAiB;AACrE,YAAM,SAAS,MAAM,YAAY,IAAI,YAAY;AACjD,2BAAqB,WAAW,gBAAgB,IAAI,iBAAiB;AACrE,cAAQ,OAAO,QAAQ,CAAC,GAAG,IAAI,CAAC,SAAc;AAAA,QAC5C,OAAO,IAAI,UAAU,MAAM;AAAA,QAC3B,MAAM,IAAI,UAAU,QAAQ;AAAA,QAC5B,SAAS,IAAI,UAAU,WAAW,IAAI,UAAU,SAAS,MAAM,GAAG,GAAG,KAAK;AAAA,QAC1E,OAAO,IAAI,SAAS;AAAA,MACtB,EAAE;AAAA,IACJ,SAAS,KAAK;AACZ,UAAI,MAAM,wBAAwB,IAAI,aAAa,GAAG,EAAE;AACxD,UAAI,CAAC,gBAAgB,SAAS,GAAG;AAC/B,gCAAwB,WAAW;AAAA,UACjC,SAAS;AAAA,UACT,MAAM;AAAA,UACN,QAAQ,eAAe,QAAQ,IAAI,OAAO,OAAO;AAAA,QACnD,CAAC;AAAA,MACH;AACA,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,MAAc,gCACZ,IACA,OACA,WACyC;AACzC,UAAM,cAAc,MAAM,KAAK,YAAY,kBAAkB,OAAO,EAAE,QAAQ,WAAW,OAAO,CAAC;AACjG,yBAAqB,WAAW,sCAAsC;AACtE,QAAI,CAAC,eAAe,CAAC,KAAK,0BAA0B,YAAY,MAAM,EAAG,QAAO;AAEhF,UAAM,yBAAyB,MAAM,KAAK,mCAAmC,IAAI,SAAS;AAC1F,QAAI,CAAC,wBAAwB;AAC3B,WAAK,oCAAoC,IAAI,YAAY,kBAAkB,KAAK;AAChF,aAAO;AAAA,IACT;AACA,QAAI,2BAA2B,YAAY,iBAAkB,QAAO;AAEpE,UAAM,gBAAgB,MAAM,KAAK,qCAAqC,OAAO,wBAAwB,SAAS;AAC9G,yBAAqB,WAAW,+CAA+C;AAC/E,QACE,iBACA,cAAc,qBAAqB,0BACnC,KAAK,0BAA0B,cAAc,MAAM,GACnD;AACA,aAAO;AAAA,IACT;AAEA,SAAK,oCAAoC,IAAI,YAAY,kBAAkB,KAAK;AAChF,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,qCACZ,OACA,kBACA,WACyC;AACzC,UAAM,oBAAqB,KAAK,YAM7B;AACH,QAAI,OAAO,sBAAsB,WAAY,QAAO;AACpD,WAAO,kBAAkB,KAAK,KAAK,aAAa,OAAO,kBAAkB,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,EACxG;AAAA,EAEA,MAAc,mCACZ,IACA,WACuC;AACvC,UAAM,EAAE,QAAQ,aAAa,MAAM,IAAI,KAAK;AAC5C,QAAI;AACF,YAAM,SAAS,KAAK,4BAA4B,IAAI,EAAE;AACtD,UAAI,QAAQ,WAAY,QAAO,OAAO;AACtC,YAAM,gBAAgB,MAAM,MAAM,EAAE;AACpC,UAAI,kBAAkB,EAAG,QAAO;AAChC,YAAM,UAAU,MAAM,YAAY,IAAI;AAAA,QACpC,MAAM;AAAA,QACN,OAAO,gBAAgB;AAAA,QACvB,YAAY,CAAC,gBAAgB;AAAA,MAC/B,CAAC;AACD,UAAI,mBAAiD;AACrD,UAAI,cAAc,QAAQ,QAAQ,CAAC,GAAG,SAAS;AAC/C,iBAAW,OAAO,QAAQ,QAAQ,CAAC,GAAG;AACpC,6BAAqB,WAAW,4CAA4C;AAC5E,cAAM,MAAM,KAAK,kBAAkB,IAAI,GAAG;AAC1C,YACE,OAAO,IAAI,mBAAmB,YAC9B,IAAI,eAAe,WAAW,KAC9B,CAAC,KAAK,yBAAyB,KAAK,gBAAgB,IAAI,KAAK,GAAG,CAAC,GACjE;AACA,uBAAa;AACb;AAAA,QACF;AACA,YAAI,oBAAoB,IAAI,mBAAmB,kBAAkB;AAC/D,uBAAa;AACb;AAAA,QACF;AACA,2BAAmB,IAAI;AAAA,MACzB;AACA,UAAI,cAAc,kBAAkB;AAClC,aAAK,4BAA4B,IAAI,IAAI;AAAA,UACvC;AAAA,UACA,YAAY;AAAA,QACd,CAAC;AACD,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,UAAI,gBAAgB,SAAS,EAAG,OAAM;AACtC,UAAI,MAAM,qDAAqD,GAAG,EAAE;AACpE,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAc,uBACZ,IACA,kBACA,WACkB;AAClB,UAAM,EAAE,QAAQ,aAAa,MAAM,IAAI,KAAK;AAC5C,QAAI;AACF,YAAM,SAAS,KAAK,4BAA4B,IAAI,EAAE;AACtD,UAAI,QAAQ,qBAAqB,iBAAkB,QAAO,OAAO;AACjE,YAAM,gBAAgB,MAAM,MAAM,EAAE;AACpC,UAAI,kBAAkB,EAAG,QAAO;AAChC,YAAM,UAAU,MAAM,YAAY,IAAI;AAAA,QACpC,MAAM;AAAA,QACN,OAAO,gBAAgB;AAAA,QACvB,YAAY,CAAC,gBAAgB;AAAA,MAC/B,CAAC;AACD,UAAI,cAAc,QAAQ,QAAQ,CAAC,GAAG,SAAS;AAC/C,iBAAW,OAAO,QAAQ,QAAQ,CAAC,GAAG;AACpC,6BAAqB,WAAW,4CAA4C;AAC5E,cAAM,MAAM,KAAK,kBAAkB,IAAI,GAAG;AAC1C,YACE,IAAI,mBAAmB,oBACvB,CAAC,KAAK,yBAAyB,KAAK,gBAAgB,IAAI,KAAK,GAAG,CAAC,GACjE;AACA,uBAAa;AACb;AAAA,QACF;AAAA,MACF;AACA,WAAK,4BAA4B,IAAI,IAAI,EAAE,kBAAkB,WAAW,CAAC;AACzE,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,UAAI,gBAAgB,SAAS,EAAG,OAAM;AACtC,UAAI,MAAM,8CAA8C,GAAG,EAAE;AAC7D,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEQ,oCACN,IACA,kBACA,YACM;AACN,QAAI,CAAC,MAAM,OAAO,OAAO,SAAU;AACnC,QAAI,CAAC,kBAAkB;AACrB,WAAK,4BAA4B,OAAO,EAAE;AAC1C;AAAA,IACF;AACA,SAAK,4BAA4B,IAAI,IAAI,EAAE,kBAAkB,WAAW,CAAC;AAAA,EAC3E;AAAA,EAEQ,kBAAkB,IAAS,KAAmC;AACpE,UAAM,aAAa,KAAK,sBAAsB,IAAI,GAAG;AACrD,UAAM,cACJ,eAAe,UAAa,eAAe,OACvC,IAAI,MAAM,MAAM,OAAO,OAAO,UAAU,CAAC,IACzC;AACN,QAAI,eAAe,OAAO,gBAAgB,UAAU;AAClD,aAAO;AAAA,IACT;AACA,WAAO,KAAK,YAAY,OAAO,IAAI,aAAa,WAC5C,IAAI,WACJ,CAAC;AAAA,EACP;AAAA,EAEQ,gBAAgB,IAAS,KAAU,KAA+C;AACxF,UAAM,iBAAiB,KAAK,sBAAsB,IAAI,MAAM;AAC5D,QAAI,eAAgB,QAAO;AAC3B,UAAM,aAAa,KAAK,sBAAsB,IAAI,GAAG;AACrD,QAAI,eAAe,UAAa,eAAe,KAAM,QAAO;AAC5D,UAAM,cAAc,IAAI,MAAM,OAAO,eAAe,QAAQ,MAAM,SAAS,MAAM,UAAU;AAC3F,UAAM,SAAS,MAAM,QAAQ,WAAW,IAAI,YAAY,CAAC,IAAI;AAC7D,WAAO,KAAK,sBAAsB,MAAM;AAAA,EAC1C;AAAA,EAEQ,sBAAsB,IAAS,KAAmB;AACxD,UAAM,WACJ,OAAO,KAAK,OAAO,WACf,IAAI,KACJ,OAAO,KAAK,UAAU,OAAO,WAC3B,IAAI,SAAS,KACb;AACR,WAAO,YAAY,OAAO,IAAI,yBAAyB,gBAAgB,QAAQ,aAC3E,GAAG,wBAAwB,eAAe,IAAI,QAAQ,IACtD;AAAA,EACN;AAAA,EAEQ,0BAA0B,QAAyD;AACzF,WAAO,MAAM,QAAQ,MAAM,KAAK,OAAO,WAAW,KAAK;AAAA,EACzD;AAAA,EAEQ,yBAAyB,QAAqC;AACpE,QAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,UAAM,MAAM,MAAM,KAAK,MAA2B;AAClD,WACE,IAAI,WAAW,KAAK,sBACpB,IAAI,MAAM,CAAC,UAAU,OAAO,SAAS,KAAK,CAAC,KAC3C,IAAI,KAAK,CAAC,UAAU,UAAU,CAAC;AAAA,EAEnC;AAAA,EAEQ,aAAuB;AAC7B,WAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,mBAAmB,GAAG,MAAM,CAAC;AAAA,EAChE;AAAA,EAEQ,sBAAsB,QAAkC;AAC9D,UAAM,SACJ,MAAM,QAAQ,MAAM,IAChB,SACA,YAAY,OAAO,MAAM,KAAK,EAAE,kBAAkB,YAChD,MAAM,KAAK,MAAuC,IAClD;AACR,QAAI,CAAC,UAAU,OAAO,WAAW,KAAK,mBAAoB,QAAO;AACjE,UAAM,aAAa,OAAO,IAAI,CAAC,UAAU,OAAO,KAAK,CAAC;AACtD,WAAO,WAAW,MAAM,CAAC,UAAU,OAAO,SAAS,KAAK,CAAC,IAAI,aAAa;AAAA,EAC5E;AACF;","names":["migrated"]}

package/dist/{chunk-ZBJMUXZH.js → chunk-DJ7HMNSL.js}

@@ -16,21 +16,26 @@ function shouldKeepRecallToken(token, minTokenLength, stopWords) {
   const hasNonAsciiCodepoint = [...token].some((ch) => (ch.codePointAt(0) ?? 0) > 127);
   return token.length >= 2 && hasNonAsciiCodepoint && /\p{L}/u.test(token);
 }
-function addUnsegmentableRecallSegment(tokens, segment, stopWords) {
+function expandUnsegmentableRecallNGrams(segment) {
   const chars = [...segment].filter((ch) => /[\p{L}\p{N}\p{M}]/u.test(ch) || isUnsegmentableRecallChar(ch));
+  const tokens = [];
   for (const ch of chars) {
-    if (!stopWords.has(ch)) tokens.add(ch);
+    tokens.push(ch);
   }
   for (const size of [2, 3, 4]) {
     if (chars.length < size) continue;
     for (let index = 0; index <= chars.length - size; index += 1) {
-      const token = chars.slice(index, index + size).join("");
-      if (!stopWords.has(token)) tokens.add(token);
+      tokens.push(chars.slice(index, index + size).join(""));
     }
   }
-  const whole = chars.join("");
-  if (whole.length > 3 && !stopWords.has(whole)) {
-    tokens.add(whole);
+  if (chars.length > 3) {
+    tokens.push(chars.join(""));
+  }
+  return tokens;
+}
+function addUnsegmentableRecallSegment(tokens, segment, stopWords) {
+  for (const token of expandUnsegmentableRecallNGrams(segment)) {
+    if (!stopWords.has(token)) tokens.add(token);
   }
 }
 function isUnsegmentableRecallToken(token) {
@@ -114,8 +119,10 @@ function countRecallTokenOverlap(queryTokens, value, extraStopWords = []) {
 }
 
 export {
+  isUnsegmentableRecallChar,
+  expandUnsegmentableRecallNGrams,
   normalizeRecallTokenSet,
   normalizeRecallTokens,
   countRecallTokenOverlap
 };
-//# sourceMappingURL=chunk-ZBJMUXZH.js.map
+//# sourceMappingURL=chunk-DJ7HMNSL.js.map

package/dist/chunk-DJ7HMNSL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/recall-tokenization.ts"],"sourcesContent":["export interface NormalizeRecallTokenOptions {\n  minTokenLength?: number;\n}\n\nconst DEFAULT_RECALL_STOP_WORDS = [\"the\", \"and\", \"for\", \"with\", \"from\", \"into\", \"that\", \"this\", \"why\", \"did\"];\n\nexport function isUnsegmentableRecallChar(char: string): boolean {\n  if (char === \"ー\" || char === \"ｰ\") return true;\n  return /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}]/u.test(char);\n}\n\nfunction isRecallCombiningMark(char: string): boolean {\n  return /\\p{M}/u.test(char);\n}\n\nfunction buildRecallStopWords(extraStopWords: string[]): Set<string> {\n  return new Set([...DEFAULT_RECALL_STOP_WORDS, ...extraStopWords.map((word) => word.toLowerCase())]);\n}\n\nfunction shouldKeepRecallToken(token: string, minTokenLength: number, stopWords: Set<string>): boolean {\n  if (stopWords.has(token)) return false;\n  if (token.length >= minTokenLength) return true;\n  const hasNonAsciiCodepoint = [...token].some((ch) => (ch.codePointAt(0) ?? 0) > 0x7f);\n  return token.length >= 2 && hasNonAsciiCodepoint && /\\p{L}/u.test(token);\n}\n\nexport function expandUnsegmentableRecallNGrams(segment: string): string[] {\n  const chars = [...segment].filter((ch) => /[\\p{L}\\p{N}\\p{M}]/u.test(ch) || isUnsegmentableRecallChar(ch));\n  const tokens: string[] = [];\n  for (const ch of chars) {\n    tokens.push(ch);\n  }\n  for (const size of [2, 3, 4]) {\n    if (chars.length < size) continue;\n    for (let index = 0; index <= chars.length - size; index += 1) {\n      tokens.push(chars.slice(index, index + size).join(\"\"));\n    }\n  }\n  if (chars.length > 3) {\n    tokens.push(chars.join(\"\"));\n  }\n  return tokens;\n}\n\nfunction addUnsegmentableRecallSegment(tokens: Set<string>, segment: string, stopWords: Set<string>): void {\n  for (const token of expandUnsegmentableRecallNGrams(segment)) {\n    if (!stopWords.has(token)) tokens.add(token);\n  }\n}\n\nfunction isUnsegmentableRecallToken(token: string): boolean {\n  const chars = [...token].filter((ch) => /[\\p{L}\\p{N}\\p{M}]/u.test(ch) || isUnsegmentableRecallChar(ch));\n  return (\n    chars.length > 0 &&\n    chars.some(isUnsegmentableRecallChar) &&\n    chars.every((ch) => isUnsegmentableRecallChar(ch) || isRecallCombiningMark(ch))\n  );\n}\n\nfunction addBridgedUnsegmentableRecallSegments(tokens: Set<string>, cleaned: string, stopWords: Set<string>): void {\n  let segment = \"\";\n  const flushSegment = () => {\n    addUnsegmentableRecallSegment(tokens, segment, stopWords);\n    segment = \"\";\n  };\n\n  for (const token of cleaned.split(/\\s+/)) {\n    if (isUnsegmentableRecallToken(token)) {\n      segment += token;\n    } else {\n      flushSegment();\n    }\n  }\n  flushSegment();\n}\n\nexport function normalizeRecallTokenSet(\n  value: string,\n  extraStopWords: string[] = [],\n  options: NormalizeRecallTokenOptions = {}\n): Set<string> {\n  const minTokenLength = Math.max(1, Math.floor(options.minTokenLength ?? 3));\n  const stopWords = buildRecallStopWords(extraStopWords);\n  const cleaned = value\n    .toLowerCase()\n    .normalize(\"NFC\")\n    .replace(/[^\\p{L}\\p{N}\\p{M}\\u30fc\\uff70]+/gu, \" \")\n    .trim();\n  if (cleaned.length === 0) return new Set();\n\n  const tokens = new Set<string>();\n  addBridgedUnsegmentableRecallSegments(tokens, cleaned, stopWords);\n  for (const token of cleaned.split(/\\s+/)) {\n    if (!token) continue;\n    if ([...token].some(isUnsegmentableRecallChar)) {\n      let segment = \"\";\n      let unsegmentableSegment = \"\";\n      const flushSegment = () => {\n        if (shouldKeepRecallToken(segment, minTokenLength, stopWords)) {\n          tokens.add(segment);\n        }\n        segment = \"\";\n      };\n      const flushUnsegmentableSegment = () => {\n        addUnsegmentableRecallSegment(tokens, unsegmentableSegment, stopWords);\n        unsegmentableSegment = \"\";\n      };\n      for (const ch of token) {\n        if (!/[\\p{L}\\p{N}\\p{M}]/u.test(ch) && !isUnsegmentableRecallChar(ch)) continue;\n        if (isUnsegmentableRecallChar(ch)) {\n          flushSegment();\n          unsegmentableSegment += ch;\n        } else if (isRecallCombiningMark(ch)) {\n          if (unsegmentableSegment.length > 0) {\n            unsegmentableSegment += ch;\n          } else {\n            segment += ch;\n          }\n        } else {\n          flushUnsegmentableSegment();\n          segment += ch;\n        }\n      }\n      flushUnsegmentableSegment();\n      flushSegment();\n      continue;\n    }\n    if (shouldKeepRecallToken(token, minTokenLength, stopWords)) {\n      tokens.add(token);\n    }\n  }\n  return tokens;\n}\n\nexport function normalizeRecallTokens(value: string, extraStopWords: string[] = []): string[] {\n  return Array.from(normalizeRecallTokenSet(value, extraStopWords));\n}\n\nexport function countRecallTokenOverlap(\n  queryTokens: Set<string>,\n  value: string | undefined,\n  extraStopWords: string[] = []\n): number {\n  if (!value) return 0;\n  const tokens = normalizeRecallTokenSet(value, extraStopWords);\n  let matches = 0;\n  for (const token of queryTokens) {\n    if (tokens.has(token)) matches += 1;\n  }\n  return matches;\n}\n"],"mappings":";AAIA,IAAM,4BAA4B,CAAC,OAAO,OAAO,OAAO,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,OAAO,KAAK;AAErG,SAAS,0BAA0B,MAAuB;AAC/D,MAAI,SAAS,YAAO,SAAS,SAAK,QAAO;AACzC,SAAO,0DAA0D,KAAK,IAAI;AAC5E;AAEA,SAAS,sBAAsB,MAAuB;AACpD,SAAO,SAAS,KAAK,IAAI;AAC3B;AAEA,SAAS,qBAAqB,gBAAuC;AACnE,SAAO,oBAAI,IAAI,CAAC,GAAG,2BAA2B,GAAG,eAAe,IAAI,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC,CAAC;AACpG;AAEA,SAAS,sBAAsB,OAAe,gBAAwB,WAAiC;AACrG,MAAI,UAAU,IAAI,KAAK,EAAG,QAAO;AACjC,MAAI,MAAM,UAAU,eAAgB,QAAO;AAC3C,QAAM,uBAAuB,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC,KAAK,KAAK,GAAI;AACpF,SAAO,MAAM,UAAU,KAAK,wBAAwB,SAAS,KAAK,KAAK;AACzE;AAEO,SAAS,gCAAgC,SAA2B;AACzE,QAAM,QAAQ,CAAC,GAAG,OAAO,EAAE,OAAO,CAAC,OAAO,qBAAqB,KAAK,EAAE,KAAK,0BAA0B,EAAE,CAAC;AACxG,QAAM,SAAmB,CAAC;AAC1B,aAAW,MAAM,OAAO;AACtB,WAAO,KAAK,EAAE;AAAA,EAChB;AACA,aAAW,QAAQ,CAAC,GAAG,GAAG,CAAC,GAAG;AAC5B,QAAI,MAAM,SAAS,KAAM;AACzB,aAAS,QAAQ,GAAG,SAAS,MAAM,SAAS,MAAM,SAAS,GAAG;AAC5D,aAAO,KAAK,MAAM,MAAM,OAAO,QAAQ,IAAI,EAAE,KAAK,EAAE,CAAC;AAAA,IACvD;AAAA,EACF;AACA,MAAI,MAAM,SAAS,GAAG;AACpB,WAAO,KAAK,MAAM,KAAK,EAAE,CAAC;AAAA,EAC5B;AACA,SAAO;AACT;AAEA,SAAS,8BAA8B,QAAqB,SAAiB,WAA8B;AACzG,aAAW,SAAS,gCAAgC,OAAO,GAAG;AAC5D,QAAI,CAAC,UAAU,IAAI,KAAK,EAAG,QAAO,IAAI,KAAK;AAAA,EAC7C;AACF;AAEA,SAAS,2BAA2B,OAAwB;AAC1D,QAAM,QAAQ,CAAC,GAAG,KAAK,EAAE,OAAO,CAAC,OAAO,qBAAqB,KAAK,EAAE,KAAK,0BAA0B,EAAE,CAAC;AACtG,SACE,MAAM,SAAS,KACf,MAAM,KAAK,yBAAyB,KACpC,MAAM,MAAM,CAAC,OAAO,0BAA0B,EAAE,KAAK,sBAAsB,EAAE,CAAC;AAElF;AAEA,SAAS,sCAAsC,QAAqB,SAAiB,WAA8B;AACjH,MAAI,UAAU;AACd,QAAM,eAAe,MAAM;AACzB,kCAA8B,QAAQ,SAAS,SAAS;AACxD,cAAU;AAAA,EACZ;AAEA,aAAW,SAAS,QAAQ,MAAM,KAAK,GAAG;AACxC,QAAI,2BAA2B,KAAK,GAAG;AACrC,iBAAW;AAAA,IACb,OAAO;AACL,mBAAa;AAAA,IACf;AAAA,EACF;AACA,eAAa;AACf;AAEO,SAAS,wBACd,OACA,iBAA2B,CAAC,GAC5B,UAAuC,CAAC,GAC3B;AACb,QAAM,iBAAiB,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,kBAAkB,CAAC,CAAC;AAC1E,QAAM,YAAY,qBAAqB,cAAc;AACrD,QAAM,UAAU,MACb,YAAY,EACZ,UAAU,KAAK,EACf,QAAQ,qCAAqC,GAAG,EAChD,KAAK;AACR,MAAI,QAAQ,WAAW,EAAG,QAAO,oBAAI,IAAI;AAEzC,QAAM,SAAS,oBAAI,IAAY;AAC/B,wCAAsC,QAAQ,SAAS,SAAS;AAChE,aAAW,SAAS,QAAQ,MAAM,KAAK,GAAG;AACxC,QAAI,CAAC,MAAO;AACZ,QAAI,CAAC,GAAG,KAAK,EAAE,KAAK,yBAAyB,GAAG;AAC9C,UAAI,UAAU;AACd,UAAI,uBAAuB;AAC3B,YAAM,eAAe,MAAM;AACzB,YAAI,sBAAsB,SAAS,gBAAgB,SAAS,GAAG;AAC7D,iBAAO,IAAI,OAAO;AAAA,QACpB;AACA,kBAAU;AAAA,MACZ;AACA,YAAM,4BAA4B,MAAM;AACtC,sCAA8B,QAAQ,sBAAsB,SAAS;AACrE,+BAAuB;AAAA,MACzB;AACA,iBAAW,MAAM,OAAO;AACtB,YAAI,CAAC,qBAAqB,KAAK,EAAE,KAAK,CAAC,0BAA0B,EAAE,EAAG;AACtE,YAAI,0BAA0B,EAAE,GAAG;AACjC,uBAAa;AACb,kCAAwB;AAAA,QAC1B,WAAW,sBAAsB,EAAE,GAAG;AACpC,cAAI,qBAAqB,SAAS,GAAG;AACnC,oCAAwB;AAAA,UAC1B,OAAO;AACL,uBAAW;AAAA,UACb;AAAA,QACF,OAAO;AACL,oCAA0B;AAC1B,qBAAW;AAAA,QACb;AAAA,MACF;AACA,gCAA0B;AAC1B,mBAAa;AACb;AAAA,IACF;AACA,QAAI,sBAAsB,OAAO,gBAAgB,SAAS,GAAG;AAC3D,aAAO,IAAI,KAAK;AAAA,IAClB;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,sBAAsB,OAAe,iBAA2B,CAAC,GAAa;AAC5F,SAAO,MAAM,KAAK,wBAAwB,OAAO,cAAc,CAAC;AAClE;AAEO,SAAS,wBACd,aACA,OACA,iBAA2B,CAAC,GACpB;AACR,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,SAAS,wBAAwB,OAAO,cAAc;AAC5D,MAAI,UAAU;AACd,aAAW,SAAS,aAAa;AAC/B,QAAI,OAAO,IAAI,KAAK,EAAG,YAAW;AAAA,EACpC;AACA,SAAO;AACT;","names":[]}

package/dist/{chunk-WTMG5GAK.js → chunk-DROAPP6N.js}

@@ -1,9 +1,9 @@
 import {
   NamespaceStorageRouter
-} from "./chunk-67MKNKJQ.js";
+} from "./chunk-AYIHDC5S.js";
 import {
   namespaceCollectionName
-} from "./chunk-QWPAQLYV.js";
+} from "./chunk-JFLKSD5V.js";
 import {
   isSafeRouteNamespace
 } from "./chunk-PCXDUHQH.js";
@@ -213,4 +213,4 @@ export {
   verifyNamespaces,
   runNamespaceMigration
 };
-//# sourceMappingURL=chunk-WTMG5GAK.js.map
+//# sourceMappingURL=chunk-DROAPP6N.js.map

package/dist/{chunk-PBH2Z2NY.js → chunk-DTLF2SLZ.js}

@@ -4,7 +4,7 @@ import {
 } from "./chunk-RDFF2NAU.js";
 import {
   StorageManager
-} from "./chunk-ZJKLGS76.js";
+} from "./chunk-CALURYWR.js";
 import {
   excludeSupportPassportPrivateMemories
 } from "./chunk-XDL5CYZN.js";
@@ -138,4 +138,4 @@ export {
   runCodexMaterialize,
   runPostConsolidationMaterialize
 };
-//# sourceMappingURL=chunk-PBH2Z2NY.js.map
+//# sourceMappingURL=chunk-DTLF2SLZ.js.map

package/dist/{chunk-7JAXC526.js → chunk-EFWHCTYJ.js}

@@ -1,6 +1,6 @@
 import {
   listNamespaces
-} from "./chunk-WTMG5GAK.js";
+} from "./chunk-DROAPP6N.js";
 import {
   capabilityAllowsNamespace
 } from "./chunk-MZWSOLMT.js";
@@ -282,4 +282,4 @@ export {
   computeServiceCorpusWatermarks,
   computeServiceCorpusCensus
 };
-//# sourceMappingURL=chunk-7JAXC526.js.map
+//# sourceMappingURL=chunk-EFWHCTYJ.js.map