@remnic/core 9.62.0 → 9.62.2

package/src/search/orama-backend.test.ts

@@ -1,8 +1,46 @@
 import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
 import path from "node:path";
 import test from "node:test";
 
-import { resolveOramaCollectionDbFilePath } from "./orama-backend.js";
+import { EmbedHelper } from "./embed-helper.js";
+import { OramaBackend, resolveOramaCollectionDbFilePath } from "./orama-backend.js";
+import type { PluginConfig } from "../types.js";
+
+function noEmbedHelper(): EmbedHelper {
+  return new EmbedHelper({} as PluginConfig);
+}
+
+async function writeMemoryFact(
+  memoryDir: string,
+  fileName: string,
+  content: string,
+): Promise<void> {
+  await mkdir(path.join(memoryDir, "facts"), { recursive: true });
+  await writeFile(path.join(memoryDir, "facts", fileName), content, "utf8");
+}
+
+async function createBackend(
+  memoryDir: string,
+  dbPath: string,
+  cjkSegmentationEnabled?: boolean,
+  options: { update?: boolean } = {},
+): Promise<OramaBackend> {
+  const backend = new OramaBackend({
+    dbPath,
+    collection: "openclaw-engram",
+    embedHelper: noEmbedHelper(),
+    memoryDir,
+    embeddingDimension: 4,
+    cjkSegmentationEnabled,
+  });
+  assert.equal(await backend.probe(), true);
+  if (options.update !== false) {
+    await backend.update();
+  }
+  return backend;
+}
 
 test("Orama collection filenames cannot escape dbPath", () => {
   const dbPath = path.join("/tmp", "remnic-orama-db");
@@ -25,3 +63,172 @@ test("Orama collection filenames cannot escape dbPath", () => {
     );
   }
 });
+
+test("Orama lexical search finds a Japanese fact with no embeddings, at parity with English (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-cjk-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "japanese-fact.md",
+      "---\nid: mem-japanese-fact\n---\n東京都庁の所在地は新宿区にある。最寄り駅は都庁前駅だ。",
+    );
+    await writeMemoryFact(
+      root,
+      "english-fact.md",
+      "---\nid: mem-english-fact\n---\nThe Tokyo Metropolitan Government Office is located in Shinjuku. The nearest station is Tochomae.",
+    );
+
+    const backend = await createBackend(root, path.join(root, "orama-db"));
+
+    const japanese = await backend.bm25Search("東京都庁の所在地");
+    const english = await backend.bm25Search("Tokyo Metropolitan Government Office location");
+
+    assert.equal(japanese.some((r) => r.docid === "mem-japanese-fact"), true);
+    assert.equal(english.some((r) => r.docid === "mem-english-fact"), true);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama CJK tokenizer rebuilds stale pre-CJK indexes on first open (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-rebuild-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "japanese-fact.md",
+      "---\nid: mem-japanese-fact\n---\n東京都庁の所在地は新宿区にある。",
+    );
+    await writeMemoryFact(
+      root,
+      "russian-fact.md",
+      "---\nid: mem-russian-fact\n---\nПравительство Токио находится в Синдзюку.",
+    );
+    await writeMemoryFact(
+      root,
+      "english-fact.md",
+      "---\nid: mem-english-fact\n---\nNebula-472 deploy finished in production.",
+    );
+
+    // Phase 1: index with the pre-CJK stock tokenizer (marker "english").
+    const legacyBackend = await createBackend(root, path.join(root, "orama-db"), false);
+    const legacyJapanese = await legacyBackend.bm25Search("東京都庁の所在地");
+    assert.equal(
+      legacyJapanese.some((r) => r.docid === "mem-japanese-fact"),
+      false,
+      "stock tokenizer must not match Japanese phrases",
+    );
+    const legacyEnglish = await legacyBackend.bm25Search("Nebula-472 deploy");
+    assert.equal(legacyEnglish.some((r) => r.docid === "mem-english-fact"), true);
+
+    // Phase 2: reopen with CJK segmentation on and NO update() call — the
+    // stale marker alone must trigger the in-place rebuild during probe().
+    const rebuiltBackend = await createBackend(root, path.join(root, "orama-db"), true, {
+      update: false,
+    });
+    const rebuiltJapanese = await rebuiltBackend.bm25Search("東京都庁の所在地");
+    assert.equal(rebuiltJapanese.some((r) => r.docid === "mem-japanese-fact"), true);
+
+    // Non-CJK non-Latin scripts are re-indexed too (whole-word terms), not
+    // just CJK/Thai.
+    const rebuiltRussian = await rebuiltBackend.bm25Search("Правительство Токио");
+    assert.equal(rebuiltRussian.some((r) => r.docid === "mem-russian-fact"), true);
+
+    // English-corpus behavior is unchanged by the rebuild.
+    const rebuiltEnglish = await rebuiltBackend.bm25Search("Nebula-472 deploy");
+    assert.equal(rebuiltEnglish.some((r) => r.docid === "mem-english-fact"), true);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama English-only corpora are not re-indexed when the tokenizer version advances (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-english-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "english-fact.md",
+      "---\nid: mem-english-fact\n---\nNebula-472 deploy finished in production.",
+    );
+
+    const legacyBackend = await createBackend(root, path.join(root, "orama-db"), false);
+    assert.equal(
+      (await legacyBackend.bm25Search("Nebula-472 deploy")).some((r) => r.docid === "mem-english-fact"),
+      true,
+    );
+
+    const upgradedBackend = await createBackend(root, path.join(root, "orama-db"), true);
+    assert.equal(
+      (await upgradedBackend.bm25Search("Nebula-472 deploy")).some((r) => r.docid === "mem-english-fact"),
+      true,
+      "English recall must survive the tokenizer upgrade",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama downgrades persisted CJK indexes when segmentation is disabled (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-downgrade-"));
+  try {
+    await writeMemoryFact(
+      root,
+      "japanese-fact.md",
+      "---\nid: mem-japanese-fact\n---\n東京都庁の所在地は新宿区にある。",
+    );
+
+    // Phase 1: index with CJK segmentation on (marker "english+cjk-v1").
+    const cjkBackend = await createBackend(root, path.join(root, "orama-db"), true);
+    assert.equal(
+      (await cjkBackend.bm25Search("東京都庁の所在地")).some((r) => r.docid === "mem-japanese-fact"),
+      true,
+      "CJK tokenizer must match Japanese phrases before the downgrade",
+    );
+
+    // Phase 2: reopen with segmentation disabled and NO update() call — the
+    // persisted CJK marker alone must trigger the stock re-index.
+    const downgraded = await createBackend(root, path.join(root, "orama-db"), false, {
+      update: false,
+    });
+    assert.equal(
+      (await downgraded.bm25Search("東京都庁の所在地")).some((r) => r.docid === "mem-japanese-fact"),
+      false,
+      "stock tokenizer must not match Japanese phrases after the downgrade",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("Orama rebuild gate inspects every indexed string field, not only content (issue #2187)", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-orama-pathfield-"));
+  try {
+    // The only non-Latin text is the FILENAME, which lands in the indexed
+    // path field; the body is pure English.
+    await writeMemoryFact(
+      root,
+      "東京-fact.md",
+      "---\nid: mem-path-only\n---\nAn English body with no non-Latin characters.",
+    );
+
+    // Phase 1: pre-CJK stock index — the CJK filename never becomes a term.
+    const legacyBackend = await createBackend(root, path.join(root, "orama-db"), false);
+    assert.equal(
+      (await legacyBackend.bm25Search("東京")).some((r) => r.docid === "mem-path-only"),
+      false,
+      "stock tokenizer must not match the CJK filename",
+    );
+
+    // Phase 2: reopen with segmentation on and NO update() call — the
+    // path field alone must gate the in-place re-index.
+    const rebuilt = await createBackend(root, path.join(root, "orama-db"), true, {
+      update: false,
+    });
+    assert.equal(
+      (await rebuilt.bm25Search("東京")).some((r) => r.docid === "mem-path-only"),
+      true,
+      "non-Latin text in the path field must gate the rebuild",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/src/search/orama-backend.ts

@@ -9,17 +9,28 @@ import {
   type SearchQueryOptions,
   type SearchResult,
 } from "./port.js";
-import type { EmbedHelper, EmbedProviderIdentity, EmbedWithProviderResult } from "./embed-helper.js";
-import { scanMemoryDir } from "./document-scanner.js";
-import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
-
-export interface OramaBackendOptions {
-  dbPath: string;
-  collection: string;
-  embedHelper: EmbedHelper;
-  memoryDir: string;
-  embeddingDimension: number;
-}
+ import type { EmbedHelper, EmbedProviderIdentity, EmbedWithProviderResult } from "./embed-helper.js";
+ import { scanMemoryDir } from "./document-scanner.js";
+ import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
+import {
+  containsNonLegacyTokenizerChars,
+  createCjkCapableTokenizer,
+  ORAMA_CJK_TOKENIZER_LANGUAGE,
+} from "./orama-cjk-tokenizer.js";
+ 
+ export interface OramaBackendOptions {
+   dbPath: string;
+   collection: string;
+   embedHelper: EmbedHelper;
+   memoryDir: string;
+   embeddingDimension: number;
+  /**
+   * Segment space-free scripts (CJK/Thai) in the lexical index. Default
+   * `true`; `false` restores the stock English-only tokenizer (issue #2187).
+   */
+  cjkSegmentationEnabled?: boolean;
+ }
+
 
 const ORAMA_COLLECTION_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
 
@@ -57,6 +68,7 @@ export class OramaBackend implements SearchBackend {
   private readonly embedHelper: EmbedHelper;
   private readonly memoryDir: string;
   private readonly embeddingDimension: number;
+  private readonly cjkSegmentationEnabled: boolean;
   private available = false;
   private db: any = null;
   private oramaModule: any = null;
@@ -72,6 +84,7 @@ export class OramaBackend implements SearchBackend {
     this.embedHelper = opts.embedHelper;
     this.memoryDir = opts.memoryDir;
     this.embeddingDimension = opts.embeddingDimension;
+    this.cjkSegmentationEnabled = opts.cjkSegmentationEnabled !== false;
   }
 
   async probe(): Promise<boolean> {
@@ -127,6 +140,11 @@ export class OramaBackend implements SearchBackend {
     }
   }
 
+  /**
+   * Full-text (FTS) search only. With the stock tokenizer this cannot match
+   * space-free CJK/Thai text lexically — hybrid or vector search provides
+   * the fallback for those scripts (issue #2187).
+   */
   async bm25Search(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
     if (isSearchAborted(execution)) return [];
     const db = await this.ensureDbForCollection(collection ?? this.collection);
@@ -403,8 +421,11 @@ export class OramaBackend implements SearchBackend {
       return this.db;
     }
 
-    this.db = await this.migrateLegacyVectorProviderSchema(
-      await this.persistModule.restore("json", raw),
+    this.db = await this.applyTokenizerVersion(
+      await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        this.collection,
+      ),
       this.collection,
     );
     return this.db;
@@ -426,8 +447,11 @@ export class OramaBackend implements SearchBackend {
       return await this.createDb();
     }
 
-    return await this.migrateLegacyVectorProviderSchema(
-      await this.persistModule.restore("json", raw),
+    return await this.applyTokenizerVersion(
+      await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        collection,
+      ),
       collection,
     );
   }
@@ -442,7 +466,95 @@ export class OramaBackend implements SearchBackend {
       vectorProvider: "string",
       vector: `vector[${this.embeddingDimension}]`,
     };
-    return await create({ schema });
+    if (!this.cjkSegmentationEnabled) {
+      return await create({ schema });
+    }
+    return await create({
+      schema,
+      components: { tokenizer: createCjkCapableTokenizer(this.oramaModule) },
+    });
+  }
+
+  /**
+   * Attach the CJK-capable tokenizer to a restored index and rebuild the
+   * full-text index when it predates CJK tokenization. Orama persists
+   * `tokenizer.language` with the index, so a marker other than
+   * `ORAMA_CJK_TOKENIZER_LANGUAGE` identifies an index whose terms were
+   * produced by the stock English tokenizer. Corpora whose content the stock
+   * tokenizer and this tokenizer treat identically (pure legacy Latin) need
+   * no re-indexing — only the version marker is rewritten for them.
+   */
+  private async applyTokenizerVersion(db: any, collection: string): Promise<any> {
+    const persistedLanguage =
+      db?.tokenizer?.language && typeof db.tokenizer.language === "string"
+        ? db.tokenizer.language
+        : "";
+    if (this.cjkSegmentationEnabled) {
+      db.tokenizer = createCjkCapableTokenizer(this.oramaModule);
+      if (persistedLanguage === ORAMA_CJK_TOKENIZER_LANGUAGE) return db;
+    } else {
+      // Segmentation disabled: a database persisted under the CJK tokenizer
+      // keeps its CJK terms unless it is re-indexed under the stock one, so
+      // downgrade it instead of returning it unchanged (issue #2187 round 2).
+      if (persistedLanguage !== ORAMA_CJK_TOKENIZER_LANGUAGE) return db;
+      db.tokenizer = this.oramaModule.components.tokenizer.createTokenizer({ language: "english" });
+    }
+
+
+    try {
+      const { search: oramaSearch, count, update: oramaUpdate } = this.oramaModule;
+      const existingCount = await count(db);
+      if (existingCount === 0) {
+        await this.persistDbForCollection(db, collection);
+        return db;
+      }
+      const allHits = await oramaSearch(db, { term: "", limit: existingCount + 100 });
+      const hits = allHits.hits ?? [];
+      const hasNonLegacyContent = hits.some((hit: any) => {
+        const doc = this.getStoredDocument(db, hit);
+        // Orama full-text queries every string field by default, so every
+        // indexed string field gates the rebuild — non-Latin text that only
+        // rides in snippet/path/id would otherwise keep stale terms
+        // (issue #2187 review round 2).
+        for (const field of [doc.content, doc.snippet, doc.path, doc.id, doc.vectorProvider]) {
+          if (typeof field === "string" && containsNonLegacyTokenizerChars(field)) return true;
+        }
+        return false;
+      });
+      if (hasNonLegacyContent) {
+        // Re-index every document in place (Orama update is remove+insert,
+        // so content is re-tokenized; vectors are carried through). A single
+        // failing document must not block the rest of the corpus: it keeps
+        // its pre-upgrade terms, which is no worse than before the upgrade.
+        for (const hit of hits) {
+          try {
+            const doc = this.getStoredDocument(db, hit);
+            const vector = this.getStoredVector(db, hit, doc);
+            await oramaUpdate(db, hit.id, {
+              id: typeof doc.id === "string" && doc.id.length > 0 ? doc.id : String(hit.id),
+              path: typeof doc.path === "string" ? doc.path : "",
+              content: typeof doc.content === "string" ? doc.content : "",
+              snippet:
+                typeof doc.snippet === "string"
+                  ? doc.snippet
+                  : typeof doc.content === "string"
+                    ? doc.content.slice(0, 200)
+                    : "",
+              vectorProvider: typeof doc.vectorProvider === "string" ? doc.vectorProvider : "",
+              vector: vector ?? this.zeroVector(),
+            });
+          } catch (docErr) {
+            log.debug(
+              `OramaBackend tokenizer-version re-index skipped document ${hit.id} in ${collection}: ${docErr}`,
+            );
+          }
+        }
+      }
+      await this.persistDbForCollection(db, collection);
+    } catch (err) {
+      log.debug(`OramaBackend tokenizer-version rebuild for ${collection} failed: ${err}`);
+    }
+    return db;
   }
 
   private async migrateLegacyVectorProviderSchema(db: any, collection: string): Promise<any> {
@@ -526,8 +638,11 @@ export class OramaBackend implements SearchBackend {
       await this.ensureModules();
       const raw = await readFile(filePath, "utf-8");
       const collection = path.basename(filePath, ".msp");
-      return await this.migrateLegacyVectorProviderSchema(
-        await this.persistModule.restore("json", raw),
+      return await this.applyTokenizerVersion(
+        await this.migrateLegacyVectorProviderSchema(
+          await this.persistModule.restore("json", raw),
+          collection,
+        ),
         collection,
       );
     } catch (err) {

package/src/search/orama-cjk-tokenizer.test.ts

@@ -0,0 +1,108 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import * as orama from "@orama/orama";
+
+import {
+  containsNonLegacyTokenizerChars,
+  createCjkCapableTokenizer,
+  isSpaceFreeScriptChar,
+  ORAMA_CJK_TOKENIZER_LANGUAGE,
+} from "./orama-cjk-tokenizer.js";
+
+const stockTokenizer = orama.components.tokenizer.createTokenizer({ language: "english" });
+const tokenizer = createCjkCapableTokenizer(orama);
+
+test("CJK tokenizer matches the stock English tokenizer for legacy Latin input", () => {
+  for (const input of [
+    "The quick brown fox jumps over the lazy dog",
+    "Nebula-472 deploy finished; it's live (prod)!",
+    "snake_case camelCase 42.7%",
+    "",
+  ]) {
+    assert.deepEqual(tokenizer.tokenize(input), stockTokenizer.tokenize(input), input);
+  }
+});
+
+test("CJK tokenizer expands Japanese runs into n-grams", () => {
+  const tokens = tokenizer.tokenize("東京都庁の所在地");
+
+  assert.equal(tokens.includes("東京都庁"), true);
+  assert.equal(tokens.includes("所在"), true);
+  assert.equal(tokens.includes("地"), true);
+  assert.equal(tokens.includes("東京都庁の所在地"), true);
+});
+
+test("CJK tokenizer reuses the recall segmentation n-gram sizes", () => {
+  // A 5-char run yields chars + 2/3/4-grams + the whole run.
+  const tokens = tokenizer.tokenize("用户喜欢深");
+  const expected = new Set(["用", "户", "喜", "欢", "深"]);
+  for (const size of [2, 3, 4]) {
+    for (let i = 0; i <= 5 - size; i++) {
+      expected.add("用户喜欢深".slice(i, i + size));
+    }
+  }
+  expected.add("用户喜欢深");
+  for (const token of expected) {
+    assert.equal(tokens.includes(token), true, token);
+  }
+});
+
+test("CJK tokenizer keeps Latin tokens intact inside mixed-script input", () => {
+  const tokens = tokenizer.tokenize("Nebula-472 東京都庁");
+
+  assert.equal(tokens.includes("nebula-472"), true);
+  assert.equal(tokens.includes("東京都庁"), true);
+});
+
+test("CJK tokenizer n-grams Thai runs", () => {
+  const tokens = tokenizer.tokenize("กรุงเทพมหานคร");
+
+  assert.equal(tokens.includes("กรุง"), true);
+  assert.equal(tokens.includes("งเทพ"), true);
+  assert.equal(tokens.includes("กรุงเทพมหานคร"), true);
+});
+
+test("CJK tokenizer keeps Hangul and other alphabetic scripts as whole words", () => {
+  const tokens = tokenizer.tokenize("사용자 설정 Привет мир");
+
+  assert.equal(tokens.includes("사용자"), true);
+  assert.equal(tokens.includes("설정"), true);
+  assert.equal(tokens.includes("사"), false);
+  assert.equal(tokens.includes("привет"), true);
+  assert.equal(tokens.includes("мир"), true);
+});
+
+test("CJK tokenizer persists a versioned language marker", () => {
+  assert.equal(tokenizer.language, ORAMA_CJK_TOKENIZER_LANGUAGE);
+  assert.equal(typeof tokenizer.tokenize, "function");
+  assert.ok(tokenizer.normalizationCache instanceof Map);
+});
+
+test("isSpaceFreeScriptChar covers CJK and Thai only", () => {
+  assert.equal(isSpaceFreeScriptChar("東"), true);
+  assert.equal(isSpaceFreeScriptChar("あ"), true);
+  assert.equal(isSpaceFreeScriptChar("ア"), true);
+  assert.equal(isSpaceFreeScriptChar("ก"), true);
+  assert.equal(isSpaceFreeScriptChar("사"), false);
+  assert.equal(isSpaceFreeScriptChar("a"), false);
+});
+
+test("CJK tokenizer returns no tokens for non-string input", () => {
+  assert.deepEqual(tokenizer.tokenize(undefined as unknown as string), []);
+  assert.deepEqual(tokenizer.tokenize(null as unknown as string), []);
+  assert.deepEqual(tokenizer.tokenize(42 as unknown as string), []);
+});
+
+test("rebuild gate flags only tokenization-changing characters", () => {
+  // Word material outside the stock keep-set changes tokens (whole words,
+  // n-grams, attached marks) — the gate must flag it.
+  assert.equal(containsNonLegacyTokenizerChars("naïve"), true);
+  assert.equal(containsNonLegacyTokenizerChars("東京都庁"), true);
+  assert.equal(containsNonLegacyTokenizerChars("사용자"), true);
+  assert.equal(containsNonLegacyTokenizerChars("cafe\u0301"), true);
+  // Separators in BOTH tokenizers — punctuation and symbols — leave the
+  // stock token stream unchanged, so no re-index is warranted.
+  assert.equal(containsNonLegacyTokenizerChars("plain — text „quoted” · done"), false);
+  assert.equal(containsNonLegacyTokenizerChars("Nebula-472 deploy (prod)"), false);
+});

package/src/search/orama-cjk-tokenizer.ts

@@ -0,0 +1,143 @@
+import { expandUnsegmentableRecallNGrams, isUnsegmentableRecallChar } from "../recall-tokenization.js";
+
+/**
+ * Language marker persisted inside Orama index files (`.msp`). Orama persists
+ * `tokenizer.language` with the index, so this doubles as the tokenization
+ * version: `OramaBackend` detects any other marker on restore and rebuilds
+ * the full-text index (issue #2187). Bump the suffix whenever tokenization
+ * changes.
+ */
+export const ORAMA_CJK_TOKENIZER_LANGUAGE = "english+cjk-v1";
+
+/** Minimal structural surface of `@orama/orama` this module consumes. */
+export interface OramaTokenizerComponents {
+  components: {
+    tokenizer: {
+      createTokenizer: (config: { language: string }) => OramaTokenizer;
+    };
+  };
+}
+
+export interface OramaTokenizer {
+  language: string;
+  normalizationCache: Map<string, string>;
+  tokenize: (raw: string, language?: string, prop?: string, withCache?: boolean) => string[];
+}
+
+const THAI_SCRIPT_CHAR = /[\p{Script=Thai}]/u;
+const WORD_CHAR = /[\p{L}\p{N}\p{M}]/u;
+const COMBINING_MARK = /\p{M}/u;
+const WHITESPACE_CHAR = /\s/u;
+/**
+ * Chars the stock English tokenizer keeps inside a token. Input made only of
+ * these (plus whitespace) is delegated to the stock tokenizer byte-for-byte so
+ * existing English indexes stay term-compatible.
+ */
+const LEGACY_TOKENIZER_CHAR = /[A-Za-zàèéìòóù0-9_'-]/u;
+
+/**
+ * Space-free scripts: CJK per the shared recall segmentation strategy, plus
+ * Thai. Runs in these scripts are indexed as character n-grams so phrase
+ * queries match without word boundaries.
+ */
+export function isSpaceFreeScriptChar(char: string): boolean {
+  return isUnsegmentableRecallChar(char) || THAI_SCRIPT_CHAR.test(char);
+}
+
+/**
+ * True when tokenizing the value differs from the stock English tokenizer —
+ * i.e. the value contains characters outside the stock keep-set that this
+ * tokenizer still indexes as token material: word characters (whole-word
+ * non-ASCII scripts), combining marks (attached to runs), and space-free
+ * scripts (CJK/Thai n-grams). Symbols and punctuation that are separators in
+ * BOTH tokenizers do not count. `OramaBackend` uses this to decide whether a
+ * stale pre-CJK index needs re-indexing on upgrade.
+ */
+export function containsNonLegacyTokenizerChars(value: string): boolean {
+  for (const ch of value) {
+    if (LEGACY_TOKENIZER_CHAR.test(ch) || WHITESPACE_CHAR.test(ch)) continue;
+    if (WORD_CHAR.test(ch) || COMBINING_MARK.test(ch) || isSpaceFreeScriptChar(ch)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Orama tokenizer component that segments space-free scripts (CJK, Thai).
+ *
+ * - CJK/Thai runs expand to the same n-gram set the recall query-side
+ *   tokenizer (`recall-tokenization.ts`) produces, so index-side and
+ *   query-side tokens agree.
+ * - Other non-Latin scripts (Hangul, Cyrillic, Greek, Arabic, ...) are kept
+ *   as whole words instead of being dropped by the English splitter.
+ * - Legacy Latin content tokenizes exactly like the stock English tokenizer;
+ *   content with other characters keeps every stock token and additionally
+ *   indexes non-ASCII words whole (e.g. "über" adds "über" beside "ber").
+ */
+export function createCjkCapableTokenizer(oramaModule: OramaTokenizerComponents): OramaTokenizer {
+  const base = oramaModule.components.tokenizer.createTokenizer({ language: "english" });
+
+  const tokenize = (raw: string, _language?: string, prop?: string, withCache?: boolean): string[] => {
+    if (typeof raw !== "string") return [];
+    const normalized = raw.normalize("NFC");
+
+    if (!containsNonLegacyTokenizerChars(normalized)) {
+      return base.tokenize(normalized, "english", prop, withCache);
+    }
+
+    // The stock tokenizer treats every non-legacy char as a separator, so its
+    // output over the raw input already carries the legacy-Latin tokens
+    // (e.g. the "nebula-472" in "Nebula-472 東京都庁").
+    const tokens = base.tokenize(normalized, "english", prop, withCache);
+    const seen = new Set(tokens);
+
+    const pushToken = (token: string) => {
+      if (token && !seen.has(token)) {
+        seen.add(token);
+        tokens.push(token);
+      }
+    };
+
+    let spaceFreeRun = "";
+    let wordRun = "";
+    const flushSpaceFreeRun = () => {
+      if (!spaceFreeRun) return;
+      for (const token of expandUnsegmentableRecallNGrams(spaceFreeRun)) {
+        pushToken(token);
+      }
+      spaceFreeRun = "";
+    };
+    const flushWordRun = () => {
+      if (!wordRun) return;
+      pushToken(wordRun.toLowerCase());
+      wordRun = "";
+    };
+
+    for (const ch of normalized) {
+      if (isSpaceFreeScriptChar(ch)) {
+        flushWordRun();
+        spaceFreeRun += ch;
+      } else if (COMBINING_MARK.test(ch)) {
+        if (spaceFreeRun) spaceFreeRun += ch;
+        else if (wordRun) wordRun += ch;
+      } else if (WORD_CHAR.test(ch) && !LEGACY_TOKENIZER_CHAR.test(ch)) {
+        flushSpaceFreeRun();
+        wordRun += ch;
+      } else {
+        flushSpaceFreeRun();
+        flushWordRun();
+      }
+    }
+    flushSpaceFreeRun();
+    flushWordRun();
+
+    return tokens;
+  };
+
+  return {
+    language: ORAMA_CJK_TOKENIZER_LANGUAGE,
+    normalizationCache: base.normalizationCache,
+    tokenize,
+  };
+}