@cachly-dev/mcp-server 0.5.36 → 0.5.37

package/src/index.ts

@@ -301,8 +301,11 @@ function embedProviderHint(): string {
 //   • Recency boost — newer entries rank higher (exponential decay, 7-day half-life)
 //   • Multi-query splitting — numbered lists, semicolons, conjunctions
 //   • Fuzzy matching with Levenshtein distance ≤ 2
-//   • Multilingual stopwords (EN, DE, FR, ES, IT, PT, ZH, JA, KO)
+//   • Multilingual stopwords (EN, DE, FR, ES, IT, PT, ZH, JA, KO, AR, HE)
 //   • CJK character bigram extraction — handles Chinese, Japanese, Korean
+//   • RTL language support (Arabic, Hebrew) — word tokenization + light stemming
+//   • Romanization matching — katakana → romaji tokens for romaji queries
+//   • Cross-language retrieval — tech term synonyms EN↔JA↔ZH↔KO↔AR↔HE
 //   • Pipeline Redis reads for performance
 //
 
@@ -400,16 +403,331 @@ const STOPWORDS = new Set([
   '이', '가', '은', '는', '을', '를', '의', '에', '와', '과', '도',
   '로', '에서', '한', '하', '있', '없', '것', '수', '않', '들',
   '이다', '하다', '이고', '이며', '라고', '에게', '에서', '으로',
+  // ── Arabic — high-frequency function words & particles ──
+  'في', 'من', 'إلى', 'على', 'عن', 'مع', 'هذا', 'هذه', 'ذلك', 'تلك',
+  'الذي', 'التي', 'الذين', 'اللواتي', 'هو', 'هي', 'هم', 'هن', 'نحن',
+  'أنت', 'أنتم', 'أنا', 'كان', 'كانت', 'كانوا', 'يكون', 'تكون',
+  'أن', 'إن', 'لأن', 'لكن', 'أو', 'بل', 'ثم', 'حتى', 'إذا', 'كي',
+  'قد', 'لقد', 'لم', 'لن', 'ما', 'لا', 'ليس', 'غير', 'بعض', 'كل',
+  'جميع', 'أي', 'كيف', 'متى', 'أين', 'لماذا', 'ماذا', 'هل', 'ال',
+  'و', 'ف', 'ب', 'ل', 'ك', 'يا', 'أم', 'إلا', 'عند', 'بين',
+  // ── Hebrew — common particles, pronouns, conjunctions ──
+  'של', 'את', 'אל', 'על', 'עם', 'לא', 'הוא', 'היא', 'הם', 'הן',
+  'אני', 'אתה', 'אנחנו', 'אתם', 'כי', 'אם', 'אבל', 'גם', 'כבר',
+  'רק', 'עוד', 'יש', 'אין', 'מה', 'זה', 'זאת', 'אלה', 'כל', 'כן',
+  'לו', 'לה', 'להם', 'בו', 'בה', 'בהם', 'שם', 'כך', 'כן', 'מי',
+  'אשר', 'אחרי', 'לפני', 'תחת', 'בין', 'מאז', 'עד', 'כמו', 'אז',
 ]);
 
 // CJK Unicode ranges used for bigram extraction
 const CJK_RE = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]/;
 const SEGMENT_RE = /([\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]+|[^\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]+)/g;
 
+// RTL Unicode ranges: Arabic (U+0600–U+06FF + extended), Hebrew (U+0590–U+05FF)
+const RTL_RE = /[\u0590-\u05ff\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff\ufb1d-\ufb4f\ufb50-\ufdff\ufe70-\ufeff]/;
+
+// ── Katakana → Romaji conversion table (Hepburn system) ──────────────────────
+// Digraphs must be listed before single chars so they match first.
+const KATA_ROMAJI_TABLE: [string, string][] = [
+  // Special combinations for loanwords
+  ['ファ', 'fa'], ['フィ', 'fi'], ['フェ', 'fe'], ['フォ', 'fo'],
+  ['ティ', 'ti'], ['ディ', 'di'], ['トゥ', 'tu'], ['ドゥ', 'du'],
+  ['ウィ', 'wi'], ['ウェ', 'we'], ['ウォ', 'wo'],
+  ['チェ', 'che'], ['ジェ', 'je'], ['シェ', 'she'],
+  ['イェ', 'ye'], ['ヴァ', 'va'], ['ヴィ', 'vi'], ['ヴェ', 've'], ['ヴォ', 'vo'],
+  // Digraphs (2-char → romaji)
+  ['キャ', 'kya'], ['キュ', 'kyu'], ['キョ', 'kyo'],
+  ['シャ', 'sha'], ['シュ', 'shu'], ['ショ', 'sho'],
+  ['チャ', 'cha'], ['チュ', 'chu'], ['チョ', 'cho'],
+  ['ニャ', 'nya'], ['ニュ', 'nyu'], ['ニョ', 'nyo'],
+  ['ヒャ', 'hya'], ['ヒュ', 'hyu'], ['ヒョ', 'hyo'],
+  ['ミャ', 'mya'], ['ミュ', 'myu'], ['ミョ', 'myo'],
+  ['リャ', 'rya'], ['リュ', 'ryu'], ['リョ', 'ryo'],
+  ['ギャ', 'gya'], ['ギュ', 'gyu'], ['ギョ', 'gyo'],
+  ['ジャ', 'ja'], ['ジュ', 'ju'], ['ジョ', 'jo'],
+  ['ビャ', 'bya'], ['ビュ', 'byu'], ['ビョ', 'byo'],
+  ['ピャ', 'pya'], ['ピュ', 'pyu'], ['ピョ', 'pyo'],
+  // Single chars
+  ['ア', 'a'], ['イ', 'i'], ['ウ', 'u'], ['エ', 'e'], ['オ', 'o'],
+  ['カ', 'ka'], ['キ', 'ki'], ['ク', 'ku'], ['ケ', 'ke'], ['コ', 'ko'],
+  ['サ', 'sa'], ['シ', 'shi'], ['ス', 'su'], ['セ', 'se'], ['ソ', 'so'],
+  ['タ', 'ta'], ['チ', 'chi'], ['ツ', 'tsu'], ['テ', 'te'], ['ト', 'to'],
+  ['ナ', 'na'], ['ニ', 'ni'], ['ヌ', 'nu'], ['ネ', 'ne'], ['ノ', 'no'],
+  ['ハ', 'ha'], ['ヒ', 'hi'], ['フ', 'fu'], ['ヘ', 'he'], ['ホ', 'ho'],
+  ['マ', 'ma'], ['ミ', 'mi'], ['ム', 'mu'], ['メ', 'me'], ['モ', 'mo'],
+  ['ヤ', 'ya'], ['ユ', 'yu'], ['ヨ', 'yo'],
+  ['ラ', 'ra'], ['リ', 'ri'], ['ル', 'ru'], ['レ', 're'], ['ロ', 'ro'],
+  ['ワ', 'wa'], ['ヲ', 'wo'], ['ン', 'n'],
+  // Voiced
+  ['ガ', 'ga'], ['ギ', 'gi'], ['グ', 'gu'], ['ゲ', 'ge'], ['ゴ', 'go'],
+  ['ザ', 'za'], ['ジ', 'ji'], ['ズ', 'zu'], ['ゼ', 'ze'], ['ゾ', 'zo'],
+  ['ダ', 'da'], ['ヂ', 'ji'], ['ヅ', 'zu'], ['デ', 'de'], ['ド', 'do'],
+  ['バ', 'ba'], ['ビ', 'bi'], ['ブ', 'bu'], ['ベ', 'be'], ['ボ', 'bo'],
+  ['パ', 'pa'], ['ピ', 'pi'], ['プ', 'pu'], ['ペ', 'pe'], ['ポ', 'po'],
+  ['ヴ', 'v'],
+  // Long vowel / small chars
+  ['ー', ''], ['ァ', 'a'], ['ィ', 'i'], ['ゥ', 'u'], ['ェ', 'e'], ['ォ', 'o'],
+  ['ッ', ''],  // handled separately (doubles next consonant)
+];
+
+const KATA_MAP = new Map<string, string>(KATA_ROMAJI_TABLE);
+
+/**
+ * Convert a katakana string to Hepburn romaji.
+ * Handles digraphs, geminate consonants (ッ), and long vowel marks (ー).
+ */
+function katakanaToRomaji(kata: string): string {
+  let result = '';
+  let i = 0;
+  while (i < kata.length) {
+    // Geminate consonant: ッ doubles the following consonant
+    if (kata[i] === 'ッ' && i + 1 < kata.length) {
+      const next = KATA_MAP.get(kata[i + 1]) ?? KATA_MAP.get(kata[i + 1] + kata[i + 2]) ?? '';
+      if (next.length > 0) result += next[0]; // double first consonant
+      i++;
+      continue;
+    }
+    // Try 2-char digraph first
+    if (i + 1 < kata.length) {
+      const two = kata[i] + kata[i + 1];
+      const r2 = KATA_MAP.get(two);
+      if (r2 !== undefined) { result += r2; i += 2; continue; }
+    }
+    // Single char
+    const r1 = KATA_MAP.get(kata[i]);
+    if (r1 !== undefined) result += r1;
+    i++;
+  }
+  return result;
+}
+
+/**
+ * Arabic light stemmer — strips definite article and common prefix particles.
+ * Handles: ال (al-), و (wa-), ب (bi-), ل (li-), ف (fa-), ك (ka-).
+ * Only strips when the result is still ≥ 3 chars to avoid over-stemming.
+ */
+function arabicLightStem(word: string): string {
+  if (word.startsWith('ال') && word.length > 4) return word.slice(2);
+  if (word.length > 3 && 'وبلفك'.includes(word[0]) && RTL_RE.test(word[1])) return word.slice(1);
+  return word;
+}
+
+// ── Cross-lingual tech term dictionary ────────────────────────────────────────
+// Maps tokens (English, Japanese katakana, Chinese, Korean, Arabic, Hebrew)
+// to their equivalents in other languages. Used to expand tokens at search time
+// so "deploy" finds documents containing "デプロイ" / "部署" / "배포", and vice versa.
+//
+// Keys must be lowercase. CJK values will be bigram-expanded at use time.
+const CROSS_LINGUAL_MAP = new Map<string, string[]>([
+  // English → all
+  ['deploy',      ['デプロイ', '部署', '배포', 'deployment', 'deploying']],
+  ['deployment',  ['デプロイ', '部署', '배포', 'deploy']],
+  ['container',   ['コンテナ', '容器', '컨테이너', 'docker']],
+  ['server',      ['サーバー', 'サーバ', '服务器', '서버', 'سيرفر', 'שרת']],
+  ['database',    ['データベース', '数据库', '데이터베이스', 'db', 'قاعدة البيانات']],
+  ['cache',       ['キャッシュ', '缓存', '캐시', 'كاش', 'מטמון']],
+  ['error',       ['エラー', '错误', '오류', 'خطأ', 'שגיאה', 'exception', 'err']],
+  ['bug',         ['バグ', '缺陷', '버그', 'issue', 'defect', 'خلل']],
+  ['fix',         ['修正', '修复', '수정', 'bugfix', 'patch', 'hotfix', 'إصلاح', 'תיקון']],
+  ['build',       ['ビルド', '构建', '빌드', 'بناء', 'בנייה']],
+  ['test',        ['テスト', '测试', '테스트', 'اختبار', 'בדיקה']],
+  ['auth',        ['認証', '认证', '인증', 'authentication', 'login', 'oauth', 'مصادقة', 'אימות']],
+  ['kubernetes',  ['クベルネテス', 'k8s', 'kube']],
+  ['network',     ['ネットワーク', '网络', '네트워크', 'شبكة', 'רשת']],
+  ['timeout',     ['タイムアウト', '超时', '타임아웃', 'مهلة']],
+  ['memory',      ['メモリ', '内存', '메모리', 'ram', 'ذاكرة', 'זיכרון']],
+  ['config',      ['設定', '配置', '설정', 'configuration', 'settings', 'conf', 'إعدادات', 'הגדרות']],
+  ['install',     ['インストール', '安装', '설치', 'تثبيت', 'התקנה']],
+  ['update',      ['アップデート', '更新', '업데이트', 'upgrade', 'تحديث', 'עדכון']],
+  ['log',         ['ログ', '日志', '로그', 'logging', 'سجل', 'יומן']],
+  ['port',        ['ポート', '端口', '포트', 'منفذ', 'פורט']],
+  ['file',        ['ファイル', '文件', '파일', 'ملف', 'קובץ']],
+  ['image',       ['イメージ', '镜像', '이미지', 'صورة', 'תמונה']],
+  ['volume',      ['ボリューム', '卷', '볼륨']],
+  ['cluster',     ['クラスター', '集群', '클러스터']],
+  ['node',        ['ノード', '节点', '노드']],
+  ['service',     ['サービス', '服务', '서비스', 'svc', 'خدمة', 'שירות']],
+  ['certificate', ['証明書', '证书', '인증서', 'cert', 'ssl', 'tls', 'شهادة', 'תעודה']],
+  ['password',    ['パスワード', '密码', '비밀번호', 'passwd', 'pwd', 'كلمة المرور', 'סיסמה']],
+  ['token',       ['トークン', '令牌', '토큰', 'jwt', 'secret', 'رمز', 'אסימון']],
+  ['health',      ['ヘルス', '健康', '헬스', 'healthcheck', 'probe', 'صحة']],
+  ['migration',   ['マイグレーション', '迁移', '마이그레이션', 'migrate', 'هجرة']],
+  ['backup',      ['バックアップ', '备份', '백업', 'نسخ احتياطي', 'גיבוי']],
+  ['monitor',     ['モニター', '监控', '모니터링', 'monitoring', 'metrics', 'alert', 'مراقبة', 'ניטור']],
+  ['performance', ['パフォーマンス', '性能', '성능', 'latency', 'throughput']],
+  ['connection',  ['接続', '连接', '연결', 'conn', 'socket', 'اتصال', 'חיבור']],
+  ['queue',       ['キュー', '队列', '큐', 'طابور']],
+  ['redis',       ['レディス', '레디스']],
+  ['nginx',       ['エンジンエックス']],
+  ['linux',       ['リナックス', 'لينكس']],
+  ['api',         ['エーピーアイ', '接口', 'endpoint', 'واجهة برمجية', 'ממשק']],
+  ['healthcheck', ['ヘルスチェック', 'health', 'probe', 'فحص الصحة']],
+  ['loadbalancer',['ロードバランサー', '负载均衡', '로드밸런서', 'lb']],
+  ['ssl',         ['tls', 'https', 'certificate', 'cert']],
+  ['tls',         ['ssl', 'https', 'certificate', 'cert']],
+  ['docker',      ['container', 'コンテナ', '容器', '컨테이너', 'دوكر']],
+  ['git',         ['version control', 'repo', 'repository', 'commit', 'push', 'pull']],
+  ['ci',          ['pipeline', 'github actions', 'gitlab', 'jenkins', 'build']],
+  ['cd',          ['deploy', 'deployment', 'release']],
+  ['debug',       ['デバッグ', '调试', '디버그', 'debugging', 'breakpoint', 'تصحيح', 'איתור באגים']],
+  ['crash',       ['クラッシュ', '崩溃', '크래시', 'panic', 'segfault', 'انهيار']],
+  ['restart',     ['再起動', '重启', '재시작', 'reboot', 'إعادة تشغيل', 'הפעלה מחדש']],
+  ['permission',  ['権限', '权限', '권한', 'access', 'acl', 'chmod', 'صلاحية', 'הרשאה']],
+  // Japanese katakana → English
+  ['デプロイ',    ['deploy', 'deployment']],
+  ['コンテナ',    ['container', 'docker']],
+  ['サーバー',    ['server']],
+  ['サーバ',      ['server']],
+  ['データベース',['database', 'db']],
+  ['キャッシュ',  ['cache']],
+  ['エラー',      ['error', 'err']],
+  ['バグ',        ['bug', 'issue']],
+  ['ビルド',      ['build']],
+  ['テスト',      ['test']],
+  ['認証',        ['auth', 'authentication']],
+  ['設定',        ['config', 'configuration']],
+  ['インストール',['install']],
+  ['アップデート',['update', 'upgrade']],
+  ['ログ',        ['log', 'logs']],
+  ['ポート',      ['port']],
+  ['ファイル',    ['file']],
+  ['イメージ',    ['image']],
+  ['クラスター',  ['cluster']],
+  ['ノード',      ['node']],
+  ['サービス',    ['service']],
+  ['パスワード',  ['password']],
+  ['トークン',    ['token']],
+  ['バックアップ',['backup']],
+  ['モニター',    ['monitor', 'monitoring']],
+  ['接続',        ['connection']],
+  ['キュー',      ['queue']],
+  ['ヘルスチェック',['healthcheck', 'health check', 'health']],
+  ['ロードバランサー',['loadbalancer', 'load balancer']],
+  ['デバッグ',    ['debug']],
+  ['クラッシュ',  ['crash', 'panic']],
+  ['再起動',      ['restart', 'reboot']],
+  ['権限',        ['permission', 'access']],
+  // Chinese → English
+  ['部署',        ['deploy', 'deployment']],
+  ['容器',        ['container', 'docker']],
+  ['服务器',      ['server']],
+  ['数据库',      ['database']],
+  ['缓存',        ['cache']],
+  ['错误',        ['error']],
+  ['修复',        ['fix', 'bugfix']],
+  ['构建',        ['build']],
+  ['测试',        ['test']],
+  ['认证',        ['auth']],
+  ['配置',        ['config']],
+  ['安装',        ['install']],
+  ['更新',        ['update']],
+  ['日志',        ['log']],
+  ['端口',        ['port']],
+  ['镜像',        ['image']],
+  ['集群',        ['cluster']],
+  ['节点',        ['node']],
+  ['服务',        ['service']],
+  ['密码',        ['password']],
+  ['令牌',        ['token']],
+  ['备份',        ['backup']],
+  ['监控',        ['monitor']],
+  ['连接',        ['connection']],
+  ['队列',        ['queue']],
+  ['性能',        ['performance']],
+  ['内存',        ['memory']],
+  ['调试',        ['debug']],
+  ['崩溃',        ['crash']],
+  ['重启',        ['restart']],
+  ['权限',        ['permission']],
+  // Korean → English
+  ['배포',        ['deploy', 'deployment']],
+  ['컨테이너',    ['container']],
+  ['서버',        ['server']],
+  ['데이터베이스',['database']],
+  ['캐시',        ['cache']],
+  ['오류',        ['error']],
+  ['수정',        ['fix']],
+  ['빌드',        ['build']],
+  ['테스트',      ['test']],
+  ['인증',        ['auth']],
+  ['설정',        ['config']],
+  ['설치',        ['install']],
+  ['업데이트',    ['update']],
+  ['로그',        ['log']],
+  ['포트',        ['port']],
+  ['이미지',      ['image']],
+  ['클러스터',    ['cluster']],
+  ['노드',        ['node']],
+  ['서비스',      ['service']],
+  ['토큰',        ['token']],
+  ['백업',        ['backup']],
+  ['모니터링',    ['monitor']],
+  ['연결',        ['connection']],
+  ['큐',          ['queue']],
+  ['성능',        ['performance']],
+  ['메모리',      ['memory']],
+  ['디버그',      ['debug']],
+  ['크래시',      ['crash']],
+  ['재시작',      ['restart']],
+  ['권한',        ['permission']],
+  // Arabic → English
+  ['خطأ',         ['error']],
+  ['إصلاح',       ['fix']],
+  ['بناء',        ['build']],
+  ['اختبار',      ['test']],
+  ['مصادقة',      ['auth']],
+  ['إعدادات',     ['config']],
+  ['تثبيت',       ['install']],
+  ['تحديث',       ['update']],
+  ['سجل',         ['log']],
+  ['صورة',        ['image']],
+  ['خدمة',        ['service']],
+  ['نسخ احتياطي', ['backup']],
+  ['مراقبة',      ['monitor']],
+  ['اتصال',       ['connection']],
+  ['ذاكرة',       ['memory']],
+  ['تصحيح',       ['debug']],
+  ['شبكة',        ['network']],
+  // Hebrew → English
+  ['שגיאה',       ['error']],
+  ['תיקון',       ['fix']],
+  ['בנייה',       ['build']],
+  ['בדיקה',       ['test']],
+  ['אימות',       ['auth']],
+  ['הגדרות',      ['config']],
+  ['התקנה',       ['install']],
+  ['עדכון',       ['update']],
+  ['יומן',        ['log']],
+  ['שרת',         ['server']],
+  ['שירות',       ['service']],
+  ['גיבוי',       ['backup']],
+  ['ניטור',       ['monitor']],
+  ['חיבור',       ['connection']],
+  ['זיכרון',      ['memory']],
+  ['רשת',         ['network']],
+  ['תמונה',       ['image']],
+  ['סיסמה',       ['password']],
+  ['הרשאה',       ['permission']],
+]);
+
+/**
+ * Expand a token to its cross-lingual synonyms.
+ * Returns synonyms to be added to the token stream (pre-tokenized).
+ * CJK synonyms are NOT pre-bigram'd here — caller handles that.
+ */
+function expandCrossLingual(token: string): string[] {
+  return CROSS_LINGUAL_MAP.get(token) ?? [];
+}
+
 /**
  * Tokenize text into meaningful keywords.
- * Handles accented Latin characters and CJK (Chinese, Japanese, Korean).
- * CJK text is split into character bigrams (the standard for indexing logographic languages).
+ * Handles accented Latin, RTL (Arabic, Hebrew), and CJK (Chinese, Japanese, Korean).
+ *
+ * Features:
+ *  • CJK → character bigrams
+ *  • Katakana → additionally emits Hepburn romaji tokens (enables romaji queries)
+ *  • Arabic/Hebrew → word tokenization with light Arabic stemming
+ *  • All scripts → cross-lingual synonym expansion (EN↔JA↔ZH↔KO↔AR↔HE)
  */
 function tokenize(text: string): string[] {
   const tokens: string[] = [];
@@ -429,8 +747,33 @@ function tokenize(text: string): string[] {
         const last = seg[seg.length - 1];
         if (!STOPWORDS.has(last)) tokens.push(last);
       }
+      // Whole-segment cross-lingual lookup (for known tech terms like デプロイ, 部署, 배포)
+      // This ensures CJK full-words map to their English equivalents
+      const segSyns = expandCrossLingual(seg);
+      for (const syn of segSyns) {
+        if (!CJK_RE.test(syn)) {
+          syn.split(/\s+/).filter(w => w.length > 1 && !STOPWORDS.has(w)).forEach(w => tokens.push(w));
+        }
+      }
+      // Romanization: extract katakana-only portion and emit romaji tokens.
+      // This allows users to search "kontena" or "depuroi" to find katakana docs.
+      const kataOnly = seg.replace(/[^\u30a0-\u30ff]/g, '');
+      if (kataOnly.length >= 2) {
+        const romaji = katakanaToRomaji(kataOnly);
+        if (romaji.length >= 2 && romaji.length <= 40 && !STOPWORDS.has(romaji)) {
+          tokens.push(romaji);
+        }
+      }
+    } else if (RTL_RE.test(seg)) {
+      // RTL segment (Arabic / Hebrew) — space-separated words with light stemming
+      const words = seg.trim().split(/\s+/);
+      for (const w of words) {
+        if (w.length <= 1 || STOPWORDS.has(w)) continue;
+        const stemmed = arabicLightStem(w);
+        if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) tokens.push(stemmed);
+      }
     } else {
-      // Latin / other — existing logic
+      // Latin / other — existing logic preserved exactly
       const words = seg
         .replace(/[^a-záàâãäåæçéèêëíìîïñóòôõöúùûüýÿßœ0-9:_\-./]/g, ' ')
         .split(/\s+/)
@@ -439,6 +782,39 @@ function tokenize(text: string): string[] {
     }
   }
 
+  // ── Cross-lingual expansion ──────────────────────────────────────────────
+  // For each token, look up synonyms in other languages and add them.
+  // CJK synonyms are bigram-expanded inline; Latin/RTL synonyms added as-is.
+  const expansions: string[] = [];
+  const seen = new Set(tokens);
+  for (const tok of tokens) {
+    const syns = expandCrossLingual(tok);
+    for (const syn of syns) {
+      if (seen.has(syn)) continue;
+      seen.add(syn);
+      if (CJK_RE.test(syn)) {
+        // CJK synonym → expand to character bigrams
+        for (let i = 0; i < syn.length - 1; i++) {
+          const bg = syn[i] + syn[i + 1];
+          if (!STOPWORDS.has(syn[i]) && !STOPWORDS.has(bg)) expansions.push(bg);
+        }
+        if (syn.length >= 1) {
+          const last = syn[syn.length - 1];
+          if (!STOPWORDS.has(last)) expansions.push(last);
+        }
+      } else if (RTL_RE.test(syn)) {
+        // RTL synonym
+        const stemmed = arabicLightStem(syn);
+        if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) expansions.push(stemmed);
+      } else {
+        // Latin synonym — split in case of multi-word (e.g. "load balancer")
+        const parts = syn.split(/\s+/).filter(w => w.length > 1 && !STOPWORDS.has(w));
+        expansions.push(...parts);
+      }
+    }
+  }
+  tokens.push(...expansions);
+
   return tokens;
 }
 
@@ -752,7 +1128,8 @@ async function keywordSearch(
 }
 
 // ── Exported for testing ──────────────────────────────────────────────────────
-export { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS };
+export { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS,
+         katakanaToRomaji, arabicLightStem, expandCrossLingual, CROSS_LINGUAL_MAP };
 
 // ── Types ─────────────────────────────────────────────────────────────────────