@cachly-dev/mcp-server 0.5.36 → 0.5.37

package/src/search.test.ts

@@ -4,13 +4,16 @@
  * Run: npx vitest run src/search.test.ts
  */
 import { describe, it, expect } from 'vitest';
-import { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS } from './index.js';
+import { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS,
+         katakanaToRomaji, arabicLightStem, expandCrossLingual, CROSS_LINGUAL_MAP } from './index.js';
 
 // ── Tokenizer ─────────────────────────────────────────────────────────────────
 
 describe('tokenize', () => {
   it('lowercases and splits on whitespace', () => {
-    expect(tokenize('Deploy API')).toEqual(['deploy', 'api']);
+    const result = tokenize('Deploy API');
+    expect(result).toContain('deploy');
+    expect(result).toContain('api');
   });
 
   it('filters English stopwords', () => {
@@ -69,7 +72,12 @@ describe('tokenize', () => {
   });
 
   it('removes single-char tokens', () => {
-    expect(tokenize('a b c deploy')).toEqual(['deploy']);
+    const result = tokenize('a b c deploy');
+    expect(result).toContain('deploy');
+    // single chars a, b, c must not appear
+    expect(result).not.toContain('a');
+    expect(result).not.toContain('b');
+    expect(result).not.toContain('c');
   });
 
   it('preserves colons and dots in tokens (topic slugs)', () => {
@@ -280,3 +288,209 @@ describe('extractTimestamp', () => {
     expect(extractTimestamp('{"ts":"not-a-date"}')).toBeUndefined();
   });
 });
+
+// ── Katakana → Romaji ──────────────────────────────────────────────────────────
+
+describe('katakanaToRomaji', () => {
+  it('converts basic katakana vowels', () => {
+    expect(katakanaToRomaji('アイウエオ')).toBe('aiueo');
+  });
+
+  it('converts コンテナ (container)', () => {
+    expect(katakanaToRomaji('コンテナ')).toBe('kontena');
+  });
+
+  it('converts デプロイ (deploy)', () => {
+    expect(katakanaToRomaji('デプロイ')).toBe('depuroi');
+  });
+
+  it('converts サーバー (server) — long vowel stripped', () => {
+    const r = katakanaToRomaji('サーバー');
+    expect(r).toContain('sa');
+    expect(r).toContain('ba');
+  });
+
+  it('converts digraph チェック (check)', () => {
+    const r = katakanaToRomaji('チェック');
+    expect(r).toContain('che');
+  });
+
+  it('converts ファイル (file)', () => {
+    const r = katakanaToRomaji('ファイル');
+    expect(r).toContain('fa');
+  });
+
+  it('handles empty string', () => {
+    expect(katakanaToRomaji('')).toBe('');
+  });
+});
+
+// ── Arabic Light Stemmer ───────────────────────────────────────────────────────
+
+describe('arabicLightStem', () => {
+  it('strips definite article ال (al-)', () => {
+    // الخطأ (al-khata = the error) → خطأ (khata)
+    expect(arabicLightStem('الخطأ')).toBe('خطأ');
+  });
+
+  it('strips و (wa-) prefix from long words', () => {
+    // وخطأ → خطأ (and error → error)
+    expect(arabicLightStem('وخطأ')).toBe('خطأ');
+  });
+
+  it('strips ب (bi-) prefix', () => {
+    expect(arabicLightStem('بسرعة')).toBe('سرعة');
+  });
+
+  it('does not strip from short words (≤3 chars)', () => {
+    // بلد (3 chars) should not be stripped
+    expect(arabicLightStem('بلد')).toBe('بلد');
+  });
+
+  it('returns unchanged for Latin input', () => {
+    expect(arabicLightStem('deploy')).toBe('deploy');
+  });
+});
+
+// ── Cross-lingual Expansion ────────────────────────────────────────────────────
+
+describe('expandCrossLingual', () => {
+  it('returns Japanese katakana for "deploy"', () => {
+    const syns = expandCrossLingual('deploy');
+    expect(syns).toContain('デプロイ');
+  });
+
+  it('returns Chinese for "deploy"', () => {
+    expect(expandCrossLingual('deploy')).toContain('部署');
+  });
+
+  it('returns Korean for "deploy"', () => {
+    expect(expandCrossLingual('deploy')).toContain('배포');
+  });
+
+  it('returns English for Japanese デプロイ', () => {
+    expect(expandCrossLingual('デプロイ')).toContain('deploy');
+  });
+
+  it('returns English for Chinese 部署', () => {
+    expect(expandCrossLingual('部署')).toContain('deploy');
+  });
+
+  it('returns English for Korean 배포', () => {
+    expect(expandCrossLingual('배포')).toContain('deploy');
+  });
+
+  it('returns English for Arabic خطأ (error)', () => {
+    expect(expandCrossLingual('خطأ')).toContain('error');
+  });
+
+  it('returns English for Hebrew שגיאה (error)', () => {
+    expect(expandCrossLingual('שגיאה')).toContain('error');
+  });
+
+  it('returns empty array for unknown tokens', () => {
+    expect(expandCrossLingual('foobar')).toEqual([]);
+  });
+
+  it('covers all major languages for "server"', () => {
+    const syns = expandCrossLingual('server');
+    expect(syns).toContain('サーバー');
+    expect(syns).toContain('服务器');
+    expect(syns).toContain('서버');
+  });
+
+  it('has at least 50 tech terms in the map', () => {
+    expect(CROSS_LINGUAL_MAP.size).toBeGreaterThanOrEqual(50);
+  });
+});
+
+// ── RTL Tokenization ───────────────────────────────────────────────────────────
+
+describe('tokenize — RTL (Arabic/Hebrew)', () => {
+  it('tokenizes Arabic text into words', () => {
+    // "خطأ في النظام" = error in the system
+    const tokens = tokenize('خطأ في النظام');
+    expect(tokens.length).toBeGreaterThan(0);
+    // خطأ = error, not a stopword
+    const hasError = tokens.some(t => t === 'خطأ' || t === 'error');
+    expect(hasError).toBe(true);
+  });
+
+  it('strips Arabic definite article via light stemmer', () => {
+    // النظام → نظام (the system → system)
+    const tokens = tokenize('النظام');
+    expect(tokens).toContain('نظام');
+  });
+
+  it('filters Arabic stopwords', () => {
+    const tokens = tokenize('في على من');
+    // All three are stopwords
+    expect(tokens.filter(t => RTL_RE_TEST(t)).length).toBe(0);
+  });
+
+  it('tokenizes Hebrew text', () => {
+    // שגיאה = error
+    const tokens = tokenize('שגיאה בשרת');
+    expect(tokens.length).toBeGreaterThan(0);
+    const hasErr = tokens.some(t => t === 'שגיאה' || t === 'error');
+    expect(hasErr).toBe(true);
+  });
+
+  it('handles mixed Arabic + English text', () => {
+    const tokens = tokenize('deploy خطأ error');
+    expect(tokens).toContain('deploy');
+    expect(tokens.some(t => t === 'خطأ' || t === 'error')).toBe(true);
+  });
+});
+
+// Helper: test if a string contains RTL characters
+function RTL_RE_TEST(s: string) {
+  return /[\u0590-\u05ff\u0600-\u06ff]/.test(s);
+}
+
+// ── Cross-lingual Search Integration ──────────────────────────────────────────
+
+describe('tokenize — cross-lingual expansion', () => {
+  it('English "deploy" expands to Japanese bigrams', () => {
+    const tokens = tokenize('deploy');
+    // デプロイ bigrams: デプ, プロ, ロイ
+    expect(tokens.some(t => t === 'デプ' || t === 'プロ' || t === 'ロイ')).toBe(true);
+  });
+
+  it('Japanese デプロイ expands to include "deploy"', () => {
+    const tokens = tokenize('デプロイ');
+    expect(tokens).toContain('deploy');
+  });
+
+  it('Chinese 部署 expands to "deploy"', () => {
+    const tokens = tokenize('部署');
+    expect(tokens).toContain('deploy');
+  });
+
+  it('Korean 배포 expands to "deploy"', () => {
+    const tokens = tokenize('배포');
+    expect(tokens).toContain('deploy');
+  });
+
+  it('Arabic خطأ expands to "error"', () => {
+    const tokens = tokenize('خطأ');
+    expect(tokens).toContain('error');
+  });
+
+  it('Hebrew שגיאה expands to "error"', () => {
+    const tokens = tokenize('שגיאה');
+    expect(tokens).toContain('error');
+  });
+
+  it('English "container" expands to all CJK variants', () => {
+    const tokens = tokenize('container');
+    // コンテナ bigrams should be present
+    expect(tokens.some(t => t === 'コン' || t === 'ンテ' || t === 'テナ')).toBe(true);
+  });
+
+  it('katakana コンテナ generates romaji "kontena"', () => {
+    const tokens = tokenize('コンテナ');
+    expect(tokens).toContain('kontena');
+  });
+});
+