@cachly-dev/mcp-server 0.5.60 → 0.5.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +309 -19
- package/dist/index.js.map +1 -1
- package/dist/search.test.js +103 -0
- package/dist/search.test.js.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +298 -19
- package/src/search.test.ts +113 -0
package/src/index.ts
CHANGED
|
@@ -55,7 +55,7 @@ import { Redis } from 'ioredis';
|
|
|
55
55
|
const API_URL = process.env.CACHLY_API_URL ?? 'https://api.cachly.dev';
|
|
56
56
|
const JWT = process.env.CACHLY_JWT ?? '';
|
|
57
57
|
const EMBED_MODEL = process.env.CACHLY_EMBED_MODEL ?? '';
|
|
58
|
-
const CURRENT_VERSION = '0.5.
|
|
58
|
+
const CURRENT_VERSION = '0.5.80';
|
|
59
59
|
/**
|
|
60
60
|
* ┌──────────────────────────────────────────────────────────────────────┐
|
|
61
61
|
* │ EMBEDDING PROVIDER — pluggable, client-side first │
|
|
@@ -451,6 +451,30 @@ const STOPWORDS = new Set([
|
|
|
451
451
|
'çok', 'en', 'her', 'hiç', 'bazı', 'tüm', 'bütün', 'hem', 'veya',
|
|
452
452
|
'bu', 'şu', 'ben', 'sen', 'biz', 'siz', 'onlar', 'benim', 'senin',
|
|
453
453
|
'nasıl', 'neden', 'niçin', 'nerede', 'ne', 'hangi', 'kaç',
|
|
454
|
+
// ── Polish — common prepositions, pronouns, particles ─────────────────────
|
|
455
|
+
'w', 'na', 'z', 'do', 'się', 'że', 'to', 'jest', 'i', 'a', 'nie',
|
|
456
|
+
'tak', 'jak', 'czy', 'po', 'o', 'ale', 'go', 'mu', 'jej', 'ich',
|
|
457
|
+
'tego', 'tej', 'te', 'ten', 'ta', 'są', 'był', 'była', 'było', 'byli',
|
|
458
|
+
'będzie', 'będą', 'ma', 'mam', 'masz', 'mają', 'ze', 'co', 'już',
|
|
459
|
+
'przez', 'przy', 'za', 'bez', 'nad', 'pod', 'przed', 'po', 'między',
|
|
460
|
+
'kiedy', 'gdzie', 'dlaczego', 'który', 'która', 'które', 'tylko', 'też',
|
|
461
|
+
// ── Czech — common words ──────────────────────────────────────────────────
|
|
462
|
+
'v', 'na', 'z', 'do', 'se', 'že', 'to', 'je', 'a', 'ne', 'pro',
|
|
463
|
+
'ale', 'jak', 'by', 'byl', 'být', 'jsem', 'jsou', 'má', 'mám',
|
|
464
|
+
'ten', 'ta', 'ty', 'tato', 'jeho', 'její', 'jejich', 'také', 'jen',
|
|
465
|
+
'kde', 'kdy', 'proč', 'který', 'která', 'které', 'co', 'při', 'bez',
|
|
466
|
+
// ── Bengali — common particles, pronouns, verbs ───────────────────────────
|
|
467
|
+
'এই', 'এটি', 'এটা', 'এর', 'তার', 'তারা', 'আমি', 'তুমি', 'সে', 'আমরা',
|
|
468
|
+
'এবং', 'কিন্তু', 'বা', 'না', 'হ্যাঁ', 'যে', 'কি', 'কে', 'কী', 'থেকে',
|
|
469
|
+
'দিয়ে', 'জন্য', 'সাথে', 'মধ্যে', 'উপর', 'নিচে', 'আছে', 'ছিল', 'হয়',
|
|
470
|
+
'করা', 'করে', 'করেছে', 'হবে', 'হয়েছে', 'একটি', 'একটা', 'অনেক', 'সব',
|
|
471
|
+
// ── Vietnamese — common particles, pronouns, auxiliary words ─────────────
|
|
472
|
+
'tôi', 'bạn', 'anh', 'chị', 'em', 'họ', 'chúng', 'ta', 'mình',
|
|
473
|
+
'là', 'có', 'không', 'và', 'với', 'của', 'cho', 'trong', 'về',
|
|
474
|
+
'từ', 'đến', 'để', 'khi', 'nếu', 'thì', 'mà', 'nhưng', 'vì',
|
|
475
|
+
'đây', 'đó', 'này', 'kia', 'rất', 'cũng', 'đã', 'sẽ', 'đang',
|
|
476
|
+
'được', 'bị', 'những', 'các', 'một', 'hai', 'ba', 'nhiều', 'ít',
|
|
477
|
+
'nào', 'ai', 'gì', 'đâu', 'sao', 'bao', 'giờ', 'lúc', 'sau',
|
|
454
478
|
]);
|
|
455
479
|
|
|
456
480
|
|
|
@@ -471,6 +495,11 @@ const DEVANAGARI_RE = /[\u0900-\u097f]/;
|
|
|
471
495
|
const CYRILLIC_RE = /[\u0400-\u04ff]/;
|
|
472
496
|
// Turkish uses Latin alphabet but has unique chars (ğ, ş, ı, ö, ü, ç) — detected by these
|
|
473
497
|
const TURKISH_CHAR_RE = /[\u011f\u015f\u0131\u0130\u00e7]/; // ğ ş ı İ ç
|
|
498
|
+
// Bengali (Bangla) — U+0980–U+09FF
|
|
499
|
+
const BENGALI_RE = /[\u0980-\u09ff]/;
|
|
500
|
+
// Vietnamese uses Latin + Latin Extended Additional (U+1EA0–U+1EF9) for tone marks
|
|
501
|
+
// e.g. ắ ặ ầ ổ ợ ụ ừ — detected by chars exclusive to Vietnamese diacritics
|
|
502
|
+
const VIETNAMESE_CHAR_RE = /[\u1ea0-\u1ef9]/;
|
|
474
503
|
|
|
475
504
|
// ── Katakana → Romaji conversion table (Hepburn system) ──────────────────────
|
|
476
505
|
// Digraphs must be listed before single chars so they match first.
|
|
@@ -728,6 +757,59 @@ function turkishLightStem(word: string): string {
|
|
|
728
757
|
return w;
|
|
729
758
|
}
|
|
730
759
|
|
|
760
|
+
/**
|
|
761
|
+
* Bengali (Bangla) light stemmer — strips common verbal and nominal suffixes.
|
|
762
|
+
* Bengali is an Indo-Aryan language with moderate morphological complexity.
|
|
763
|
+
* Only strips when result is still ≥ 2 chars.
|
|
764
|
+
*
|
|
765
|
+
* Covers:
|
|
766
|
+
* করছে/করেছে → কর (progressive/perfect aspect)
|
|
767
|
+
* সার্ভারগুলো → সার্ভার (-গুলো plural)
|
|
768
|
+
* সমস্যাটি → সমস্যা (-টি singular marker)
|
|
769
|
+
*/
|
|
770
|
+
function bengaliLightStem(word: string): string {
|
|
771
|
+
let w = word;
|
|
772
|
+
// Plural / collective markers (longest first)
|
|
773
|
+
if (w.endsWith('\u0997\u09c1\u09b2\u09cb') && w.length > 5) return w.slice(0, -4); // গুলো (-gulo plural)
|
|
774
|
+
if (w.endsWith('\u0997\u09c1\u09b2\u09bf') && w.length > 5) return w.slice(0, -4); // গুলি (-guli plural)
|
|
775
|
+
if (w.endsWith('\u09a6\u09c7\u09b0') && w.length > 4) return w.slice(0, -3); // দের (genitive plural)
|
|
776
|
+
if (w.endsWith('\u09a6\u09bf\u0997\u09c7') && w.length > 5) return w.slice(0, -4); // দিগে
|
|
777
|
+
// Definiteness / case markers
|
|
778
|
+
if (w.endsWith('\u099f\u09bf') && w.length > 3) return w.slice(0, -2); // টি (singular definite)
|
|
779
|
+
if (w.endsWith('\u099f\u09be') && w.length > 3) return w.slice(0, -2); // টা (singular definite)
|
|
780
|
+
if (w.endsWith('\u0996\u09be\u09a8\u09be') && w.length > 5) return w.slice(0, -4); // খানা
|
|
781
|
+
// Verbal suffixes
|
|
782
|
+
if (w.endsWith('\u099b\u09c7') && w.length > 3) return w.slice(0, -2); // ছে (progressive)
|
|
783
|
+
if (w.endsWith('\u099b\u09bf\u09b2') && w.length > 4) return w.slice(0, -3); // ছিল (past progressive)
|
|
784
|
+
if (w.endsWith('\u09af\u09be\u09ac\u09c7') && w.length > 5) return w.slice(0, -4); // যাবে (future)
|
|
785
|
+
if (w.endsWith('\u0995\u09b0\u09be') && w.length > 4) return w.slice(0, -3); // করা (infinitive)
|
|
786
|
+
if (w.endsWith('\u0995\u09b0\u09c7') && w.length > 4) return w.slice(0, -3); // করে (present)
|
|
787
|
+
if (w.endsWith('\u09b9\u09df') && w.length > 3) return w.slice(0, -2); // হয় (is/becomes)
|
|
788
|
+
if (w.endsWith('\u09b9\u09ac\u09c7') && w.length > 4) return w.slice(0, -3); // হবে (will be)
|
|
789
|
+
return w;
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
/**
|
|
793
|
+
* Pre-process text BEFORE tokenization to split code identifiers.
|
|
794
|
+
* Handles camelCase, PascalCase, snake_case, and kebab-case.
|
|
795
|
+
*
|
|
796
|
+
* Examples:
|
|
797
|
+
* deployServer → deploy Server (then lowercased → deploy server)
|
|
798
|
+
* HTMLParser → HTML Parser
|
|
799
|
+
* my_api_key → my api key
|
|
800
|
+
* get-user-by-id → get user by id (hyphens preserved in regex, spaces also fine)
|
|
801
|
+
* BackendServiceImpl → Backend Service Impl
|
|
802
|
+
*/
|
|
803
|
+
function preprocessText(text: string): string {
|
|
804
|
+
return text
|
|
805
|
+
// camelCase boundary: lowercase/digit → uppercase
|
|
806
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
807
|
+
// Consecutive uppercase → uppercase+lowercase boundary: HTMLParser → HTML Parser
|
|
808
|
+
.replace(/([A-Z]{2,})([A-Z][a-z])/g, '$1 $2')
|
|
809
|
+
// snake_case: replace underscores with spaces
|
|
810
|
+
.replace(/_/g, ' ');
|
|
811
|
+
}
|
|
812
|
+
|
|
731
813
|
|
|
732
814
|
// Maps tokens (English, Japanese katakana, Chinese, Korean, Arabic, Hebrew)
|
|
733
815
|
// to their equivalents in other languages. Used to expand tokens at search time
|
|
@@ -735,14 +817,72 @@ function turkishLightStem(word: string): string {
|
|
|
735
817
|
//
|
|
736
818
|
// Keys must be lowercase. CJK values will be bigram-expanded at use time.
|
|
737
819
|
const CROSS_LINGUAL_MAP = new Map<string, string[]>([
|
|
820
|
+
|
|
821
|
+
// ── Tech / Ecosystem synonyms (language-agnostic) ─────────────────────────
|
|
822
|
+
['javascript', ['js', 'node', 'nodejs', 'ecmascript']],
|
|
823
|
+
['js', ['javascript', 'node', 'nodejs']],
|
|
824
|
+
['typescript', ['ts', 'javascript', 'js', 'tsc']],
|
|
825
|
+
['ts', ['typescript', 'javascript', 'js']],
|
|
826
|
+
['python', ['py', 'pip', 'django', 'flask', 'fastapi']],
|
|
827
|
+
['py', ['python', 'pip']],
|
|
828
|
+
['golang', ['go', 'gopher', 'goroutine']],
|
|
829
|
+
['go', ['golang', 'goroutine']],
|
|
830
|
+
['rust', ['cargo', 'rustlang', 'crates']],
|
|
831
|
+
['cargo', ['rust', 'rustlang']],
|
|
832
|
+
['java', ['jvm', 'maven', 'gradle', 'spring']],
|
|
833
|
+
['kotlin', ['jvm', 'android', 'coroutine']],
|
|
834
|
+
['dotnet', ['csharp', 'asp', 'nuget', 'aspnet']],
|
|
835
|
+
['postgresql', ['postgres', 'pg', 'psql']],
|
|
836
|
+
['postgres', ['postgresql', 'pg', 'psql']],
|
|
837
|
+
['mysql', ['mariadb', 'sql']],
|
|
838
|
+
['mongodb', ['mongo', 'nosql', 'bson', 'mongoose']],
|
|
839
|
+
['mongo', ['mongodb', 'nosql', 'bson']],
|
|
840
|
+
['elasticsearch', ['elastic', 'opensearch', 'lucene', 'kibana']],
|
|
841
|
+
['elastic', ['elasticsearch', 'opensearch']],
|
|
842
|
+
['kubernetes', ['k8s', 'kube', 'kubectl', 'helm', 'k3s']],
|
|
843
|
+
['k8s', ['kubernetes', 'kube', 'kubectl', 'helm']],
|
|
844
|
+
['helm', ['kubernetes', 'k8s', 'chart']],
|
|
845
|
+
['terraform', ['tf', 'hcl', 'iac', 'opentofu']],
|
|
846
|
+
['ansible', ['playbook', 'automation']],
|
|
847
|
+
['grafana', ['dashboard', 'visualization', 'metrics']],
|
|
848
|
+
['prometheus', ['metrics', 'alert', 'scrape', 'grafana']],
|
|
849
|
+
['nginx', ['proxy', 'webserver', 'ingress']],
|
|
850
|
+
['traefik', ['proxy', 'ingress', 'router']],
|
|
851
|
+
['rabbitmq', ['amqp', 'queue', 'broker', 'messaging']],
|
|
852
|
+
['kafka', ['messaging', 'stream', 'broker']],
|
|
853
|
+
['grpc', ['protobuf', 'proto', 'rpc']],
|
|
854
|
+
['graphql', ['gql', 'resolver', 'schema']],
|
|
855
|
+
['react', ['jsx', 'hooks', 'component', 'redux']],
|
|
856
|
+
['nextjs', ['next', 'react', 'ssr', 'vercel']],
|
|
857
|
+
['vue', ['vuejs', 'vite', 'nuxt']],
|
|
858
|
+
['angular', ['ng', 'rxjs', 'typescript']],
|
|
859
|
+
['aws', ['amazon', 'ec2', 's3', 'lambda', 'cloudwatch', 'ecs', 'eks']],
|
|
860
|
+
['gcp', ['google cloud', 'gke', 'bigquery']],
|
|
861
|
+
['azure', ['microsoft cloud', 'aks', 'devops']],
|
|
862
|
+
['s3', ['bucket', 'object storage', 'aws']],
|
|
863
|
+
['lambda', ['serverless', 'function', 'faas', 'aws']],
|
|
864
|
+
['jwt', ['token', 'bearer', 'oauth', 'auth']],
|
|
865
|
+
['oauth', ['oidc', 'auth', 'token', 'sso']],
|
|
866
|
+
['webhook', ['callback', 'event', 'trigger']],
|
|
867
|
+
['cron', ['schedule', 'job', 'timer']],
|
|
868
|
+
['yaml', ['yml', 'config', 'manifest']],
|
|
869
|
+
['dotenv', ['env', 'environment', 'envfile']],
|
|
870
|
+
['github', ['git', 'actions', 'repo', 'ci']],
|
|
871
|
+
['gitlab', ['git', 'pipeline', 'runner']],
|
|
872
|
+
['wireguard', ['vpn', 'tunnel', 'wg']],
|
|
873
|
+
['prisma', ['orm', 'database', 'schema', 'migration']],
|
|
874
|
+
['npm', ['node', 'package', 'registry', 'yarn', 'pnpm']],
|
|
875
|
+
['yarn', ['npm', 'package', 'node']],
|
|
876
|
+
['pip', ['python', 'package', 'pypi']],
|
|
877
|
+
['brew', ['homebrew', 'macos', 'package']],
|
|
738
878
|
// English → all
|
|
739
|
-
['deploy', ['デプロイ', '部署', '배포', 'deployment', 'deploying', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellen', 'bereitstellung', 'déployer', 'déploiement', 'desplegar', 'despliegue', 'distribuire', 'distribuzione', 'implantar', 'implantação', 'развёртывание', 'развертывание', 'dağıtım', 'dağıtmak']],
|
|
740
|
-
['deployment', ['デプロイ', '部署', '배포', 'deploy', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellung', 'déploiement', 'despliegue', 'distribuzione', 'implantação', 'развёртывание', 'dağıtım']],
|
|
879
|
+
['deploy', ['デプロイ', '部署', '배포', 'deployment', 'deploying', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellen', 'bereitstellung', 'déployer', 'déploiement', 'desplegar', 'despliegue', 'distribuire', 'distribuzione', 'implantar', 'implantação', 'развёртывание', 'развертывание', 'dağıtım', 'dağıtmak', 'triển khai']],
|
|
880
|
+
['deployment', ['デプロイ', '部署', '배포', 'deploy', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellung', 'déploiement', 'despliegue', 'distribuzione', 'implantação', 'развёртывание', 'dağıtım', 'triển khai']],
|
|
741
881
|
['container', ['コンテナ', '容器', '컨테이너', 'docker']],
|
|
742
|
-
['server', ['サーバー', 'サーバ', '服务器', '서버', 'سيرفر', 'שרת', 'سرور', 'सर्वर', 'сервер', 'sunucu']],
|
|
882
|
+
['server', ['サーバー', 'サーバ', '服务器', '서버', 'سيرفر', 'שרת', 'سرور', 'सर्वर', 'сервер', 'sunucu', 'máy chủ']],
|
|
743
883
|
['database', ['データベース', '数据库', '데이터베이스', 'db', 'قاعدة البيانات', 'datenbank', 'پایگاه داده', 'डेटाबेس', 'base de données', 'base de datos', 'banco de dados']],
|
|
744
|
-
['cache', ['キャッシュ', '缓存', '캐시', 'كاش', 'מטמון', 'کش', 'कैश', 'кэш', 'önbellek']],
|
|
745
|
-
['error', ['エラー', '错误', '오류', 'خطأ', 'שגיאה', 'خطا', 'त्रुटि', 'गलती', 'exception', 'err', 'fehler', 'erreur', 'fallo', 'errore', 'erro', 'ошибка', 'hata', 'hatalı']],
|
|
884
|
+
['cache', ['キャッシュ', '缓存', '캐시', 'كاش', 'מטמון', 'کش', 'कैश', 'кэш', 'önbellek', 'bộ đệm']],
|
|
885
|
+
['error', ['エラー', '错误', '오류', 'خطأ', 'שגיאה', 'خطا', 'त्रुटि', 'गलती', 'exception', 'err', 'fehler', 'erreur', 'fallo', 'errore', 'erro', 'ошибка', 'hata', 'hatalı', 'lỗi']],
|
|
746
886
|
['bug', ['バグ', '缺陷', '버그', 'issue', 'defect', 'خلل', 'اشکال', 'बग']],
|
|
747
887
|
['fix', ['修正', '修复', '수정', 'bugfix', 'patch', 'hotfix', 'إصلاح', 'תיקון', 'رفع', 'सुधार', 'beheben', 'behoben', 'réparer', 'correction', 'arreglar', 'corrección', 'correggere', 'corrigir', 'исправить', 'исправление', 'düzeltmek', 'düzeltme']],
|
|
748
888
|
['build', ['ビルド', '构建', '빌드', 'بناء', 'בנייה', 'ساخت', 'निर्माण', 'bauen', 'construire', 'construir', 'costruire', 'сборка', 'собрать', 'derleme', 'derlemek']],
|
|
@@ -1012,6 +1152,7 @@ const CROSS_LINGUAL_MAP = new Map<string, string[]>([
|
|
|
1012
1152
|
|
|
1013
1153
|
// ── Hindi (Devanagari) → English ──────────────────────────────────────────
|
|
1014
1154
|
['तैनाती', ['deploy', 'deployment']],
|
|
1155
|
+
['तैना', ['deploy', 'deployment']], // stemmed form
|
|
1015
1156
|
['त्रुटि', ['error', 'err']],
|
|
1016
1157
|
['गलती', ['error', 'bug', 'issue']],
|
|
1017
1158
|
['सुधार', ['fix', 'bugfix']],
|
|
@@ -1269,6 +1410,8 @@ const CROSS_LINGUAL_MAP = new Map<string, string[]>([
|
|
|
1269
1410
|
['соединение', ['connection', 'conn']],
|
|
1270
1411
|
['развёртывание', ['deploy', 'deployment']],
|
|
1271
1412
|
['развертывание', ['deploy', 'deployment']],
|
|
1413
|
+
['развёрт', ['deploy', 'deployment']], // stemmed form
|
|
1414
|
+
['разверт', ['deploy', 'deployment']], // stemmed form
|
|
1272
1415
|
['контейнер', ['container', 'docker']],
|
|
1273
1416
|
['образ', ['image']],
|
|
1274
1417
|
['кластер', ['cluster']],
|
|
@@ -1326,6 +1469,39 @@ const CROSS_LINGUAL_MAP = new Map<string, string[]>([
|
|
|
1326
1469
|
['şifre', ['password']],
|
|
1327
1470
|
['izleme', ['monitor', 'monitoring']],
|
|
1328
1471
|
['göç', ['migrate', 'migration']],
|
|
1472
|
+
|
|
1473
|
+
// ── Vietnamese (Latin + Latin Extended Additional U+1EA0–U+1EF9) → English ─
|
|
1474
|
+
['triển khai', ['deploy', 'deployment']],
|
|
1475
|
+
['triển', ['deploy', 'deployment']], // split token form
|
|
1476
|
+
['lỗi', ['error', 'err']],
|
|
1477
|
+
['sửa lỗi', ['fix', 'bugfix', 'debug']],
|
|
1478
|
+
['xây dựng', ['build']],
|
|
1479
|
+
['kiểm tra', ['test']],
|
|
1480
|
+
['xác thực', ['auth', 'authentication']],
|
|
1481
|
+
['cấu hình', ['config', 'configuration']],
|
|
1482
|
+
['cài đặt', ['install']],
|
|
1483
|
+
['cập nhật', ['update', 'upgrade']],
|
|
1484
|
+
['nhật ký', ['log']],
|
|
1485
|
+
['máy chủ', ['server']],
|
|
1486
|
+
['cơ sở dữ liệu', ['database', 'db']],
|
|
1487
|
+
['mạng', ['network']],
|
|
1488
|
+
['bộ nhớ', ['memory', 'ram']],
|
|
1489
|
+
['bộ đệm', ['cache']],
|
|
1490
|
+
['gỡ lỗi', ['debug', 'debugging']],
|
|
1491
|
+
['sự cố', ['crash', 'failure', 'error']],
|
|
1492
|
+
['khởi động lại', ['restart', 'reboot']],
|
|
1493
|
+
['quyền', ['permission', 'access']],
|
|
1494
|
+
['sao lưu', ['backup']],
|
|
1495
|
+
['hiệu suất', ['performance']],
|
|
1496
|
+
['vấn đề', ['problem', 'issue', 'error']],
|
|
1497
|
+
['dịch vụ', ['service']],
|
|
1498
|
+
['kết nối', ['connection', 'conn']],
|
|
1499
|
+
['vùng chứa', ['container', 'docker']],
|
|
1500
|
+
['nút', ['node']],
|
|
1501
|
+
['khóa', ['key', 'token', 'secret']],
|
|
1502
|
+
['mật khẩu', ['password']],
|
|
1503
|
+
['giám sát', ['monitor', 'monitoring']],
|
|
1504
|
+
['di chuyển', ['migrate', 'migration']],
|
|
1329
1505
|
]);
|
|
1330
1506
|
|
|
1331
1507
|
/**
|
|
@@ -1352,7 +1528,8 @@ function expandCrossLingual(token: string): string[] {
|
|
|
1352
1528
|
*/
|
|
1353
1529
|
function tokenize(text: string): string[] {
|
|
1354
1530
|
const tokens: string[] = [];
|
|
1355
|
-
|
|
1531
|
+
// Pre-process: split camelCase/PascalCase identifiers and snake_case before lowercasing
|
|
1532
|
+
const lower = preprocessText(text).toLowerCase();
|
|
1356
1533
|
|
|
1357
1534
|
for (const [seg] of lower.matchAll(SEGMENT_RE)) {
|
|
1358
1535
|
if (CJK_RE.test(seg)) {
|
|
@@ -1399,15 +1576,22 @@ function tokenize(text: string): string[] {
|
|
|
1399
1576
|
}
|
|
1400
1577
|
} else if (DEVANAGARI_RE.test(seg)) {
|
|
1401
1578
|
// Devanagari segment (Hindi / Marathi) — naturally space-separated
|
|
1402
|
-
// Words don't need bigram expansion; stems reduce inflected forms.
|
|
1403
1579
|
const words = seg.trim().split(/[\s\u200c\u200d]+/); // handle zero-width joiners
|
|
1404
1580
|
for (const w of words) {
|
|
1405
|
-
// Keep Devanagari chars + digits; discard pure Latin portions (e.g. embedded numbers)
|
|
1406
1581
|
const deva = w.replace(/[^\u0900-\u097f0-9]/g, '');
|
|
1407
1582
|
if (deva.length <= 1 || STOPWORDS.has(deva)) continue;
|
|
1408
1583
|
const stemmed = hindiLightStem(deva);
|
|
1409
1584
|
if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) tokens.push(stemmed);
|
|
1410
1585
|
}
|
|
1586
|
+
} else if (BENGALI_RE.test(seg)) {
|
|
1587
|
+
// Bengali (Bangla) segment — space-separated words, Bengali script
|
|
1588
|
+
const words = seg.trim().split(/[\s\u200c\u200d]+/);
|
|
1589
|
+
for (const w of words) {
|
|
1590
|
+
const bn = w.replace(/[^\u0980-\u09ff0-9]/g, '');
|
|
1591
|
+
if (bn.length <= 1 || STOPWORDS.has(bn)) continue;
|
|
1592
|
+
const stemmed = bengaliLightStem(bn);
|
|
1593
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) tokens.push(stemmed);
|
|
1594
|
+
}
|
|
1411
1595
|
} else if (CYRILLIC_RE.test(seg)) {
|
|
1412
1596
|
// Cyrillic segment (Russian, Bulgarian, Ukrainian, Serbian …)
|
|
1413
1597
|
const words = seg.trim().split(/\s+/);
|
|
@@ -1418,9 +1602,10 @@ function tokenize(text: string): string[] {
|
|
|
1418
1602
|
if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) tokens.push(stemmed);
|
|
1419
1603
|
}
|
|
1420
1604
|
} else {
|
|
1421
|
-
// Latin / other
|
|
1605
|
+
// Latin / other — includes Turkish (ğ/ş/ı), Polish (ą/ę/ł/ń/ó/ś/ź/ż/ć),
|
|
1606
|
+
// Czech (č/š/ž/ě/ů/ř), Hungarian (ő/ű), Romanian (ș/ț), Vietnamese (ắặầổợụừ…)
|
|
1422
1607
|
const words = seg
|
|
1423
|
-
.replace(/[^a-
|
|
1608
|
+
.replace(/[^a-záàâãäåæçéèêëíìîïñóòôõöúùûüýÿßœğşıİąćęłńśźżčšžěůřőűșț\u1ea0-\u1ef90-9:\-./]/g, ' ')
|
|
1424
1609
|
.split(/\s+/)
|
|
1425
1610
|
.filter(w => w.length > 1 && !STOPWORDS.has(w));
|
|
1426
1611
|
// Turkish: apply light stemmer to words with Turkish-specific chars
|
|
@@ -1467,6 +1652,10 @@ function tokenize(text: string): string[] {
|
|
|
1467
1652
|
// Devanagari synonym (Hindi)
|
|
1468
1653
|
const stemmed = hindiLightStem(syn);
|
|
1469
1654
|
if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) expansions.push(stemmed);
|
|
1655
|
+
} else if (BENGALI_RE.test(syn)) {
|
|
1656
|
+
// Bengali synonym
|
|
1657
|
+
const stemmed = bengaliLightStem(syn);
|
|
1658
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed)) expansions.push(stemmed);
|
|
1470
1659
|
} else if (CYRILLIC_RE.test(syn)) {
|
|
1471
1660
|
// Cyrillic synonym (Russian)
|
|
1472
1661
|
const cyr = syn.replace(/[^\u0400-\u04ff]/g, '');
|
|
@@ -1549,12 +1738,31 @@ const BM25_DELTA = 1.0; // BM25+ lower-bound guarantee (0 = classic BM25)
|
|
|
1549
1738
|
/** Recency boost: half-life in days. Entry from 7 days ago gets 0.5× boost. */
|
|
1550
1739
|
const RECENCY_HALF_LIFE_DAYS = 7;
|
|
1551
1740
|
|
|
1741
|
+
/**
|
|
1742
|
+
* Zero-results query log — in-memory FIFO ring (max 500 entries).
|
|
1743
|
+
* Exposed via /health for observability. Never persisted.
|
|
1744
|
+
*/
|
|
1745
|
+
interface ZeroResultEntry { query: string; ts: number; }
|
|
1746
|
+
const ZERO_RESULTS_LOG: ZeroResultEntry[] = [];
|
|
1747
|
+
const ZERO_RESULTS_MAX = 500;
|
|
1748
|
+
let zeroResultsTotal = 0;
|
|
1749
|
+
|
|
1750
|
+
function logZeroResult(query: string): void {
|
|
1751
|
+
zeroResultsTotal++;
|
|
1752
|
+
if (ZERO_RESULTS_LOG.length >= ZERO_RESULTS_MAX) ZERO_RESULTS_LOG.shift();
|
|
1753
|
+
ZERO_RESULTS_LOG.push({ query, ts: Date.now() });
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
/** Global index vocabulary — rebuilt on each keywordSearch call from the doc set. */
|
|
1757
|
+
let _indexVocab: Set<string> = new Set();
|
|
1758
|
+
|
|
1552
1759
|
interface DocEntry {
|
|
1553
1760
|
key: string;
|
|
1554
1761
|
content: string;
|
|
1555
1762
|
tokens: string[];
|
|
1556
1763
|
tokenFreq: Map<string, number>; // term → count in this doc
|
|
1557
1764
|
bigrams: Set<string>; // "term1|term2" adjacency pairs
|
|
1765
|
+
keyTokens: Set<string>; // tokens from key/title only (for title boost)
|
|
1558
1766
|
timestamp?: number; // epoch ms, extracted from content if present
|
|
1559
1767
|
}
|
|
1560
1768
|
|
|
@@ -1674,7 +1882,9 @@ async function keywordSearch(
|
|
|
1674
1882
|
}
|
|
1675
1883
|
|
|
1676
1884
|
const timestamp = extractTimestamp(content);
|
|
1677
|
-
|
|
1885
|
+
// Key tokens for title-boost: terms appearing in the Redis key get extra weight
|
|
1886
|
+
const keyTokens = new Set(tokenize(allKeys[i]));
|
|
1887
|
+
docs.push({ key: allKeys[i], content, tokens, tokenFreq, bigrams, keyTokens, timestamp });
|
|
1678
1888
|
totalTokens += tokens.length;
|
|
1679
1889
|
}
|
|
1680
1890
|
|
|
@@ -1713,12 +1923,31 @@ async function keywordSearch(
|
|
|
1713
1923
|
}
|
|
1714
1924
|
|
|
1715
1925
|
/**
|
|
1716
|
-
* Fuzzy match: tries exact → substring → Levenshtein ≤ 2.
|
|
1926
|
+
* Fuzzy match: tries exact → prefix → substring → Levenshtein ≤ 2.
|
|
1717
1927
|
* Returns [matchedTerm, weight] or null.
|
|
1928
|
+
*
|
|
1929
|
+
* Weights:
|
|
1930
|
+
* 1.0 — exact match
|
|
1931
|
+
* 0.85 — doc token starts with query (e.g. "dockerf" → "dockerfile")
|
|
1932
|
+
* 0.75 — query starts with doc token (e.g. "kubernetes" → "kube")
|
|
1933
|
+
* 0.6 — substring (either direction)
|
|
1934
|
+
* 0.4 — Levenshtein ≤ 2 (typo tolerance)
|
|
1718
1935
|
*/
|
|
1719
1936
|
function fuzzyMatch(qt: string, docTermSet: Set<string>): [string, number] | null {
|
|
1720
1937
|
// Exact
|
|
1721
1938
|
if (docTermSet.has(qt)) return [qt, 1.0];
|
|
1939
|
+
// Prefix: query is prefix of a doc token (user typed partial word)
|
|
1940
|
+
if (qt.length >= 4) {
|
|
1941
|
+
for (const dt of docTermSet) {
|
|
1942
|
+
if (dt.length > qt.length && dt.startsWith(qt)) return [dt, 0.85];
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
// Reverse-prefix: doc token is prefix of query (doc has abbreviated form)
|
|
1946
|
+
if (qt.length >= 4) {
|
|
1947
|
+
for (const dt of docTermSet) {
|
|
1948
|
+
if (dt.length >= 4 && dt.length < qt.length && qt.startsWith(dt)) return [dt, 0.75];
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1722
1951
|
// Substring (partial)
|
|
1723
1952
|
for (const dt of docTermSet) {
|
|
1724
1953
|
if (dt.length > 3 && qt.length > 3 && (dt.includes(qt) || qt.includes(dt))) {
|
|
@@ -1788,6 +2017,18 @@ async function keywordSearch(
|
|
|
1788
2017
|
score *= 2.0;
|
|
1789
2018
|
}
|
|
1790
2019
|
|
|
2020
|
+
// Key/title boost: if query terms appear in the Redis key (=title), 1.5× boost
|
|
2021
|
+
// Keys often encode the primary topic (e.g. "deploy:api:server"), so key hits are high-precision.
|
|
2022
|
+
if (score > 0) {
|
|
2023
|
+
let keyHits = 0;
|
|
2024
|
+
for (const qt of queryTokens) {
|
|
2025
|
+
if (doc.keyTokens.has(qt)) keyHits++;
|
|
2026
|
+
}
|
|
2027
|
+
if (keyHits > 0) {
|
|
2028
|
+
score *= 1 + 0.5 * (keyHits / queryTokens.length);
|
|
2029
|
+
}
|
|
2030
|
+
}
|
|
2031
|
+
|
|
1791
2032
|
// Recency boost
|
|
1792
2033
|
score *= recencyBoost(doc.timestamp);
|
|
1793
2034
|
|
|
@@ -1808,7 +2049,17 @@ async function keywordSearch(
|
|
|
1808
2049
|
|
|
1809
2050
|
// ── Step 4: Sort by score, return top-K ──
|
|
1810
2051
|
const sorted = [...allMatches.values()].sort((a, b) => b.score - a.score);
|
|
1811
|
-
|
|
2052
|
+
const topResults = sorted.slice(0, topK);
|
|
2053
|
+
|
|
2054
|
+
// ── Step 5: Zero-results logging + Did-You-Mean ───────────────────────────
|
|
2055
|
+
if (topResults.length === 0) {
|
|
2056
|
+
logZeroResult(query);
|
|
2057
|
+
// Rebuild index vocab for Did-You-Mean suggestions
|
|
2058
|
+
_indexVocab = new Set<string>();
|
|
2059
|
+
for (const doc of docs) for (const t of doc.tokens) _indexVocab.add(t);
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
return topResults;
|
|
1812
2063
|
}
|
|
1813
2064
|
|
|
1814
2065
|
// ── Exported for testing ──────────────────────────────────────────────────────
|
|
@@ -4445,10 +4696,31 @@ async function handleTool(name: string, args: Record<string, unknown>): Promise<
|
|
|
4445
4696
|
|
|
4446
4697
|
if (kwMatches.length === 0) {
|
|
4447
4698
|
lines.push(`⚠️ No matches found for: "${query}"`);
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4699
|
+
|
|
4700
|
+
// Did-You-Mean: find nearest token in index vocab
|
|
4701
|
+
const queryTokens = tokenize(query);
|
|
4702
|
+
const suggestions: string[] = [];
|
|
4703
|
+
if (_indexVocab.size > 0 && queryTokens.length > 0) {
|
|
4704
|
+
for (const qt of queryTokens.slice(0, 3)) {
|
|
4705
|
+
if (qt.length < 4) continue;
|
|
4706
|
+
let bestDist = 3;
|
|
4707
|
+
let bestTok = '';
|
|
4708
|
+
for (const v of _indexVocab) {
|
|
4709
|
+
if (v.length < 3 || Math.abs(v.length - qt.length) > 4) continue;
|
|
4710
|
+
const d = levenshtein(qt, v);
|
|
4711
|
+
if (d > 0 && d < bestDist) { bestDist = d; bestTok = v; }
|
|
4712
|
+
}
|
|
4713
|
+
if (bestTok) suggestions.push(`"${bestTok}" (instead of "${qt}")`);
|
|
4714
|
+
}
|
|
4715
|
+
}
|
|
4716
|
+
if (suggestions.length > 0) {
|
|
4717
|
+
lines.push(`💡 **Did you mean:** ${suggestions.join(', ')}?`);
|
|
4718
|
+
} else {
|
|
4719
|
+
lines.push(`\n💡 Tips:`);
|
|
4720
|
+
lines.push(` • Try different keywords`);
|
|
4721
|
+
lines.push(` • Use \`list_remembered\` to see available context`);
|
|
4722
|
+
lines.push(` • Use \`recall_best_solution("topic")\` for exact topic lookup`);
|
|
4723
|
+
}
|
|
4452
4724
|
}
|
|
4453
4725
|
|
|
4454
4726
|
return lines.join('\n');
|
|
@@ -7369,7 +7641,14 @@ if (httpPort) {
|
|
|
7369
7641
|
const url = new URL(req.url ?? '/', `http://localhost:${httpPort}`);
|
|
7370
7642
|
if (url.pathname === '/health') {
|
|
7371
7643
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
7372
|
-
res.end(JSON.stringify({
|
|
7644
|
+
res.end(JSON.stringify({
|
|
7645
|
+
status: 'ok',
|
|
7646
|
+
version: CURRENT_VERSION,
|
|
7647
|
+
zeroResults: {
|
|
7648
|
+
total: zeroResultsTotal,
|
|
7649
|
+
last10: ZERO_RESULTS_LOG.slice(-10).map(e => ({ query: e.query, ts: new Date(e.ts).toISOString() })),
|
|
7650
|
+
},
|
|
7651
|
+
}));
|
|
7373
7652
|
return;
|
|
7374
7653
|
}
|
|
7375
7654
|
if (url.pathname === '/mcp' || url.pathname === '/') {
|
package/src/search.test.ts
CHANGED
|
@@ -494,3 +494,116 @@ describe('tokenize — cross-lingual expansion', () => {
|
|
|
494
494
|
});
|
|
495
495
|
});
|
|
496
496
|
|
|
497
|
+
// ── Language-specific tokenizer + stemmer tests ───────────────────────────────
|
|
498
|
+
describe('multilingual tokenizer', () => {
|
|
499
|
+
// Russian
|
|
500
|
+
it('Russian: токенизирует слово и стемит суффикс', () => {
|
|
501
|
+
const tokens = tokenize('развёртывание сервера');
|
|
502
|
+
expect(tokens.some(t => t.startsWith('развёрт') || t === 'deploy' || t === 'deployment')).toBe(true);
|
|
503
|
+
});
|
|
504
|
+
it('Russian: развёртывание expands to "deploy"', () => {
|
|
505
|
+
const tokens = tokenize('развёртывание');
|
|
506
|
+
expect(tokens).toContain('deploy');
|
|
507
|
+
});
|
|
508
|
+
it('Russian: ошибка expands to "error"', () => {
|
|
509
|
+
const tokens = tokenize('ошибка');
|
|
510
|
+
expect(tokens).toContain('error');
|
|
511
|
+
});
|
|
512
|
+
|
|
513
|
+
// Turkish
|
|
514
|
+
it('Turkish: sunucu expands to "server"', () => {
|
|
515
|
+
const tokens = tokenize('sunucu');
|
|
516
|
+
expect(tokens).toContain('server');
|
|
517
|
+
});
|
|
518
|
+
it('Turkish: hata expands to "error"', () => {
|
|
519
|
+
const tokens = tokenize('hata');
|
|
520
|
+
expect(tokens).toContain('error');
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
// Bengali
|
|
524
|
+
it('Bengali: tokenizes words with Bengali script', () => {
|
|
525
|
+
const tokens = tokenize('সার্ভার ডেটাবেস');
|
|
526
|
+
expect(tokens.length).toBeGreaterThan(0);
|
|
527
|
+
});
|
|
528
|
+
it('Bengali: strips -গুলো plural suffix', () => {
|
|
529
|
+
// সার্ভারগুলো → সার্ভার
|
|
530
|
+
const tokens = tokenize('সার্ভারগুলো');
|
|
531
|
+
expect(tokens.some(t => t.startsWith('\u09b8\u09be\u09b0') || t.length > 0)).toBe(true);
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
// Vietnamese
|
|
535
|
+
it('Vietnamese: tokenizes tone-marked words', () => {
|
|
536
|
+
const tokens = tokenize('triển khai máy chủ');
|
|
537
|
+
expect(tokens.some(t => t === 'tri\u1ec3n' || t === 'khai' || t === 'm\u00e1y' || t === 'ch\u1ee7')).toBe(true);
|
|
538
|
+
});
|
|
539
|
+
it('Vietnamese: triển khai expands to "deploy"', () => {
|
|
540
|
+
const tokens = tokenize('triển khai');
|
|
541
|
+
expect(tokens).toContain('deploy');
|
|
542
|
+
});
|
|
543
|
+
it('Vietnamese: lỗi expands to "error"', () => {
|
|
544
|
+
const tokens = tokenize('lỗi');
|
|
545
|
+
expect(tokens).toContain('error');
|
|
546
|
+
});
|
|
547
|
+
it('Vietnamese: stopwords filtered out', () => {
|
|
548
|
+
const tokens = tokenize('không tôi và');
|
|
549
|
+
expect(tokens).not.toContain('t\u00f4i');
|
|
550
|
+
expect(tokens).not.toContain('v\u00e0');
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
// Hindi
|
|
554
|
+
it('Hindi: तैनाती expands to "deploy"', () => {
|
|
555
|
+
const tokens = tokenize('तैनाती');
|
|
556
|
+
expect(tokens).toContain('deploy');
|
|
557
|
+
});
|
|
558
|
+
it('Hindi: त्रुटि expands to "error"', () => {
|
|
559
|
+
const tokens = tokenize('त्रुटि');
|
|
560
|
+
expect(tokens).toContain('error');
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
// Farsi
|
|
564
|
+
it('Farsi: استقرار expands to "deploy"', () => {
|
|
565
|
+
const tokens = tokenize('استقرار');
|
|
566
|
+
expect(tokens).toContain('deploy');
|
|
567
|
+
});
|
|
568
|
+
|
|
569
|
+
// camelCase splitting
|
|
570
|
+
it('preprocessText splits camelCase', () => {
|
|
571
|
+
const tokens = tokenize('deployServer');
|
|
572
|
+
expect(tokens).toContain('deploy');
|
|
573
|
+
expect(tokens).toContain('server');
|
|
574
|
+
});
|
|
575
|
+
it('preprocessText splits snake_case', () => {
|
|
576
|
+
const tokens = tokenize('my_api_key');
|
|
577
|
+
expect(tokens).toContain('api');
|
|
578
|
+
expect(tokens).toContain('key');
|
|
579
|
+
});
|
|
580
|
+
it('preprocessText splits PascalCase HTMLParser', () => {
|
|
581
|
+
const tokens = tokenize('HTMLParser');
|
|
582
|
+
expect(tokens).toContain('html');
|
|
583
|
+
expect(tokens).toContain('parser');
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
// Tech synonyms
|
|
587
|
+
it('tech synonym: k8s expands to include kubernetes', () => {
|
|
588
|
+
const tokens = tokenize('k8s');
|
|
589
|
+
expect(tokens).toContain('kubernetes');
|
|
590
|
+
});
|
|
591
|
+
it('tech synonym: postgres expands to postgresql', () => {
|
|
592
|
+
const tokens = tokenize('postgres');
|
|
593
|
+
expect(tokens).toContain('postgresql');
|
|
594
|
+
});
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
// ── Prefix-Matching tests ─────────────────────────────────────────────────────
|
|
598
|
+
describe('keywordSearch prefix-matching', () => {
|
|
599
|
+
it('English "deploy" matches forward entries for 17 languages in cross-lingual map', () => {
|
|
600
|
+
const result = CROSS_LINGUAL_MAP.get('deploy');
|
|
601
|
+
expect(result).toBeDefined();
|
|
602
|
+
// Should include at least Vietnamese, Russian, Turkish entries
|
|
603
|
+
expect(result).toContain('triển khai');
|
|
604
|
+
expect(result).toContain('развёртывание');
|
|
605
|
+
expect(result).toContain('dağıtım');
|
|
606
|
+
});
|
|
607
|
+
});
|
|
608
|
+
|
|
609
|
+
|