@cachly-dev/mcp-server 0.5.60 → 0.5.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +309 -19
- package/dist/index.js.map +1 -1
- package/dist/search.test.js +103 -0
- package/dist/search.test.js.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +298 -19
- package/src/search.test.ts +113 -0
package/dist/index.js
CHANGED
|
@@ -47,7 +47,7 @@ import { Redis } from 'ioredis';
|
|
|
47
47
|
const API_URL = process.env.CACHLY_API_URL ?? 'https://api.cachly.dev';
|
|
48
48
|
const JWT = process.env.CACHLY_JWT ?? '';
|
|
49
49
|
const EMBED_MODEL = process.env.CACHLY_EMBED_MODEL ?? '';
|
|
50
|
-
const CURRENT_VERSION = '0.5.
|
|
50
|
+
const CURRENT_VERSION = '0.5.80';
|
|
51
51
|
/**
|
|
52
52
|
* ┌──────────────────────────────────────────────────────────────────────┐
|
|
53
53
|
* │ EMBEDDING PROVIDER — pluggable, client-side first │
|
|
@@ -437,6 +437,30 @@ const STOPWORDS = new Set([
|
|
|
437
437
|
'çok', 'en', 'her', 'hiç', 'bazı', 'tüm', 'bütün', 'hem', 'veya',
|
|
438
438
|
'bu', 'şu', 'ben', 'sen', 'biz', 'siz', 'onlar', 'benim', 'senin',
|
|
439
439
|
'nasıl', 'neden', 'niçin', 'nerede', 'ne', 'hangi', 'kaç',
|
|
440
|
+
// ── Polish — common prepositions, pronouns, particles ─────────────────────
|
|
441
|
+
'w', 'na', 'z', 'do', 'się', 'że', 'to', 'jest', 'i', 'a', 'nie',
|
|
442
|
+
'tak', 'jak', 'czy', 'po', 'o', 'ale', 'go', 'mu', 'jej', 'ich',
|
|
443
|
+
'tego', 'tej', 'te', 'ten', 'ta', 'są', 'był', 'była', 'było', 'byli',
|
|
444
|
+
'będzie', 'będą', 'ma', 'mam', 'masz', 'mają', 'ze', 'co', 'już',
|
|
445
|
+
'przez', 'przy', 'za', 'bez', 'nad', 'pod', 'przed', 'po', 'między',
|
|
446
|
+
'kiedy', 'gdzie', 'dlaczego', 'który', 'która', 'które', 'tylko', 'też',
|
|
447
|
+
// ── Czech — common words ──────────────────────────────────────────────────
|
|
448
|
+
'v', 'na', 'z', 'do', 'se', 'že', 'to', 'je', 'a', 'ne', 'pro',
|
|
449
|
+
'ale', 'jak', 'by', 'byl', 'být', 'jsem', 'jsou', 'má', 'mám',
|
|
450
|
+
'ten', 'ta', 'ty', 'tato', 'jeho', 'její', 'jejich', 'také', 'jen',
|
|
451
|
+
'kde', 'kdy', 'proč', 'který', 'která', 'které', 'co', 'při', 'bez',
|
|
452
|
+
// ── Bengali — common particles, pronouns, verbs ───────────────────────────
|
|
453
|
+
'এই', 'এটি', 'এটা', 'এর', 'তার', 'তারা', 'আমি', 'তুমি', 'সে', 'আমরা',
|
|
454
|
+
'এবং', 'কিন্তু', 'বা', 'না', 'হ্যাঁ', 'যে', 'কি', 'কে', 'কী', 'থেকে',
|
|
455
|
+
'দিয়ে', 'জন্য', 'সাথে', 'মধ্যে', 'উপর', 'নিচে', 'আছে', 'ছিল', 'হয়',
|
|
456
|
+
'করা', 'করে', 'করেছে', 'হবে', 'হয়েছে', 'একটি', 'একটা', 'অনেক', 'সব',
|
|
457
|
+
// ── Vietnamese — common particles, pronouns, auxiliary words ─────────────
|
|
458
|
+
'tôi', 'bạn', 'anh', 'chị', 'em', 'họ', 'chúng', 'ta', 'mình',
|
|
459
|
+
'là', 'có', 'không', 'và', 'với', 'của', 'cho', 'trong', 'về',
|
|
460
|
+
'từ', 'đến', 'để', 'khi', 'nếu', 'thì', 'mà', 'nhưng', 'vì',
|
|
461
|
+
'đây', 'đó', 'này', 'kia', 'rất', 'cũng', 'đã', 'sẽ', 'đang',
|
|
462
|
+
'được', 'bị', 'những', 'các', 'một', 'hai', 'ba', 'nhiều', 'ít',
|
|
463
|
+
'nào', 'ai', 'gì', 'đâu', 'sao', 'bao', 'giờ', 'lúc', 'sau',
|
|
440
464
|
]);
|
|
441
465
|
// CJK Unicode ranges used for bigram extraction
|
|
442
466
|
const CJK_RE = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]/;
|
|
@@ -454,6 +478,11 @@ const DEVANAGARI_RE = /[\u0900-\u097f]/;
|
|
|
454
478
|
const CYRILLIC_RE = /[\u0400-\u04ff]/;
|
|
455
479
|
// Turkish uses Latin alphabet but has unique chars (ğ, ş, ı, ö, ü, ç) — detected by these
|
|
456
480
|
const TURKISH_CHAR_RE = /[\u011f\u015f\u0131\u0130\u00e7]/; // ğ ş ı İ ç
|
|
481
|
+
// Bengali (Bangla) — U+0980–U+09FF
|
|
482
|
+
const BENGALI_RE = /[\u0980-\u09ff]/;
|
|
483
|
+
// Vietnamese uses Latin + Latin Extended Additional (U+1EA0–U+1EF9) for tone marks
|
|
484
|
+
// e.g. ắ ặ ầ ổ ợ ụ ừ — detected by chars exclusive to Vietnamese diacritics
|
|
485
|
+
const VIETNAMESE_CHAR_RE = /[\u1ea0-\u1ef9]/;
|
|
457
486
|
// ── Katakana → Romaji conversion table (Hepburn system) ──────────────────────
|
|
458
487
|
// Digraphs must be listed before single chars so they match first.
|
|
459
488
|
const KATA_ROMAJI_TABLE = [
|
|
@@ -780,20 +809,142 @@ function turkishLightStem(word) {
|
|
|
780
809
|
return w.slice(0, -1);
|
|
781
810
|
return w;
|
|
782
811
|
}
|
|
812
|
+
/**
|
|
813
|
+
* Bengali (Bangla) light stemmer — strips common verbal and nominal suffixes.
|
|
814
|
+
* Bengali is an Indo-Aryan language with moderate morphological complexity.
|
|
815
|
+
* Only strips when result is still ≥ 2 chars.
|
|
816
|
+
*
|
|
817
|
+
* Covers:
|
|
818
|
+
* করছে/করেছে → কর (progressive/perfect aspect)
|
|
819
|
+
* সার্ভারগুলো → সার্ভার (-গুলো plural)
|
|
820
|
+
* সমস্যাটি → সমস্যা (-টি singular marker)
|
|
821
|
+
*/
|
|
822
|
+
function bengaliLightStem(word) {
|
|
823
|
+
let w = word;
|
|
824
|
+
// Plural / collective markers (longest first)
|
|
825
|
+
if (w.endsWith('\u0997\u09c1\u09b2\u09cb') && w.length > 5)
|
|
826
|
+
return w.slice(0, -4); // গুলো (-gulo plural)
|
|
827
|
+
if (w.endsWith('\u0997\u09c1\u09b2\u09bf') && w.length > 5)
|
|
828
|
+
return w.slice(0, -4); // গুলি (-guli plural)
|
|
829
|
+
if (w.endsWith('\u09a6\u09c7\u09b0') && w.length > 4)
|
|
830
|
+
return w.slice(0, -3); // দের (genitive plural)
|
|
831
|
+
if (w.endsWith('\u09a6\u09bf\u0997\u09c7') && w.length > 5)
|
|
832
|
+
return w.slice(0, -4); // দিগে
|
|
833
|
+
// Definiteness / case markers
|
|
834
|
+
if (w.endsWith('\u099f\u09bf') && w.length > 3)
|
|
835
|
+
return w.slice(0, -2); // টি (singular definite)
|
|
836
|
+
if (w.endsWith('\u099f\u09be') && w.length > 3)
|
|
837
|
+
return w.slice(0, -2); // টা (singular definite)
|
|
838
|
+
if (w.endsWith('\u0996\u09be\u09a8\u09be') && w.length > 5)
|
|
839
|
+
return w.slice(0, -4); // খানা
|
|
840
|
+
// Verbal suffixes
|
|
841
|
+
if (w.endsWith('\u099b\u09c7') && w.length > 3)
|
|
842
|
+
return w.slice(0, -2); // ছে (progressive)
|
|
843
|
+
if (w.endsWith('\u099b\u09bf\u09b2') && w.length > 4)
|
|
844
|
+
return w.slice(0, -3); // ছিল (past progressive)
|
|
845
|
+
if (w.endsWith('\u09af\u09be\u09ac\u09c7') && w.length > 5)
|
|
846
|
+
return w.slice(0, -4); // যাবে (future)
|
|
847
|
+
if (w.endsWith('\u0995\u09b0\u09be') && w.length > 4)
|
|
848
|
+
return w.slice(0, -3); // করা (infinitive)
|
|
849
|
+
if (w.endsWith('\u0995\u09b0\u09c7') && w.length > 4)
|
|
850
|
+
return w.slice(0, -3); // করে (present)
|
|
851
|
+
if (w.endsWith('\u09b9\u09df') && w.length > 3)
|
|
852
|
+
return w.slice(0, -2); // হয় (is/becomes)
|
|
853
|
+
if (w.endsWith('\u09b9\u09ac\u09c7') && w.length > 4)
|
|
854
|
+
return w.slice(0, -3); // হবে (will be)
|
|
855
|
+
return w;
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Pre-process text BEFORE tokenization to split code identifiers.
|
|
859
|
+
* Handles camelCase, PascalCase, snake_case, and kebab-case.
|
|
860
|
+
*
|
|
861
|
+
* Examples:
|
|
862
|
+
* deployServer → deploy Server (then lowercased → deploy server)
|
|
863
|
+
* HTMLParser → HTML Parser
|
|
864
|
+
* my_api_key → my api key
|
|
865
|
+
* get-user-by-id → get user by id (hyphens preserved in regex, spaces also fine)
|
|
866
|
+
* BackendServiceImpl → Backend Service Impl
|
|
867
|
+
*/
|
|
868
|
+
function preprocessText(text) {
|
|
869
|
+
return text
|
|
870
|
+
// camelCase boundary: lowercase/digit → uppercase
|
|
871
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
872
|
+
// Consecutive uppercase → uppercase+lowercase boundary: HTMLParser → HTML Parser
|
|
873
|
+
.replace(/([A-Z]{2,})([A-Z][a-z])/g, '$1 $2')
|
|
874
|
+
// snake_case: replace underscores with spaces
|
|
875
|
+
.replace(/_/g, ' ');
|
|
876
|
+
}
|
|
783
877
|
// Maps tokens (English, Japanese katakana, Chinese, Korean, Arabic, Hebrew)
|
|
784
878
|
// to their equivalents in other languages. Used to expand tokens at search time
|
|
785
879
|
// so "deploy" finds documents containing "デプロイ" / "部署" / "배포", and vice versa.
|
|
786
880
|
//
|
|
787
881
|
// Keys must be lowercase. CJK values will be bigram-expanded at use time.
|
|
788
882
|
const CROSS_LINGUAL_MAP = new Map([
|
|
883
|
+
// ── Tech / Ecosystem synonyms (language-agnostic) ─────────────────────────
|
|
884
|
+
['javascript', ['js', 'node', 'nodejs', 'ecmascript']],
|
|
885
|
+
['js', ['javascript', 'node', 'nodejs']],
|
|
886
|
+
['typescript', ['ts', 'javascript', 'js', 'tsc']],
|
|
887
|
+
['ts', ['typescript', 'javascript', 'js']],
|
|
888
|
+
['python', ['py', 'pip', 'django', 'flask', 'fastapi']],
|
|
889
|
+
['py', ['python', 'pip']],
|
|
890
|
+
['golang', ['go', 'gopher', 'goroutine']],
|
|
891
|
+
['go', ['golang', 'goroutine']],
|
|
892
|
+
['rust', ['cargo', 'rustlang', 'crates']],
|
|
893
|
+
['cargo', ['rust', 'rustlang']],
|
|
894
|
+
['java', ['jvm', 'maven', 'gradle', 'spring']],
|
|
895
|
+
['kotlin', ['jvm', 'android', 'coroutine']],
|
|
896
|
+
['dotnet', ['csharp', 'asp', 'nuget', 'aspnet']],
|
|
897
|
+
['postgresql', ['postgres', 'pg', 'psql']],
|
|
898
|
+
['postgres', ['postgresql', 'pg', 'psql']],
|
|
899
|
+
['mysql', ['mariadb', 'sql']],
|
|
900
|
+
['mongodb', ['mongo', 'nosql', 'bson', 'mongoose']],
|
|
901
|
+
['mongo', ['mongodb', 'nosql', 'bson']],
|
|
902
|
+
['elasticsearch', ['elastic', 'opensearch', 'lucene', 'kibana']],
|
|
903
|
+
['elastic', ['elasticsearch', 'opensearch']],
|
|
904
|
+
['kubernetes', ['k8s', 'kube', 'kubectl', 'helm', 'k3s']],
|
|
905
|
+
['k8s', ['kubernetes', 'kube', 'kubectl', 'helm']],
|
|
906
|
+
['helm', ['kubernetes', 'k8s', 'chart']],
|
|
907
|
+
['terraform', ['tf', 'hcl', 'iac', 'opentofu']],
|
|
908
|
+
['ansible', ['playbook', 'automation']],
|
|
909
|
+
['grafana', ['dashboard', 'visualization', 'metrics']],
|
|
910
|
+
['prometheus', ['metrics', 'alert', 'scrape', 'grafana']],
|
|
911
|
+
['nginx', ['proxy', 'webserver', 'ingress']],
|
|
912
|
+
['traefik', ['proxy', 'ingress', 'router']],
|
|
913
|
+
['rabbitmq', ['amqp', 'queue', 'broker', 'messaging']],
|
|
914
|
+
['kafka', ['messaging', 'stream', 'broker']],
|
|
915
|
+
['grpc', ['protobuf', 'proto', 'rpc']],
|
|
916
|
+
['graphql', ['gql', 'resolver', 'schema']],
|
|
917
|
+
['react', ['jsx', 'hooks', 'component', 'redux']],
|
|
918
|
+
['nextjs', ['next', 'react', 'ssr', 'vercel']],
|
|
919
|
+
['vue', ['vuejs', 'vite', 'nuxt']],
|
|
920
|
+
['angular', ['ng', 'rxjs', 'typescript']],
|
|
921
|
+
['aws', ['amazon', 'ec2', 's3', 'lambda', 'cloudwatch', 'ecs', 'eks']],
|
|
922
|
+
['gcp', ['google cloud', 'gke', 'bigquery']],
|
|
923
|
+
['azure', ['microsoft cloud', 'aks', 'devops']],
|
|
924
|
+
['s3', ['bucket', 'object storage', 'aws']],
|
|
925
|
+
['lambda', ['serverless', 'function', 'faas', 'aws']],
|
|
926
|
+
['jwt', ['token', 'bearer', 'oauth', 'auth']],
|
|
927
|
+
['oauth', ['oidc', 'auth', 'token', 'sso']],
|
|
928
|
+
['webhook', ['callback', 'event', 'trigger']],
|
|
929
|
+
['cron', ['schedule', 'job', 'timer']],
|
|
930
|
+
['yaml', ['yml', 'config', 'manifest']],
|
|
931
|
+
['dotenv', ['env', 'environment', 'envfile']],
|
|
932
|
+
['github', ['git', 'actions', 'repo', 'ci']],
|
|
933
|
+
['gitlab', ['git', 'pipeline', 'runner']],
|
|
934
|
+
['wireguard', ['vpn', 'tunnel', 'wg']],
|
|
935
|
+
['prisma', ['orm', 'database', 'schema', 'migration']],
|
|
936
|
+
['npm', ['node', 'package', 'registry', 'yarn', 'pnpm']],
|
|
937
|
+
['yarn', ['npm', 'package', 'node']],
|
|
938
|
+
['pip', ['python', 'package', 'pypi']],
|
|
939
|
+
['brew', ['homebrew', 'macos', 'package']],
|
|
789
940
|
// English → all
|
|
790
|
-
['deploy', ['デプロイ', '部署', '배포', 'deployment', 'deploying', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellen', 'bereitstellung', 'déployer', 'déploiement', 'desplegar', 'despliegue', 'distribuire', 'distribuzione', 'implantar', 'implantação', 'развёртывание', 'развертывание', 'dağıtım', 'dağıtmak']],
|
|
791
|
-
['deployment', ['デプロイ', '部署', '배포', 'deploy', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellung', 'déploiement', 'despliegue', 'distribuzione', 'implantação', 'развёртывание', 'dağıtım']],
|
|
941
|
+
['deploy', ['デプロイ', '部署', '배포', 'deployment', 'deploying', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellen', 'bereitstellung', 'déployer', 'déploiement', 'desplegar', 'despliegue', 'distribuire', 'distribuzione', 'implantar', 'implantação', 'развёртывание', 'развертывание', 'dağıtım', 'dağıtmak', 'triển khai']],
|
|
942
|
+
['deployment', ['デプロイ', '部署', '배포', 'deploy', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellung', 'déploiement', 'despliegue', 'distribuzione', 'implantação', 'развёртывание', 'dağıtım', 'triển khai']],
|
|
792
943
|
['container', ['コンテナ', '容器', '컨테이너', 'docker']],
|
|
793
|
-
['server', ['サーバー', 'サーバ', '服务器', '서버', 'سيرفر', 'שרת', 'سرور', 'सर्वर', 'сервер', 'sunucu']],
|
|
944
|
+
['server', ['サーバー', 'サーバ', '服务器', '서버', 'سيرفر', 'שרת', 'سرور', 'सर्वर', 'сервер', 'sunucu', 'máy chủ']],
|
|
794
945
|
['database', ['データベース', '数据库', '데이터베이스', 'db', 'قاعدة البيانات', 'datenbank', 'پایگاه داده', 'डेटाबेس', 'base de données', 'base de datos', 'banco de dados']],
|
|
795
|
-
['cache', ['キャッシュ', '缓存', '캐시', 'كاش', 'מטמון', 'کش', 'कैश', 'кэш', 'önbellek']],
|
|
796
|
-
['error', ['エラー', '错误', '오류', 'خطأ', 'שגיאה', 'خطا', 'त्रुटि', 'गलती', 'exception', 'err', 'fehler', 'erreur', 'fallo', 'errore', 'erro', 'ошибка', 'hata', 'hatalı']],
|
|
946
|
+
['cache', ['キャッシュ', '缓存', '캐시', 'كاش', 'מטמון', 'کش', 'कैश', 'кэш', 'önbellek', 'bộ đệm']],
|
|
947
|
+
['error', ['エラー', '错误', '오류', 'خطأ', 'שגיאה', 'خطا', 'त्रुटि', 'गलती', 'exception', 'err', 'fehler', 'erreur', 'fallo', 'errore', 'erro', 'ошибка', 'hata', 'hatalı', 'lỗi']],
|
|
797
948
|
['bug', ['バグ', '缺陷', '버그', 'issue', 'defect', 'خلل', 'اشکال', 'बग']],
|
|
798
949
|
['fix', ['修正', '修复', '수정', 'bugfix', 'patch', 'hotfix', 'إصلاح', 'תיקון', 'رفع', 'सुधार', 'beheben', 'behoben', 'réparer', 'correction', 'arreglar', 'corrección', 'correggere', 'corrigir', 'исправить', 'исправление', 'düzeltmek', 'düzeltme']],
|
|
799
950
|
['build', ['ビルド', '构建', '빌드', 'بناء', 'בנייה', 'ساخت', 'निर्माण', 'bauen', 'construire', 'construir', 'costruire', 'сборка', 'собрать', 'derleme', 'derlemek']],
|
|
@@ -1368,6 +1519,37 @@ const CROSS_LINGUAL_MAP = new Map([
|
|
|
1368
1519
|
['şifre', ['password']],
|
|
1369
1520
|
['izleme', ['monitor', 'monitoring']],
|
|
1370
1521
|
['göç', ['migrate', 'migration']],
|
|
1522
|
+
// ── Vietnamese (Latin + Latin Extended Additional U+1EA0–U+1EF9) → English ─
|
|
1523
|
+
['triển khai', ['deploy', 'deployment']],
|
|
1524
|
+
['lỗi', ['error', 'err']],
|
|
1525
|
+
['sửa lỗi', ['fix', 'bugfix', 'debug']],
|
|
1526
|
+
['xây dựng', ['build']],
|
|
1527
|
+
['kiểm tra', ['test']],
|
|
1528
|
+
['xác thực', ['auth', 'authentication']],
|
|
1529
|
+
['cấu hình', ['config', 'configuration']],
|
|
1530
|
+
['cài đặt', ['install']],
|
|
1531
|
+
['cập nhật', ['update', 'upgrade']],
|
|
1532
|
+
['nhật ký', ['log']],
|
|
1533
|
+
['máy chủ', ['server']],
|
|
1534
|
+
['cơ sở dữ liệu', ['database', 'db']],
|
|
1535
|
+
['mạng', ['network']],
|
|
1536
|
+
['bộ nhớ', ['memory', 'ram']],
|
|
1537
|
+
['bộ đệm', ['cache']],
|
|
1538
|
+
['gỡ lỗi', ['debug', 'debugging']],
|
|
1539
|
+
['sự cố', ['crash', 'failure', 'error']],
|
|
1540
|
+
['khởi động lại', ['restart', 'reboot']],
|
|
1541
|
+
['quyền', ['permission', 'access']],
|
|
1542
|
+
['sao lưu', ['backup']],
|
|
1543
|
+
['hiệu suất', ['performance']],
|
|
1544
|
+
['vấn đề', ['problem', 'issue', 'error']],
|
|
1545
|
+
['dịch vụ', ['service']],
|
|
1546
|
+
['kết nối', ['connection', 'conn']],
|
|
1547
|
+
['vùng chứa', ['container', 'docker']],
|
|
1548
|
+
['nút', ['node']],
|
|
1549
|
+
['khóa', ['key', 'token', 'secret']],
|
|
1550
|
+
['mật khẩu', ['password']],
|
|
1551
|
+
['giám sát', ['monitor', 'monitoring']],
|
|
1552
|
+
['di chuyển', ['migrate', 'migration']],
|
|
1371
1553
|
]);
|
|
1372
1554
|
/**
|
|
1373
1555
|
* Expand a token to its cross-lingual synonyms.
|
|
@@ -1392,7 +1574,8 @@ function expandCrossLingual(token) {
|
|
|
1392
1574
|
*/
|
|
1393
1575
|
function tokenize(text) {
|
|
1394
1576
|
const tokens = [];
|
|
1395
|
-
|
|
1577
|
+
// Pre-process: split camelCase/PascalCase identifiers and snake_case before lowercasing
|
|
1578
|
+
const lower = preprocessText(text).toLowerCase();
|
|
1396
1579
|
for (const [seg] of lower.matchAll(SEGMENT_RE)) {
|
|
1397
1580
|
if (CJK_RE.test(seg)) {
|
|
1398
1581
|
// CJK segment — extract overlapping character bigrams
|
|
@@ -1446,10 +1629,8 @@ function tokenize(text) {
|
|
|
1446
1629
|
}
|
|
1447
1630
|
else if (DEVANAGARI_RE.test(seg)) {
|
|
1448
1631
|
// Devanagari segment (Hindi / Marathi) — naturally space-separated
|
|
1449
|
-
// Words don't need bigram expansion; stems reduce inflected forms.
|
|
1450
1632
|
const words = seg.trim().split(/[\s\u200c\u200d]+/); // handle zero-width joiners
|
|
1451
1633
|
for (const w of words) {
|
|
1452
|
-
// Keep Devanagari chars + digits; discard pure Latin portions (e.g. embedded numbers)
|
|
1453
1634
|
const deva = w.replace(/[^\u0900-\u097f0-9]/g, '');
|
|
1454
1635
|
if (deva.length <= 1 || STOPWORDS.has(deva))
|
|
1455
1636
|
continue;
|
|
@@ -1458,6 +1639,18 @@ function tokenize(text) {
|
|
|
1458
1639
|
tokens.push(stemmed);
|
|
1459
1640
|
}
|
|
1460
1641
|
}
|
|
1642
|
+
else if (BENGALI_RE.test(seg)) {
|
|
1643
|
+
// Bengali (Bangla) segment — space-separated words, Bengali script
|
|
1644
|
+
const words = seg.trim().split(/[\s\u200c\u200d]+/);
|
|
1645
|
+
for (const w of words) {
|
|
1646
|
+
const bn = w.replace(/[^\u0980-\u09ff0-9]/g, '');
|
|
1647
|
+
if (bn.length <= 1 || STOPWORDS.has(bn))
|
|
1648
|
+
continue;
|
|
1649
|
+
const stemmed = bengaliLightStem(bn);
|
|
1650
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1651
|
+
tokens.push(stemmed);
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1461
1654
|
else if (CYRILLIC_RE.test(seg)) {
|
|
1462
1655
|
// Cyrillic segment (Russian, Bulgarian, Ukrainian, Serbian …)
|
|
1463
1656
|
const words = seg.trim().split(/\s+/);
|
|
@@ -1471,9 +1664,10 @@ function tokenize(text) {
|
|
|
1471
1664
|
}
|
|
1472
1665
|
}
|
|
1473
1666
|
else {
|
|
1474
|
-
// Latin / other
|
|
1667
|
+
// Latin / other — includes Turkish (ğ/ş/ı), Polish (ą/ę/ł/ń/ó/ś/ź/ż/ć),
|
|
1668
|
+
// Czech (č/š/ž/ě/ů/ř), Hungarian (ő/ű), Romanian (ș/ț), Vietnamese (ắặầổợụừ…)
|
|
1475
1669
|
const words = seg
|
|
1476
|
-
.replace(/[^a-
|
|
1670
|
+
.replace(/[^a-záàâãäåæçéèêëíìîïñóòôõöúùûüýÿßœğşıİąćęłńśźżčšžěůřőűșț\u1ea0-\u1ef90-9:\-./]/g, ' ')
|
|
1477
1671
|
.split(/\s+/)
|
|
1478
1672
|
.filter(w => w.length > 1 && !STOPWORDS.has(w));
|
|
1479
1673
|
// Turkish: apply light stemmer to words with Turkish-specific chars
|
|
@@ -1527,6 +1721,12 @@ function tokenize(text) {
|
|
|
1527
1721
|
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1528
1722
|
expansions.push(stemmed);
|
|
1529
1723
|
}
|
|
1724
|
+
else if (BENGALI_RE.test(syn)) {
|
|
1725
|
+
// Bengali synonym
|
|
1726
|
+
const stemmed = bengaliLightStem(syn);
|
|
1727
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1728
|
+
expansions.push(stemmed);
|
|
1729
|
+
}
|
|
1530
1730
|
else if (CYRILLIC_RE.test(syn)) {
|
|
1531
1731
|
// Cyrillic synonym (Russian)
|
|
1532
1732
|
const cyr = syn.replace(/[^\u0400-\u04ff]/g, '');
|
|
@@ -1604,6 +1804,17 @@ const BM25_B = 0.75; // length normalization
|
|
|
1604
1804
|
const BM25_DELTA = 1.0; // BM25+ lower-bound guarantee (0 = classic BM25)
|
|
1605
1805
|
/** Recency boost: half-life in days. Entry from 7 days ago gets 0.5× boost. */
|
|
1606
1806
|
const RECENCY_HALF_LIFE_DAYS = 7;
|
|
1807
|
+
const ZERO_RESULTS_LOG = [];
|
|
1808
|
+
const ZERO_RESULTS_MAX = 500;
|
|
1809
|
+
let zeroResultsTotal = 0;
|
|
1810
|
+
function logZeroResult(query) {
|
|
1811
|
+
zeroResultsTotal++;
|
|
1812
|
+
if (ZERO_RESULTS_LOG.length >= ZERO_RESULTS_MAX)
|
|
1813
|
+
ZERO_RESULTS_LOG.shift();
|
|
1814
|
+
ZERO_RESULTS_LOG.push({ query, ts: Date.now() });
|
|
1815
|
+
}
|
|
1816
|
+
/** Global index vocabulary — rebuilt on each keywordSearch call from the doc set. */
|
|
1817
|
+
let _indexVocab = new Set();
|
|
1607
1818
|
/**
|
|
1608
1819
|
* Levenshtein distance — edit distance between two strings.
|
|
1609
1820
|
* Used for typo-tolerant fuzzy matching (distance ≤ 2 = match).
|
|
@@ -1706,7 +1917,9 @@ async function keywordSearch(redis, patterns, query, topK = 10) {
|
|
|
1706
1917
|
bigrams.add(`${tokens[j]}|${tokens[j + 1]}`);
|
|
1707
1918
|
}
|
|
1708
1919
|
const timestamp = extractTimestamp(content);
|
|
1709
|
-
|
|
1920
|
+
// Key tokens for title-boost: terms appearing in the Redis key get extra weight
|
|
1921
|
+
const keyTokens = new Set(tokenize(allKeys[i]));
|
|
1922
|
+
docs.push({ key: allKeys[i], content, tokens, tokenFreq, bigrams, keyTokens, timestamp });
|
|
1710
1923
|
totalTokens += tokens.length;
|
|
1711
1924
|
}
|
|
1712
1925
|
if (docs.length === 0)
|
|
@@ -1742,13 +1955,34 @@ async function keywordSearch(redis, patterns, query, topK = 10) {
|
|
|
1742
1955
|
return idf(term) * (tfNorm + BM25_DELTA);
|
|
1743
1956
|
}
|
|
1744
1957
|
/**
|
|
1745
|
-
* Fuzzy match: tries exact → substring → Levenshtein ≤ 2.
|
|
1958
|
+
* Fuzzy match: tries exact → prefix → substring → Levenshtein ≤ 2.
|
|
1746
1959
|
* Returns [matchedTerm, weight] or null.
|
|
1960
|
+
*
|
|
1961
|
+
* Weights:
|
|
1962
|
+
* 1.0 — exact match
|
|
1963
|
+
* 0.85 — doc token starts with query (e.g. "dockerf" → "dockerfile")
|
|
1964
|
+
* 0.75 — query starts with doc token (e.g. "kubernetes" → "kube")
|
|
1965
|
+
* 0.6 — substring (either direction)
|
|
1966
|
+
* 0.4 — Levenshtein ≤ 2 (typo tolerance)
|
|
1747
1967
|
*/
|
|
1748
1968
|
function fuzzyMatch(qt, docTermSet) {
|
|
1749
1969
|
// Exact
|
|
1750
1970
|
if (docTermSet.has(qt))
|
|
1751
1971
|
return [qt, 1.0];
|
|
1972
|
+
// Prefix: query is prefix of a doc token (user typed partial word)
|
|
1973
|
+
if (qt.length >= 4) {
|
|
1974
|
+
for (const dt of docTermSet) {
|
|
1975
|
+
if (dt.length > qt.length && dt.startsWith(qt))
|
|
1976
|
+
return [dt, 0.85];
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
// Reverse-prefix: doc token is prefix of query (doc has abbreviated form)
|
|
1980
|
+
if (qt.length >= 4) {
|
|
1981
|
+
for (const dt of docTermSet) {
|
|
1982
|
+
if (dt.length >= 4 && dt.length < qt.length && qt.startsWith(dt))
|
|
1983
|
+
return [dt, 0.75];
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1752
1986
|
// Substring (partial)
|
|
1753
1987
|
for (const dt of docTermSet) {
|
|
1754
1988
|
if (dt.length > 3 && qt.length > 3 && (dt.includes(qt) || qt.includes(dt))) {
|
|
@@ -1812,6 +2046,18 @@ async function keywordSearch(redis, patterns, query, topK = 10) {
|
|
|
1812
2046
|
if (sq.length >= 4 && doc.content.toLowerCase().includes(sq.toLowerCase())) {
|
|
1813
2047
|
score *= 2.0;
|
|
1814
2048
|
}
|
|
2049
|
+
// Key/title boost: if query terms appear in the Redis key (=title), 1.5× boost
|
|
2050
|
+
// Keys often encode the primary topic (e.g. "deploy:api:server"), so key hits are high-precision.
|
|
2051
|
+
if (score > 0) {
|
|
2052
|
+
let keyHits = 0;
|
|
2053
|
+
for (const qt of queryTokens) {
|
|
2054
|
+
if (doc.keyTokens.has(qt))
|
|
2055
|
+
keyHits++;
|
|
2056
|
+
}
|
|
2057
|
+
if (keyHits > 0) {
|
|
2058
|
+
score *= 1 + 0.5 * (keyHits / queryTokens.length);
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
1815
2061
|
// Recency boost
|
|
1816
2062
|
score *= recencyBoost(doc.timestamp);
|
|
1817
2063
|
if (score > 0) {
|
|
@@ -1830,7 +2076,17 @@ async function keywordSearch(redis, patterns, query, topK = 10) {
|
|
|
1830
2076
|
}
|
|
1831
2077
|
// ── Step 4: Sort by score, return top-K ──
|
|
1832
2078
|
const sorted = [...allMatches.values()].sort((a, b) => b.score - a.score);
|
|
1833
|
-
|
|
2079
|
+
const topResults = sorted.slice(0, topK);
|
|
2080
|
+
// ── Step 5: Zero-results logging + Did-You-Mean ───────────────────────────
|
|
2081
|
+
if (topResults.length === 0) {
|
|
2082
|
+
logZeroResult(query);
|
|
2083
|
+
// Rebuild index vocab for Did-You-Mean suggestions
|
|
2084
|
+
_indexVocab = new Set();
|
|
2085
|
+
for (const doc of docs)
|
|
2086
|
+
for (const t of doc.tokens)
|
|
2087
|
+
_indexVocab.add(t);
|
|
2088
|
+
}
|
|
2089
|
+
return topResults;
|
|
1834
2090
|
}
|
|
1835
2091
|
// ── Exported for testing ──────────────────────────────────────────────────────
|
|
1836
2092
|
export { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS, katakanaToRomaji, arabicLightStem, expandCrossLingual, CROSS_LINGUAL_MAP };
|
|
@@ -4135,10 +4391,37 @@ async function handleTool(name, args) {
|
|
|
4135
4391
|
}
|
|
4136
4392
|
if (kwMatches.length === 0) {
|
|
4137
4393
|
lines.push(`⚠️ No matches found for: "${query}"`);
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4394
|
+
// Did-You-Mean: find nearest token in index vocab
|
|
4395
|
+
const queryTokens = tokenize(query);
|
|
4396
|
+
const suggestions = [];
|
|
4397
|
+
if (_indexVocab.size > 0 && queryTokens.length > 0) {
|
|
4398
|
+
for (const qt of queryTokens.slice(0, 3)) {
|
|
4399
|
+
if (qt.length < 4)
|
|
4400
|
+
continue;
|
|
4401
|
+
let bestDist = 3;
|
|
4402
|
+
let bestTok = '';
|
|
4403
|
+
for (const v of _indexVocab) {
|
|
4404
|
+
if (v.length < 3 || Math.abs(v.length - qt.length) > 4)
|
|
4405
|
+
continue;
|
|
4406
|
+
const d = levenshtein(qt, v);
|
|
4407
|
+
if (d > 0 && d < bestDist) {
|
|
4408
|
+
bestDist = d;
|
|
4409
|
+
bestTok = v;
|
|
4410
|
+
}
|
|
4411
|
+
}
|
|
4412
|
+
if (bestTok)
|
|
4413
|
+
suggestions.push(`"${bestTok}" (instead of "${qt}")`);
|
|
4414
|
+
}
|
|
4415
|
+
}
|
|
4416
|
+
if (suggestions.length > 0) {
|
|
4417
|
+
lines.push(`💡 **Did you mean:** ${suggestions.join(', ')}?`);
|
|
4418
|
+
}
|
|
4419
|
+
else {
|
|
4420
|
+
lines.push(`\n💡 Tips:`);
|
|
4421
|
+
lines.push(` • Try different keywords`);
|
|
4422
|
+
lines.push(` • Use \`list_remembered\` to see available context`);
|
|
4423
|
+
lines.push(` • Use \`recall_best_solution("topic")\` for exact topic lookup`);
|
|
4424
|
+
}
|
|
4142
4425
|
}
|
|
4143
4426
|
return lines.join('\n');
|
|
4144
4427
|
}
|
|
@@ -6826,7 +7109,14 @@ if (httpPort) {
|
|
|
6826
7109
|
const url = new URL(req.url ?? '/', `http://localhost:${httpPort}`);
|
|
6827
7110
|
if (url.pathname === '/health') {
|
|
6828
7111
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
6829
|
-
res.end(JSON.stringify({
|
|
7112
|
+
res.end(JSON.stringify({
|
|
7113
|
+
status: 'ok',
|
|
7114
|
+
version: CURRENT_VERSION,
|
|
7115
|
+
zeroResults: {
|
|
7116
|
+
total: zeroResultsTotal,
|
|
7117
|
+
last10: ZERO_RESULTS_LOG.slice(-10).map(e => ({ query: e.query, ts: new Date(e.ts).toISOString() })),
|
|
7118
|
+
},
|
|
7119
|
+
}));
|
|
6830
7120
|
return;
|
|
6831
7121
|
}
|
|
6832
7122
|
if (url.pathname === '/mcp' || url.pathname === '/') {
|