@cachly-dev/mcp-server 0.9.17 → 0.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/publish.yml +42 -0
- package/LICENSE +202 -0
- package/dist/__tests__/brain-flow.test.d.ts +12 -0
- package/dist/__tests__/brain-flow.test.js +504 -0
- package/dist/__tests__/brain-flow.test.js.map +1 -0
- package/dist/ckg.d.ts +22 -0
- package/dist/ckg.js +40 -0
- package/dist/ckg.js.map +1 -0
- package/dist/confidence.d.ts +19 -0
- package/dist/confidence.js +38 -0
- package/dist/confidence.js.map +1 -0
- package/dist/embeddings.d.ts +48 -0
- package/dist/embeddings.js +213 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/handlers/brain.d.ts +21 -0
- package/dist/handlers/brain.js +1322 -0
- package/dist/handlers/brain.js.map +1 -0
- package/dist/index.d.ts +1 -62
- package/dist/index.js +60 -5069
- package/dist/index.js.map +1 -1
- package/dist/search.d.ts +101 -0
- package/dist/search.js +1818 -0
- package/dist/search.js.map +1 -0
- package/dist/tools.d.ts +1955 -0
- package/dist/tools.js +1632 -0
- package/dist/tools.js.map +1 -0
- package/glama.json +6 -0
- package/package.json +1 -1
- package/src/__tests__/brain-flow.test.ts +608 -0
- package/src/ckg.ts +53 -0
- package/src/confidence.ts +38 -0
- package/src/embeddings.ts +217 -0
- package/src/handlers/brain.ts +1506 -0
- package/src/index.ts +40 -5261
- package/src/search.ts +1781 -0
- package/src/tools.ts +1723 -0
package/dist/search.js
ADDED
|
@@ -0,0 +1,1818 @@
|
|
|
1
|
+
// ── Search Engine (BM25+ with enhancements, works without embedding) ──────────
|
|
2
|
+
//
|
|
3
|
+
// Features beyond standard BM25:
|
|
4
|
+
// • BM25+ (delta=1) — fixes BM25's "long document penalty" bug
|
|
5
|
+
// • Bigram proximity boost — adjacent query terms in doc score 2× more
|
|
6
|
+
// • Recency boost — newer entries rank higher (exponential decay, 7-day half-life)
|
|
7
|
+
// • Multi-query splitting — numbered lists, semicolons, conjunctions
|
|
8
|
+
// • Fuzzy matching with Levenshtein distance ≤ 2
|
|
9
|
+
// • Multilingual stopwords (EN, DE, FR, ES, IT, PT, ZH, JA, KO, AR, HE)
|
|
10
|
+
// • CJK character bigram extraction — handles Chinese, Japanese, Korean
|
|
11
|
+
// • RTL language support (Arabic, Hebrew) — word tokenization + light stemming
|
|
12
|
+
// • Romanization matching — katakana → romaji tokens for romaji queries
|
|
13
|
+
// • Cross-language retrieval — tech term synonyms EN↔JA↔ZH↔KO↔AR↔HE
|
|
14
|
+
// • Pipeline Redis reads for performance
|
|
15
|
+
//
|
|
16
|
+
/**
|
|
17
|
+
* Stopwords — filtered out during tokenization.
|
|
18
|
+
* Covers: English, German, French, Spanish, Italian, Portuguese,
|
|
19
|
+
* Chinese (Simplified + Traditional), Japanese, Korean.
|
|
20
|
+
* Keeps the index small and scores meaningful.
|
|
21
|
+
*/
|
|
22
|
+
const STOPWORDS = new Set([
|
|
23
|
+
// ── English ──
|
|
24
|
+
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
25
|
+
'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing',
|
|
26
|
+
'will', 'would', 'could', 'should', 'may', 'might', 'shall', 'can',
|
|
27
|
+
'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as',
|
|
28
|
+
'into', 'through', 'during', 'before', 'after', 'above', 'below',
|
|
29
|
+
'between', 'about', 'against', 'among', 'around', 'without', 'within',
|
|
30
|
+
'along', 'across', 'behind', 'beyond', 'upon', 'toward', 'towards',
|
|
31
|
+
'and', 'but', 'or', 'not', 'no', 'nor', 'so', 'if', 'then', 'than',
|
|
32
|
+
'too', 'very', 'quite', 'rather', 'just', 'also', 'only', 'even',
|
|
33
|
+
'it', 'its', 'this', 'that', 'these', 'those', 'here', 'there',
|
|
34
|
+
'my', 'your', 'his', 'her', 'our', 'their', 'mine', 'yours', 'ours',
|
|
35
|
+
'what', 'which', 'who', 'whom', 'whose', 'how', 'when', 'where', 'why',
|
|
36
|
+
'all', 'each', 'every', 'both', 'few', 'more', 'most', 'much', 'many',
|
|
37
|
+
'other', 'some', 'such', 'own', 'same', 'any', 'either', 'neither',
|
|
38
|
+
'been', 'being', 'because', 'until', 'while', 'once', 'again', 'further',
|
|
39
|
+
'already', 'always', 'never', 'sometimes', 'often', 'still', 'yet',
|
|
40
|
+
// ── German ──
|
|
41
|
+
'der', 'die', 'das', 'den', 'dem', 'des', 'ein', 'eine', 'einen',
|
|
42
|
+
'einem', 'einer', 'eines', 'und', 'oder', 'aber', 'denn', 'weil',
|
|
43
|
+
'ist', 'sind', 'war', 'waren', 'sein', 'wird', 'werden', 'wurde',
|
|
44
|
+
'hat', 'haben', 'hatte', 'hatten', 'kann', 'können', 'konnte',
|
|
45
|
+
'soll', 'sollen', 'sollte', 'muss', 'müssen', 'musste', 'darf',
|
|
46
|
+
'mag', 'möchte', 'wollen', 'wollte', 'würde', 'könnte', 'sollte',
|
|
47
|
+
'mit', 'für', 'auf', 'von', 'aus', 'bei', 'nach', 'über', 'unter',
|
|
48
|
+
'vor', 'hinter', 'neben', 'zwischen', 'durch', 'gegen', 'ohne',
|
|
49
|
+
'um', 'bis', 'seit', 'während', 'wegen', 'trotz', 'statt',
|
|
50
|
+
'wie', 'was', 'wer', 'wen', 'wem', 'wessen', 'wo', 'wann', 'warum',
|
|
51
|
+
'nicht', 'noch', 'auch', 'schon', 'nur', 'sehr', 'mehr', 'viel',
|
|
52
|
+
'alle', 'jeder', 'jede', 'jedes', 'dieser', 'diese', 'dieses',
|
|
53
|
+
'jener', 'jene', 'jenes', 'mein', 'dein', 'sein', 'ihr', 'unser',
|
|
54
|
+
'euer', 'kein', 'keine', 'sich', 'mir', 'dir', 'ihm', 'uns', 'euch',
|
|
55
|
+
'ich', 'du', 'er', 'sie', 'es', 'wir', 'ihr', 'man',
|
|
56
|
+
'hier', 'dort', 'da', 'dann', 'also', 'doch', 'mal', 'eben', 'ganz',
|
|
57
|
+
// ── French ──
|
|
58
|
+
'le', 'la', 'les', 'un', 'une', 'des', 'du', 'de', 'au', 'aux',
|
|
59
|
+
'et', 'ou', 'mais', 'donc', 'car', 'ni', 'que', 'qui', 'quoi',
|
|
60
|
+
'est', 'sont', 'était', 'ont', 'avoir', 'être', 'fait', 'faire',
|
|
61
|
+
'pour', 'par', 'avec', 'dans', 'sur', 'sous', 'entre', 'vers',
|
|
62
|
+
'chez', 'sans', 'avant', 'après', 'pendant', 'depuis', 'contre',
|
|
63
|
+
'ce', 'cette', 'ces', 'mon', 'ton', 'son', 'notre', 'votre', 'leur',
|
|
64
|
+
'je', 'tu', 'il', 'elle', 'nous', 'vous', 'ils', 'elles', 'on',
|
|
65
|
+
'ne', 'pas', 'plus', 'très', 'bien', 'aussi', 'tout', 'tous', 'toute',
|
|
66
|
+
'même', 'autre', 'quel', 'quelle', 'comment', 'quand', 'où', 'pourquoi',
|
|
67
|
+
// ── Spanish ──
|
|
68
|
+
'el', 'la', 'los', 'las', 'un', 'una', 'unos', 'unas', 'del', 'al',
|
|
69
|
+
'lo', 'que', 'en', 'es', 'por', 'con', 'para', 'como', 'pero', 'más',
|
|
70
|
+
'fue', 'ser', 'hay', 'está', 'han', 'son', 'tiene', 'había', 'era',
|
|
71
|
+
'su', 'sus', 'este', 'esta', 'estos', 'estas', 'ese', 'esa', 'esos',
|
|
72
|
+
'mi', 'tu', 'yo', 'él', 'ella', 'nosotros', 'ellos', 'ellas', 'usted',
|
|
73
|
+
'no', 'ya', 'sí', 'sin', 'sobre', 'entre', 'hasta', 'desde', 'donde',
|
|
74
|
+
'muy', 'todo', 'toda', 'todos', 'cada', 'otro', 'otra', 'otros',
|
|
75
|
+
'cuando', 'porque', 'aunque', 'también', 'solo', 'después', 'antes',
|
|
76
|
+
// ── Italian ──
|
|
77
|
+
'il', 'lo', 'la', 'li', 'le', 'gli', 'uno', 'una', 'dei', 'del',
|
|
78
|
+
'che', 'di', 'da', 'per', 'con', 'tra', 'fra', 'sul', 'nel', 'al',
|
|
79
|
+
'è', 'sono', 'ha', 'hanno', 'era', 'essere', 'fare', 'fatto', 'stato',
|
|
80
|
+
'suo', 'sua', 'suoi', 'questo', 'questa', 'questi', 'quello', 'quella',
|
|
81
|
+
'io', 'tu', 'lui', 'lei', 'noi', 'voi', 'loro', 'ci', 'si',
|
|
82
|
+
'non', 'più', 'molto', 'anche', 'solo', 'tutto', 'tutti', 'ogni',
|
|
83
|
+
'come', 'dove', 'quando', 'perché', 'ancora', 'già', 'sempre', 'mai',
|
|
84
|
+
// ── Portuguese ──
|
|
85
|
+
'um', 'uma', 'uns', 'umas', 'do', 'da', 'dos', 'das', 'no', 'na',
|
|
86
|
+
'ao', 'aos', 'em', 'por', 'com', 'para', 'sem', 'sob', 'sobre',
|
|
87
|
+
'que', 'se', 'mas', 'ou', 'como', 'mais', 'entre', 'até', 'desde',
|
|
88
|
+
'é', 'são', 'foi', 'tem', 'ser', 'ter', 'estar', 'fazer', 'havia',
|
|
89
|
+
'seu', 'sua', 'seus', 'suas', 'este', 'esta', 'esse', 'essa', 'aquele',
|
|
90
|
+
'eu', 'tu', 'ele', 'ela', 'nós', 'eles', 'elas', 'você', 'vocês',
|
|
91
|
+
'não', 'já', 'sim', 'bem', 'muito', 'também', 'ainda', 'sempre',
|
|
92
|
+
'todo', 'toda', 'todos', 'cada', 'outro', 'outra', 'quando', 'porque',
|
|
93
|
+
// ── Chinese (Simplified + Traditional) — high-frequency function characters ──
|
|
94
|
+
'的', '了', '是', '在', '不', '和', '我', '他', '这', '中', '大', '为',
|
|
95
|
+
'上', '个', '国', '以', '要', '就', '出', '说', '们', '有', '来', '到',
|
|
96
|
+
'时', '地', '年', '得', '着', '那', '过', '后', '还', '与', '也', '可',
|
|
97
|
+
'于', '从', '但', '而', '被', '把', '让', '使', '对', '很', '都', '一',
|
|
98
|
+
'会', '没', '人', '它', '这个', '那个', '什么', '如果', '因为', '所以',
|
|
99
|
+
'已经', '可以', '这些', '那些', '我们', '他们', '她们', '它们',
|
|
100
|
+
// ── Japanese — common hiragana particles and auxiliary verbs ──
|
|
101
|
+
'の', 'は', 'が', 'を', 'に', 'で', 'と', 'も', 'や', 'か', 'な', 'ね',
|
|
102
|
+
'よ', 'わ', 'て', 'い', 'う', 'え', 'お', 'き', 'く', 'け', 'こ', 'さ',
|
|
103
|
+
'し', 'す', 'せ', 'そ', 'た', 'ち', 'つ', 'ぬ', 'ん', 'から', 'まで',
|
|
104
|
+
'より', 'へ', 'です', 'ます', 'ない', 'ある', 'いる', 'する', 'こと',
|
|
105
|
+
'もの', 'ので', 'では', 'には', 'との', 'への', 'から', 'まで',
|
|
106
|
+
// ── Korean — common particles and auxiliary forms ──
|
|
107
|
+
'이', '가', '은', '는', '을', '를', '의', '에', '와', '과', '도',
|
|
108
|
+
'로', '에서', '한', '하', '있', '없', '것', '수', '않', '들',
|
|
109
|
+
'이다', '하다', '이고', '이며', '라고', '에게', '에서', '으로',
|
|
110
|
+
// ── Arabic — high-frequency function words & particles ──
|
|
111
|
+
'في', 'من', 'إلى', 'على', 'عن', 'مع', 'هذا', 'هذه', 'ذلك', 'تلك',
|
|
112
|
+
'الذي', 'التي', 'الذين', 'اللواتي', 'هو', 'هي', 'هم', 'هن', 'نحن',
|
|
113
|
+
'أنت', 'أنتم', 'أنا', 'كان', 'كانت', 'كانوا', 'يكون', 'تكون',
|
|
114
|
+
'أن', 'إن', 'لأن', 'لكن', 'أو', 'بل', 'ثم', 'حتى', 'إذا', 'كي',
|
|
115
|
+
'قد', 'لقد', 'لم', 'لن', 'ما', 'لا', 'ليس', 'غير', 'بعض', 'كل',
|
|
116
|
+
'جميع', 'أي', 'كيف', 'متى', 'أين', 'لماذا', 'ماذا', 'هل', 'ال',
|
|
117
|
+
'و', 'ف', 'ب', 'ل', 'ك', 'يا', 'أم', 'إلا', 'عند', 'بين',
|
|
118
|
+
// ── Hebrew — common particles, pronouns, conjunctions ──
|
|
119
|
+
'של', 'את', 'אל', 'על', 'עם', 'לא', 'הוא', 'היא', 'הם', 'הן',
|
|
120
|
+
'אני', 'אתה', 'אנחנו', 'אתם', 'כי', 'אם', 'אבל', 'גם', 'כבר',
|
|
121
|
+
'רק', 'עוד', 'יש', 'אין', 'מה', 'זה', 'זאת', 'אלה', 'כל', 'כן',
|
|
122
|
+
'לו', 'לה', 'להם', 'בו', 'בה', 'בהם', 'שם', 'כך', 'כן', 'מי',
|
|
123
|
+
'אשר', 'אחרי', 'לפני', 'תחת', 'בין', 'מאז', 'עד', 'כמו', 'אז',
|
|
124
|
+
// ── Farsi/Persian — common function words, prepositions, pronouns ──
|
|
125
|
+
'در', 'از', 'به', 'با', 'که', 'این', 'آن', 'را', 'است',
|
|
126
|
+
'بود', 'باشد', 'شد', 'شده', 'می', 'نه', 'نیست', 'هم', 'هر',
|
|
127
|
+
'یک', 'اما', 'و', 'یا', 'تا', 'اگر', 'برای', 'چون', 'چه',
|
|
128
|
+
'کجا', 'کی', 'چطور', 'وقتی', 'بعد', 'قبل', 'مثل', 'همه', 'بعضی',
|
|
129
|
+
'هیچ', 'آیا', 'خود', 'چند', 'دیگر', 'هنوز', 'همان', 'آنها', 'اینها',
|
|
130
|
+
'ما', 'شما', 'آنان', 'من', 'تو', 'او', 'آنان', 'ایشان',
|
|
131
|
+
'روی', 'زیر', 'بین', 'پیش', 'پس', 'طرف', 'داخل', 'خارج', 'کنار',
|
|
132
|
+
'همچنین', 'مگر', 'ولی', 'وگرنه', 'چرا', 'چگونه', 'کدام',
|
|
133
|
+
// ── Hindi — common particles, postpositions, pronouns, auxiliary verbs ──
|
|
134
|
+
'है', 'में', 'से', 'को', 'का', 'की', 'के', 'पर', 'और', 'या',
|
|
135
|
+
'नहीं', 'यह', 'वह', 'एक', 'इस', 'उस', 'भी', 'हो', 'था', 'थी',
|
|
136
|
+
'थे', 'हैं', 'हूँ', 'लिए', 'तक', 'साथ', 'बाद', 'पहले', 'जो',
|
|
137
|
+
'जब', 'कैसे', 'क्यों', 'क्या', 'कहाँ', 'कौन', 'हम', 'आप', 'वे',
|
|
138
|
+
'मैं', 'तुम', 'उन', 'इन', 'ने', 'बहुत', 'सब', 'कुछ', 'फिर',
|
|
139
|
+
'अब', 'तो', 'ही', 'तरह', 'जैसे', 'करना', 'होना', 'रहा', 'रही',
|
|
140
|
+
'रहे', 'गया', 'गई', 'गए', 'किया', 'किए', 'कर', 'हुआ', 'हुई',
|
|
141
|
+
// ── Russian — common prepositions, pronouns, auxiliary verbs ───────────────
|
|
142
|
+
'в', 'на', 'не', 'с', 'и', 'а', 'но', 'по', 'за', 'из', 'от', 'до',
|
|
143
|
+
'к', 'у', 'о', 'об', 'во', 'при', 'под', 'над', 'без', 'для',
|
|
144
|
+
'что', 'как', 'это', 'все', 'так', 'уже', 'или', 'же', 'ли',
|
|
145
|
+
'если', 'то', 'да', 'нет', 'был', 'была', 'были', 'быть', 'есть',
|
|
146
|
+
'его', 'её', 'их', 'этот', 'эта', 'эти', 'который', 'которая',
|
|
147
|
+
'которые', 'который', 'мой', 'моя', 'мои', 'ваш', 'ваша', 'ваши',
|
|
148
|
+
'он', 'она', 'они', 'мы', 'вы', 'я', 'ты', 'тот', 'та', 'те',
|
|
149
|
+
'очень', 'тоже', 'также', 'когда', 'где', 'как', 'почему', 'зачем',
|
|
150
|
+
'сейчас', 'здесь', 'там', 'потому', 'поэтому', 'чтобы', 'тут',
|
|
151
|
+
// ── Turkish — common particles, postpositions, copulas ─────────────────────
|
|
152
|
+
'bir', 'bu', 'şu', 'o', 've', 'ile', 'de', 'da', 'için', 'gibi',
|
|
153
|
+
'ama', 'fakat', 'ancak', 'ya', 'ne', 'ki', 'mi', 'mı', 'mu', 'mü',
|
|
154
|
+
'var', 'yok', 'olan', 'oldu', 'olur', 'olarak', 'ise', 'daha',
|
|
155
|
+
'çok', 'en', 'her', 'hiç', 'bazı', 'tüm', 'bütün', 'hem', 'veya',
|
|
156
|
+
'bu', 'şu', 'ben', 'sen', 'biz', 'siz', 'onlar', 'benim', 'senin',
|
|
157
|
+
'nasıl', 'neden', 'niçin', 'nerede', 'ne', 'hangi', 'kaç',
|
|
158
|
+
// ── Polish — common prepositions, pronouns, particles ─────────────────────
|
|
159
|
+
'w', 'na', 'z', 'do', 'się', 'że', 'to', 'jest', 'i', 'a', 'nie',
|
|
160
|
+
'tak', 'jak', 'czy', 'po', 'o', 'ale', 'go', 'mu', 'jej', 'ich',
|
|
161
|
+
'tego', 'tej', 'te', 'ten', 'ta', 'są', 'był', 'była', 'było', 'byli',
|
|
162
|
+
'będzie', 'będą', 'ma', 'mam', 'masz', 'mają', 'ze', 'co', 'już',
|
|
163
|
+
'przez', 'przy', 'za', 'bez', 'nad', 'pod', 'przed', 'po', 'między',
|
|
164
|
+
'kiedy', 'gdzie', 'dlaczego', 'który', 'która', 'które', 'tylko', 'też',
|
|
165
|
+
// ── Czech — common words ──────────────────────────────────────────────────
|
|
166
|
+
'v', 'na', 'z', 'do', 'se', 'že', 'to', 'je', 'a', 'ne', 'pro',
|
|
167
|
+
'ale', 'jak', 'by', 'byl', 'být', 'jsem', 'jsou', 'má', 'mám',
|
|
168
|
+
'ten', 'ta', 'ty', 'tato', 'jeho', 'její', 'jejich', 'také', 'jen',
|
|
169
|
+
'kde', 'kdy', 'proč', 'který', 'která', 'které', 'co', 'při', 'bez',
|
|
170
|
+
// ── Bengali — common particles, pronouns, verbs ───────────────────────────
|
|
171
|
+
'এই', 'এটি', 'এটা', 'এর', 'তার', 'তারা', 'আমি', 'তুমি', 'সে', 'আমরা',
|
|
172
|
+
'এবং', 'কিন্তু', 'বা', 'না', 'হ্যাঁ', 'যে', 'কি', 'কে', 'কী', 'থেকে',
|
|
173
|
+
'দিয়ে', 'জন্য', 'সাথে', 'মধ্যে', 'উপর', 'নিচে', 'আছে', 'ছিল', 'হয়',
|
|
174
|
+
'করা', 'করে', 'করেছে', 'হবে', 'হয়েছে', 'একটি', 'একটা', 'অনেক', 'সব',
|
|
175
|
+
// ── Vietnamese — common particles, pronouns, auxiliary words ─────────────
|
|
176
|
+
'tôi', 'bạn', 'anh', 'chị', 'em', 'họ', 'chúng', 'ta', 'mình',
|
|
177
|
+
'là', 'có', 'không', 'và', 'với', 'của', 'cho', 'trong', 'về',
|
|
178
|
+
'từ', 'đến', 'để', 'khi', 'nếu', 'thì', 'mà', 'nhưng', 'vì',
|
|
179
|
+
'đây', 'đó', 'này', 'kia', 'rất', 'cũng', 'đã', 'sẽ', 'đang',
|
|
180
|
+
'được', 'bị', 'những', 'các', 'một', 'hai', 'ba', 'nhiều', 'ít',
|
|
181
|
+
'nào', 'ai', 'gì', 'đâu', 'sao', 'bao', 'giờ', 'lúc', 'sau',
|
|
182
|
+
]);
|
|
183
|
+
// CJK Unicode ranges used for bigram extraction
|
|
184
|
+
const CJK_RE = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]/;
|
|
185
|
+
const SEGMENT_RE = /([\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]+|[^\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]+)/g;
|
|
186
|
+
// RTL Unicode ranges: Arabic (U+0600–U+06FF + extended), Hebrew (U+0590–U+05FF)
|
|
187
|
+
const RTL_RE = /[\u0590-\u05ff\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff\ufb1d-\ufb4f\ufb50-\ufdff\ufe70-\ufeff]/;
|
|
188
|
+
// Distinguish Arabic from Hebrew within RTL segments
|
|
189
|
+
const HEBREW_CHAR_RE = /[\u0590-\u05ff\ufb1d-\ufb4f]/;
|
|
190
|
+
const ARABIC_CHAR_RE = /[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff\ufb50-\ufdff\ufe70-\ufeff]/;
|
|
191
|
+
// Persian/Farsi: exclusive chars not in Arabic (U+067E=پ, U+0686=چ, U+0698=ژ, U+06AF=گ, U+06CC=ی)
|
|
192
|
+
const FARSI_CHAR_RE = /[\u067e\u0686\u0698\u06af\u06cc]/;
|
|
193
|
+
// Devanagari (Hindi, Sanskrit, Marathi …) — U+0900–U+097F + extended
|
|
194
|
+
const DEVANAGARI_RE = /[\u0900-\u097f]/;
|
|
195
|
+
// Cyrillic (Russian, Bulgarian, Ukrainian, Serbian …) — U+0400–U+04FF
|
|
196
|
+
const CYRILLIC_RE = /[\u0400-\u04ff]/;
|
|
197
|
+
// Turkish uses Latin alphabet but has unique chars (ğ, ş, ı, ö, ü, ç) — detected by these
|
|
198
|
+
const TURKISH_CHAR_RE = /[\u011f\u015f\u0131\u0130\u00e7]/; // ğ ş ı İ ç
|
|
199
|
+
// Bengali (Bangla) — U+0980–U+09FF
|
|
200
|
+
const BENGALI_RE = /[\u0980-\u09ff]/;
|
|
201
|
+
// Vietnamese uses Latin + Latin Extended Additional (U+1EA0–U+1EF9) for tone marks
|
|
202
|
+
// e.g. ắ ặ ầ ổ ợ ụ ừ — detected by chars exclusive to Vietnamese diacritics
|
|
203
|
+
const VIETNAMESE_CHAR_RE = /[\u1ea0-\u1ef9]/;
|
|
204
|
+
// ── Katakana → Romaji conversion table (Hepburn system) ──────────────────────
|
|
205
|
+
// Digraphs must be listed before single chars so they match first.
|
|
206
|
+
const KATA_ROMAJI_TABLE = [
|
|
207
|
+
// Special combinations for loanwords
|
|
208
|
+
['ファ', 'fa'], ['フィ', 'fi'], ['フェ', 'fe'], ['フォ', 'fo'],
|
|
209
|
+
['ティ', 'ti'], ['ディ', 'di'], ['トゥ', 'tu'], ['ドゥ', 'du'],
|
|
210
|
+
['ウィ', 'wi'], ['ウェ', 'we'], ['ウォ', 'wo'],
|
|
211
|
+
['チェ', 'che'], ['ジェ', 'je'], ['シェ', 'she'],
|
|
212
|
+
['イェ', 'ye'], ['ヴァ', 'va'], ['ヴィ', 'vi'], ['ヴェ', 've'], ['ヴォ', 'vo'],
|
|
213
|
+
// Digraphs (2-char → romaji)
|
|
214
|
+
['キャ', 'kya'], ['キュ', 'kyu'], ['キョ', 'kyo'],
|
|
215
|
+
['シャ', 'sha'], ['シュ', 'shu'], ['ショ', 'sho'],
|
|
216
|
+
['チャ', 'cha'], ['チュ', 'chu'], ['チョ', 'cho'],
|
|
217
|
+
['ニャ', 'nya'], ['ニュ', 'nyu'], ['ニョ', 'nyo'],
|
|
218
|
+
['ヒャ', 'hya'], ['ヒュ', 'hyu'], ['ヒョ', 'hyo'],
|
|
219
|
+
['ミャ', 'mya'], ['ミュ', 'myu'], ['ミョ', 'myo'],
|
|
220
|
+
['リャ', 'rya'], ['リュ', 'ryu'], ['リョ', 'ryo'],
|
|
221
|
+
['ギャ', 'gya'], ['ギュ', 'gyu'], ['ギョ', 'gyo'],
|
|
222
|
+
['ジャ', 'ja'], ['ジュ', 'ju'], ['ジョ', 'jo'],
|
|
223
|
+
['ビャ', 'bya'], ['ビュ', 'byu'], ['ビョ', 'byo'],
|
|
224
|
+
['ピャ', 'pya'], ['ピュ', 'pyu'], ['ピョ', 'pyo'],
|
|
225
|
+
// Single chars
|
|
226
|
+
['ア', 'a'], ['イ', 'i'], ['ウ', 'u'], ['エ', 'e'], ['オ', 'o'],
|
|
227
|
+
['カ', 'ka'], ['キ', 'ki'], ['ク', 'ku'], ['ケ', 'ke'], ['コ', 'ko'],
|
|
228
|
+
['サ', 'sa'], ['シ', 'shi'], ['ス', 'su'], ['セ', 'se'], ['ソ', 'so'],
|
|
229
|
+
['タ', 'ta'], ['チ', 'chi'], ['ツ', 'tsu'], ['テ', 'te'], ['ト', 'to'],
|
|
230
|
+
['ナ', 'na'], ['ニ', 'ni'], ['ヌ', 'nu'], ['ネ', 'ne'], ['ノ', 'no'],
|
|
231
|
+
['ハ', 'ha'], ['ヒ', 'hi'], ['フ', 'fu'], ['ヘ', 'he'], ['ホ', 'ho'],
|
|
232
|
+
['マ', 'ma'], ['ミ', 'mi'], ['ム', 'mu'], ['メ', 'me'], ['モ', 'mo'],
|
|
233
|
+
['ヤ', 'ya'], ['ユ', 'yu'], ['ヨ', 'yo'],
|
|
234
|
+
['ラ', 'ra'], ['リ', 'ri'], ['ル', 'ru'], ['レ', 're'], ['ロ', 'ro'],
|
|
235
|
+
['ワ', 'wa'], ['ヲ', 'wo'], ['ン', 'n'],
|
|
236
|
+
// Voiced
|
|
237
|
+
['ガ', 'ga'], ['ギ', 'gi'], ['グ', 'gu'], ['ゲ', 'ge'], ['ゴ', 'go'],
|
|
238
|
+
['ザ', 'za'], ['ジ', 'ji'], ['ズ', 'zu'], ['ゼ', 'ze'], ['ゾ', 'zo'],
|
|
239
|
+
['ダ', 'da'], ['ヂ', 'ji'], ['ヅ', 'zu'], ['デ', 'de'], ['ド', 'do'],
|
|
240
|
+
['バ', 'ba'], ['ビ', 'bi'], ['ブ', 'bu'], ['ベ', 'be'], ['ボ', 'bo'],
|
|
241
|
+
['パ', 'pa'], ['ピ', 'pi'], ['プ', 'pu'], ['ペ', 'pe'], ['ポ', 'po'],
|
|
242
|
+
['ヴ', 'v'],
|
|
243
|
+
// Long vowel / small chars
|
|
244
|
+
['ー', ''], ['ァ', 'a'], ['ィ', 'i'], ['ゥ', 'u'], ['ェ', 'e'], ['ォ', 'o'],
|
|
245
|
+
['ッ', ''], // handled separately (doubles next consonant)
|
|
246
|
+
];
|
|
247
|
+
const KATA_MAP = new Map(KATA_ROMAJI_TABLE);
|
|
248
|
+
/**
|
|
249
|
+
* Convert a katakana string to Hepburn romaji.
|
|
250
|
+
* Handles digraphs, geminate consonants (ッ), and long vowel marks (ー).
|
|
251
|
+
*/
|
|
252
|
+
function katakanaToRomaji(kata) {
|
|
253
|
+
let result = '';
|
|
254
|
+
let i = 0;
|
|
255
|
+
while (i < kata.length) {
|
|
256
|
+
// Geminate consonant: ッ doubles the following consonant
|
|
257
|
+
if (kata[i] === 'ッ' && i + 1 < kata.length) {
|
|
258
|
+
const next = KATA_MAP.get(kata[i + 1]) ?? KATA_MAP.get(kata[i + 1] + kata[i + 2]) ?? '';
|
|
259
|
+
if (next.length > 0)
|
|
260
|
+
result += next[0]; // double first consonant
|
|
261
|
+
i++;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
// Try 2-char digraph first
|
|
265
|
+
if (i + 1 < kata.length) {
|
|
266
|
+
const two = kata[i] + kata[i + 1];
|
|
267
|
+
const r2 = KATA_MAP.get(two);
|
|
268
|
+
if (r2 !== undefined) {
|
|
269
|
+
result += r2;
|
|
270
|
+
i += 2;
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
// Single char
|
|
275
|
+
const r1 = KATA_MAP.get(kata[i]);
|
|
276
|
+
if (r1 !== undefined)
|
|
277
|
+
result += r1;
|
|
278
|
+
i++;
|
|
279
|
+
}
|
|
280
|
+
return result;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Arabic light stemmer — strips definite article and common prefix particles.
|
|
284
|
+
* Handles: ال (al-), و (wa-), ب (bi-), ل (li-), ف (fa-), ك (ka-).
|
|
285
|
+
* Runs iteratively (max 3 passes) so compound prefixes like فال- are fully
|
|
286
|
+
* resolved: فالخطأ → الخطأ → خطأ.
|
|
287
|
+
* Only strips when the result is still ≥ 3 chars to avoid over-stemming.
|
|
288
|
+
*/
|
|
289
|
+
function arabicLightStem(word) {
|
|
290
|
+
let result = word;
|
|
291
|
+
for (let i = 0; i < 3; i++) {
|
|
292
|
+
const prev = result;
|
|
293
|
+
if (result.startsWith('ال') && result.length > 4) {
|
|
294
|
+
result = result.slice(2);
|
|
295
|
+
}
|
|
296
|
+
else if (result.length > 3 && 'وبلفك'.includes(result[0]) && RTL_RE.test(result[1])) {
|
|
297
|
+
result = result.slice(1);
|
|
298
|
+
}
|
|
299
|
+
if (result === prev)
|
|
300
|
+
break; // stable — no more prefixes to strip
|
|
301
|
+
}
|
|
302
|
+
return result;
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Hebrew light stemmer — strips the definite article and common prefix particles
|
|
306
|
+
* that attach directly to words (no space) in Hebrew.
|
|
307
|
+
*
|
|
308
|
+
* Handles:
|
|
309
|
+
* ה (ha-) — definite article: הפריסה → פריסה
|
|
310
|
+
* ו (ve-/u-) — conjunction “and”: ופריסה → פריסה
|
|
311
|
+
* ב (be-/bi-) — preposition “in”: בסביבה → סביבה
|
|
312
|
+
* ל (le-/li-) — preposition “for”: לשרת → שרת
|
|
313
|
+
* מ (mi-/me-) — preposition “from”: מהשרת → השרת → שרת
|
|
314
|
+
* כ (ke-/ki-) — preposition “like”: כשרת → שרת
|
|
315
|
+
*
|
|
316
|
+
* Iterative (max 3 passes): מהפריסה → הפריסה → פריסה.
|
|
317
|
+
* Only strips when result is still ≥ 3 chars.
|
|
318
|
+
*/
|
|
319
|
+
function hebrewLightStem(word) {
|
|
320
|
+
let result = word;
|
|
321
|
+
for (let i = 0; i < 3; i++) {
|
|
322
|
+
const prev = result;
|
|
323
|
+
if (result.length > 3 && 'הובלמכ'.includes(result[0]) && HEBREW_CHAR_RE.test(result[1])) {
|
|
324
|
+
result = result.slice(1);
|
|
325
|
+
}
|
|
326
|
+
if (result === prev)
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
return result;
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* Farsi/Persian light stemmer — handles Persian morphology.
|
|
333
|
+
* Strips:
|
|
334
|
+
* \u0647\u0627 / \u0647\u0627\u06cc (ha/haye) — plural suffixes: \u0633\u0631\u0648\u0631\u0647\u0627 \u2192 \u0633\u0631\u0648\u0631
|
|
335
|
+
* \u0645\u06cc\u200c / \u0645\u06cc (mi-) — present-tense prefix: \u0645\u06cc\u200c\u06a9\u0646\u062f \u2192 \u06a9\u0646\u062f
|
|
336
|
+
* \u0646\u0645\u06cc\u200c (nami-) — negated present: \u0646\u0645\u06cc\u200c\u0634\u0648\u062f \u2192 \u0634\u0648\u062f
|
|
337
|
+
* Only strips when the result is still \u2265 3 chars.
|
|
338
|
+
*/
|
|
339
|
+
function farsiLightStem(word) {
|
|
340
|
+
let result = word;
|
|
341
|
+
// Suffixes first (longest first)
|
|
342
|
+
if (result.endsWith('\u0647\u0627\u06cc') && result.length > 5)
|
|
343
|
+
result = result.slice(0, -3);
|
|
344
|
+
else if (result.endsWith('\u0647\u0627') && result.length > 4)
|
|
345
|
+
result = result.slice(0, -2);
|
|
346
|
+
// Prefixes
|
|
347
|
+
if (result.startsWith('\u0646\u0645\u06cc\u200c') && result.length > 5)
|
|
348
|
+
result = result.slice(4);
|
|
349
|
+
else if (result.startsWith('\u0645\u06cc\u200c') && result.length > 4)
|
|
350
|
+
result = result.slice(3);
|
|
351
|
+
else if (result.startsWith('\u0645\u06cc') && result.length > 4)
|
|
352
|
+
result = result.slice(2);
|
|
353
|
+
return result;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* Hindi light stemmer — strips the most common inflectional suffixes.
|
|
357
|
+
* Handles basic verb forms and oblique plural markers.
|
|
358
|
+
* Avoids over-stemming loanwords (most tech terms in Hindi are English loanwords).
|
|
359
|
+
* Only strips when result is still \u2265 3 chars.
|
|
360
|
+
*/
|
|
361
|
+
function hindiLightStem(word) {
|
|
362
|
+
const result = word;
|
|
363
|
+
// Infinitive / verb suffixes (longest first to avoid partial match)
|
|
364
|
+
if (result.endsWith('\u0928\u093e') && result.length > 4)
|
|
365
|
+
return result.slice(0, -2); // \u0928\u093e (nā) infinitive
|
|
366
|
+
if (result.endsWith('\u0928\u0947') && result.length > 4)
|
|
367
|
+
return result.slice(0, -2); // \u0928\u0947 (ne) ergative
|
|
368
|
+
if (result.endsWith('\u0928\u0940') && result.length > 4)
|
|
369
|
+
return result.slice(0, -2); // \u0928\u0940 (nī) fem infinitive
|
|
370
|
+
if (result.endsWith('\u0924\u093e') && result.length > 4)
|
|
371
|
+
return result.slice(0, -2); // \u0924\u093e (tā) m present participle
|
|
372
|
+
if (result.endsWith('\u0924\u0940') && result.length > 4)
|
|
373
|
+
return result.slice(0, -2); // \u0924\u0940 (tī) f present participle
|
|
374
|
+
if (result.endsWith('\u0915\u0930') && result.length > 4)
|
|
375
|
+
return result.slice(0, -2); // \u0915\u0930 (kar) conjunctive
|
|
376
|
+
if (result.endsWith('\u0913\u0902') && result.length > 4)
|
|
377
|
+
return result.slice(0, -2); // \u0913\u0902 (oṃ) oblique plural
|
|
378
|
+
if (result.endsWith('\u0907\u092f\u093e\u0902') && result.length > 5)
|
|
379
|
+
return result.slice(0, -4); // \u0907\u092f\u093e\u0902 (iyāṃ) f plural
|
|
380
|
+
if (result.endsWith('\u0940\u092f\u093e\u0902') && result.length > 5)
|
|
381
|
+
return result.slice(0, -4); // \u0940\u092f\u093e\u0102 (īyāṃ)
|
|
382
|
+
return result;
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Russian light stemmer — strips common inflectional suffixes.
|
|
386
|
+
* Handles the most frequent verb/noun/adjective endings to improve recall.
|
|
387
|
+
* Only strips when result is still ≥ 3 chars.
|
|
388
|
+
*
|
|
389
|
+
* Covers:
|
|
390
|
+
* ошибки/ошибка → ошибк (noun plural/genitive)
|
|
391
|
+
* развёртывание → развёртыва (gerund → stem)
|
|
392
|
+
* установить → установ (infinitive)
|
|
393
|
+
* настройки → настройк (genitive plural)
|
|
394
|
+
*/
|
|
395
|
+
function russianLightStem(word) {
|
|
396
|
+
let w = word;
|
|
397
|
+
// Longest first to avoid partial stripping
|
|
398
|
+
// Verb infinitives / gerunds
|
|
399
|
+
if (w.endsWith('ывание') && w.length > 7)
|
|
400
|
+
return w.slice(0, -6);
|
|
401
|
+
if (w.endsWith('ивание') && w.length > 7)
|
|
402
|
+
return w.slice(0, -6);
|
|
403
|
+
if (w.endsWith('ование') && w.length > 7)
|
|
404
|
+
return w.slice(0, -6);
|
|
405
|
+
if (w.endsWith('вание') && w.length > 6)
|
|
406
|
+
return w.slice(0, -5);
|
|
407
|
+
if (w.endsWith('ение') && w.length > 5)
|
|
408
|
+
return w.slice(0, -4);
|
|
409
|
+
if (w.endsWith('ание') && w.length > 5)
|
|
410
|
+
return w.slice(0, -4);
|
|
411
|
+
if (w.endsWith('ить') && w.length > 4)
|
|
412
|
+
return w.slice(0, -3);
|
|
413
|
+
if (w.endsWith('ать') && w.length > 4)
|
|
414
|
+
return w.slice(0, -3);
|
|
415
|
+
if (w.endsWith('еть') && w.length > 4)
|
|
416
|
+
return w.slice(0, -3);
|
|
417
|
+
if (w.endsWith('уть') && w.length > 4)
|
|
418
|
+
return w.slice(0, -3);
|
|
419
|
+
// Noun plural/genitive
|
|
420
|
+
if (w.endsWith('ки') && w.length > 4)
|
|
421
|
+
return w.slice(0, -2);
|
|
422
|
+
if (w.endsWith('ги') && w.length > 4)
|
|
423
|
+
return w.slice(0, -2);
|
|
424
|
+
if (w.endsWith('ов') && w.length > 4)
|
|
425
|
+
return w.slice(0, -2);
|
|
426
|
+
if (w.endsWith('ев') && w.length > 4)
|
|
427
|
+
return w.slice(0, -2);
|
|
428
|
+
if (w.endsWith('ей') && w.length > 4)
|
|
429
|
+
return w.slice(0, -2);
|
|
430
|
+
if (w.endsWith('ий') && w.length > 4)
|
|
431
|
+
return w.slice(0, -2);
|
|
432
|
+
if (w.endsWith('ый') && w.length > 4)
|
|
433
|
+
return w.slice(0, -2);
|
|
434
|
+
if (w.endsWith('ая') && w.length > 4)
|
|
435
|
+
return w.slice(0, -2);
|
|
436
|
+
if (w.endsWith('ое') && w.length > 4)
|
|
437
|
+
return w.slice(0, -2);
|
|
438
|
+
if (w.endsWith('ую') && w.length > 4)
|
|
439
|
+
return w.slice(0, -2);
|
|
440
|
+
// Verb present tense
|
|
441
|
+
if (w.endsWith('ет') && w.length > 4)
|
|
442
|
+
return w.slice(0, -2);
|
|
443
|
+
if (w.endsWith('ют') && w.length > 4)
|
|
444
|
+
return w.slice(0, -2);
|
|
445
|
+
if (w.endsWith('ут') && w.length > 4)
|
|
446
|
+
return w.slice(0, -2);
|
|
447
|
+
if (w.endsWith('ят') && w.length > 4)
|
|
448
|
+
return w.slice(0, -2);
|
|
449
|
+
return w;
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Turkish light stemmer — strips common agglutinative suffixes.
|
|
453
|
+
* Turkish is highly agglutinative; this covers the most common inflectional ends.
|
|
454
|
+
*
|
|
455
|
+
* Covers:
|
|
456
|
+
* hatalar → hata (-lar/-ler plural)
|
|
457
|
+
* sunucuda → sunucu (-da/-de locative)
|
|
458
|
+
* dağıtımı → dağıtım (-ı/-i/-u/-ü accusative)
|
|
459
|
+
* yüklemek → yükle (-mek/-mak infinitive)
|
|
460
|
+
*/
|
|
461
|
+
function turkishLightStem(word) {
|
|
462
|
+
let w = word;
|
|
463
|
+
// Suffixes longest-first
|
|
464
|
+
if (w.endsWith('lardaki') && w.length > 7)
|
|
465
|
+
return w.slice(0, -7);
|
|
466
|
+
if (w.endsWith('lerdeki') && w.length > 7)
|
|
467
|
+
return w.slice(0, -7);
|
|
468
|
+
if (w.endsWith('ların') && w.length > 6)
|
|
469
|
+
return w.slice(0, -5);
|
|
470
|
+
if (w.endsWith('lerin') && w.length > 6)
|
|
471
|
+
return w.slice(0, -5);
|
|
472
|
+
if (w.endsWith('larda') && w.length > 6)
|
|
473
|
+
return w.slice(0, -5);
|
|
474
|
+
if (w.endsWith('lerde') && w.length > 6)
|
|
475
|
+
return w.slice(0, -5);
|
|
476
|
+
if (w.endsWith('ları') && w.length > 5)
|
|
477
|
+
return w.slice(0, -4);
|
|
478
|
+
if (w.endsWith('leri') && w.length > 5)
|
|
479
|
+
return w.slice(0, -4);
|
|
480
|
+
if (w.endsWith('mek') && w.length > 4)
|
|
481
|
+
return w.slice(0, -3);
|
|
482
|
+
if (w.endsWith('mak') && w.length > 4)
|
|
483
|
+
return w.slice(0, -3);
|
|
484
|
+
if (w.endsWith('ler') && w.length > 4)
|
|
485
|
+
return w.slice(0, -3);
|
|
486
|
+
if (w.endsWith('lar') && w.length > 4)
|
|
487
|
+
return w.slice(0, -3);
|
|
488
|
+
if (w.endsWith('nın') && w.length > 4)
|
|
489
|
+
return w.slice(0, -3);
|
|
490
|
+
if (w.endsWith('nin') && w.length > 4)
|
|
491
|
+
return w.slice(0, -3);
|
|
492
|
+
if (w.endsWith('nun') && w.length > 4)
|
|
493
|
+
return w.slice(0, -3);
|
|
494
|
+
if (w.endsWith('nün') && w.length > 4)
|
|
495
|
+
return w.slice(0, -3);
|
|
496
|
+
if (w.endsWith('da') && w.length > 4)
|
|
497
|
+
return w.slice(0, -2);
|
|
498
|
+
if (w.endsWith('de') && w.length > 4)
|
|
499
|
+
return w.slice(0, -2);
|
|
500
|
+
if (w.endsWith('ta') && w.length > 4)
|
|
501
|
+
return w.slice(0, -2);
|
|
502
|
+
if (w.endsWith('te') && w.length > 4)
|
|
503
|
+
return w.slice(0, -2);
|
|
504
|
+
if (w.endsWith('dan') && w.length > 4)
|
|
505
|
+
return w.slice(0, -3);
|
|
506
|
+
if (w.endsWith('den') && w.length > 4)
|
|
507
|
+
return w.slice(0, -3);
|
|
508
|
+
if (w.endsWith('tan') && w.length > 4)
|
|
509
|
+
return w.slice(0, -3);
|
|
510
|
+
if (w.endsWith('ten') && w.length > 4)
|
|
511
|
+
return w.slice(0, -3);
|
|
512
|
+
if (w.endsWith('ın') && w.length > 4)
|
|
513
|
+
return w.slice(0, -2);
|
|
514
|
+
if (w.endsWith('in') && w.length > 4)
|
|
515
|
+
return w.slice(0, -2);
|
|
516
|
+
if (w.endsWith('un') && w.length > 4)
|
|
517
|
+
return w.slice(0, -2);
|
|
518
|
+
if (w.endsWith('ün') && w.length > 4)
|
|
519
|
+
return w.slice(0, -2);
|
|
520
|
+
if (w.endsWith('ı') && w.length > 4)
|
|
521
|
+
return w.slice(0, -1);
|
|
522
|
+
if (w.endsWith('i') && w.length > 4)
|
|
523
|
+
return w.slice(0, -1);
|
|
524
|
+
if (w.endsWith('u') && w.length > 4)
|
|
525
|
+
return w.slice(0, -1);
|
|
526
|
+
if (w.endsWith('ü') && w.length > 4)
|
|
527
|
+
return w.slice(0, -1);
|
|
528
|
+
return w;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Bengali (Bangla) light stemmer — strips common verbal and nominal suffixes.
|
|
532
|
+
* Bengali is an Indo-Aryan language with moderate morphological complexity.
|
|
533
|
+
* Only strips when result is still ≥ 2 chars.
|
|
534
|
+
*
|
|
535
|
+
* Covers:
|
|
536
|
+
* করছে/করেছে → কর (progressive/perfect aspect)
|
|
537
|
+
* সার্ভারগুলো → সার্ভার (-গুলো plural)
|
|
538
|
+
* সমস্যাটি → সমস্যা (-টি singular marker)
|
|
539
|
+
*/
|
|
540
|
+
function bengaliLightStem(word) {
|
|
541
|
+
let w = word;
|
|
542
|
+
// Plural / collective markers (longest first)
|
|
543
|
+
if (w.endsWith('\u0997\u09c1\u09b2\u09cb') && w.length > 5)
|
|
544
|
+
return w.slice(0, -4); // গুলো (-gulo plural)
|
|
545
|
+
if (w.endsWith('\u0997\u09c1\u09b2\u09bf') && w.length > 5)
|
|
546
|
+
return w.slice(0, -4); // গুলি (-guli plural)
|
|
547
|
+
if (w.endsWith('\u09a6\u09c7\u09b0') && w.length > 4)
|
|
548
|
+
return w.slice(0, -3); // দের (genitive plural)
|
|
549
|
+
if (w.endsWith('\u09a6\u09bf\u0997\u09c7') && w.length > 5)
|
|
550
|
+
return w.slice(0, -4); // দিগে
|
|
551
|
+
// Definiteness / case markers
|
|
552
|
+
if (w.endsWith('\u099f\u09bf') && w.length > 3)
|
|
553
|
+
return w.slice(0, -2); // টি (singular definite)
|
|
554
|
+
if (w.endsWith('\u099f\u09be') && w.length > 3)
|
|
555
|
+
return w.slice(0, -2); // টা (singular definite)
|
|
556
|
+
if (w.endsWith('\u0996\u09be\u09a8\u09be') && w.length > 5)
|
|
557
|
+
return w.slice(0, -4); // খানা
|
|
558
|
+
// Verbal suffixes
|
|
559
|
+
if (w.endsWith('\u099b\u09c7') && w.length > 3)
|
|
560
|
+
return w.slice(0, -2); // ছে (progressive)
|
|
561
|
+
if (w.endsWith('\u099b\u09bf\u09b2') && w.length > 4)
|
|
562
|
+
return w.slice(0, -3); // ছিল (past progressive)
|
|
563
|
+
if (w.endsWith('\u09af\u09be\u09ac\u09c7') && w.length > 5)
|
|
564
|
+
return w.slice(0, -4); // যাবে (future)
|
|
565
|
+
if (w.endsWith('\u0995\u09b0\u09be') && w.length > 4)
|
|
566
|
+
return w.slice(0, -3); // করা (infinitive)
|
|
567
|
+
if (w.endsWith('\u0995\u09b0\u09c7') && w.length > 4)
|
|
568
|
+
return w.slice(0, -3); // করে (present)
|
|
569
|
+
if (w.endsWith('\u09b9\u09df') && w.length > 3)
|
|
570
|
+
return w.slice(0, -2); // হয় (is/becomes)
|
|
571
|
+
if (w.endsWith('\u09b9\u09ac\u09c7') && w.length > 4)
|
|
572
|
+
return w.slice(0, -3); // হবে (will be)
|
|
573
|
+
return w;
|
|
574
|
+
}
|
|
575
|
+
/**
|
|
576
|
+
* Pre-process text BEFORE tokenization to split code identifiers.
|
|
577
|
+
* Handles camelCase, PascalCase, snake_case, and kebab-case.
|
|
578
|
+
*
|
|
579
|
+
* Examples:
|
|
580
|
+
* deployServer → deploy Server (then lowercased → deploy server)
|
|
581
|
+
* HTMLParser → HTML Parser
|
|
582
|
+
* my_api_key → my api key
|
|
583
|
+
* get-user-by-id → get user by id (hyphens preserved in regex, spaces also fine)
|
|
584
|
+
* BackendServiceImpl → Backend Service Impl
|
|
585
|
+
*/
|
|
586
|
+
function preprocessText(text) {
|
|
587
|
+
return text
|
|
588
|
+
// camelCase boundary: lowercase/digit → uppercase
|
|
589
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
590
|
+
// Consecutive uppercase → uppercase+lowercase boundary: HTMLParser → HTML Parser
|
|
591
|
+
.replace(/([A-Z]{2,})([A-Z][a-z])/g, '$1 $2')
|
|
592
|
+
// snake_case: replace underscores with spaces
|
|
593
|
+
.replace(/_/g, ' ');
|
|
594
|
+
}
|
|
595
|
+
// Maps tokens (English, Japanese katakana, Chinese, Korean, Arabic, Hebrew)
|
|
596
|
+
// to their equivalents in other languages. Used to expand tokens at search time
|
|
597
|
+
// so "deploy" finds documents containing "デプロイ" / "部署" / "배포", and vice versa.
|
|
598
|
+
//
|
|
599
|
+
// Keys must be lowercase. CJK values will be bigram-expanded at use time.
|
|
600
|
+
const CROSS_LINGUAL_MAP = new Map([
|
|
601
|
+
// ── Tech / Ecosystem synonyms (language-agnostic) ─────────────────────────
|
|
602
|
+
['javascript', ['js', 'node', 'nodejs', 'ecmascript']],
|
|
603
|
+
['js', ['javascript', 'node', 'nodejs']],
|
|
604
|
+
['typescript', ['ts', 'javascript', 'js', 'tsc']],
|
|
605
|
+
['ts', ['typescript', 'javascript', 'js']],
|
|
606
|
+
['python', ['py', 'pip', 'django', 'flask', 'fastapi']],
|
|
607
|
+
['py', ['python', 'pip']],
|
|
608
|
+
['golang', ['go', 'gopher', 'goroutine']],
|
|
609
|
+
['go', ['golang', 'goroutine']],
|
|
610
|
+
['rust', ['cargo', 'rustlang', 'crates']],
|
|
611
|
+
['cargo', ['rust', 'rustlang']],
|
|
612
|
+
['java', ['jvm', 'maven', 'gradle', 'spring']],
|
|
613
|
+
['kotlin', ['jvm', 'android', 'coroutine']],
|
|
614
|
+
['dotnet', ['csharp', 'asp', 'nuget', 'aspnet']],
|
|
615
|
+
['postgresql', ['postgres', 'pg', 'psql']],
|
|
616
|
+
['postgres', ['postgresql', 'pg', 'psql']],
|
|
617
|
+
['mysql', ['mariadb', 'sql']],
|
|
618
|
+
['mongodb', ['mongo', 'nosql', 'bson', 'mongoose']],
|
|
619
|
+
['mongo', ['mongodb', 'nosql', 'bson']],
|
|
620
|
+
['elasticsearch', ['elastic', 'opensearch', 'lucene', 'kibana']],
|
|
621
|
+
['elastic', ['elasticsearch', 'opensearch']],
|
|
622
|
+
['kubernetes', ['k8s', 'kube', 'kubectl', 'helm', 'k3s']],
|
|
623
|
+
['k8s', ['kubernetes', 'kube', 'kubectl', 'helm']],
|
|
624
|
+
['helm', ['kubernetes', 'k8s', 'chart']],
|
|
625
|
+
['terraform', ['tf', 'hcl', 'iac', 'opentofu']],
|
|
626
|
+
['ansible', ['playbook', 'automation']],
|
|
627
|
+
['grafana', ['dashboard', 'visualization', 'metrics']],
|
|
628
|
+
['prometheus', ['metrics', 'alert', 'scrape', 'grafana']],
|
|
629
|
+
['nginx', ['proxy', 'webserver', 'ingress']],
|
|
630
|
+
['traefik', ['proxy', 'ingress', 'router']],
|
|
631
|
+
['rabbitmq', ['amqp', 'queue', 'broker', 'messaging']],
|
|
632
|
+
['kafka', ['messaging', 'stream', 'broker']],
|
|
633
|
+
['grpc', ['protobuf', 'proto', 'rpc']],
|
|
634
|
+
['graphql', ['gql', 'resolver', 'schema']],
|
|
635
|
+
['react', ['jsx', 'hooks', 'component', 'redux']],
|
|
636
|
+
['nextjs', ['next', 'react', 'ssr', 'vercel']],
|
|
637
|
+
['vue', ['vuejs', 'vite', 'nuxt']],
|
|
638
|
+
['angular', ['ng', 'rxjs', 'typescript']],
|
|
639
|
+
['aws', ['amazon', 'ec2', 's3', 'lambda', 'cloudwatch', 'ecs', 'eks']],
|
|
640
|
+
['gcp', ['google cloud', 'gke', 'bigquery']],
|
|
641
|
+
['azure', ['microsoft cloud', 'aks', 'devops']],
|
|
642
|
+
['s3', ['bucket', 'object storage', 'aws']],
|
|
643
|
+
['lambda', ['serverless', 'function', 'faas', 'aws']],
|
|
644
|
+
['jwt', ['token', 'bearer', 'oauth', 'auth']],
|
|
645
|
+
['oauth', ['oidc', 'auth', 'token', 'sso']],
|
|
646
|
+
['webhook', ['callback', 'event', 'trigger']],
|
|
647
|
+
['cron', ['schedule', 'job', 'timer']],
|
|
648
|
+
['yaml', ['yml', 'config', 'manifest']],
|
|
649
|
+
['dotenv', ['env', 'environment', 'envfile']],
|
|
650
|
+
['github', ['git', 'actions', 'repo', 'ci']],
|
|
651
|
+
['gitlab', ['git', 'pipeline', 'runner']],
|
|
652
|
+
['wireguard', ['vpn', 'tunnel', 'wg']],
|
|
653
|
+
['prisma', ['orm', 'database', 'schema', 'migration']],
|
|
654
|
+
['npm', ['node', 'package', 'registry', 'yarn', 'pnpm']],
|
|
655
|
+
['yarn', ['npm', 'package', 'node']],
|
|
656
|
+
['pip', ['python', 'package', 'pypi']],
|
|
657
|
+
['brew', ['homebrew', 'macos', 'package']],
|
|
658
|
+
// English → all
|
|
659
|
+
['deploy', ['デプロイ', '部署', '배포', 'deployment', 'deploying', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellen', 'bereitstellung', 'déployer', 'déploiement', 'desplegar', 'despliegue', 'distribuire', 'distribuzione', 'implantar', 'implantação', 'развёртывание', 'развертывание', 'dağıtım', 'dağıtmak', 'triển khai']],
|
|
660
|
+
['deployment', ['デプロイ', '部署', '배포', 'deploy', 'نشر', 'פריסה', 'استقرار', 'तैनाती', 'bereitstellung', 'déploiement', 'despliegue', 'distribuzione', 'implantação', 'развёртывание', 'dağıtım', 'triển khai']],
|
|
661
|
+
['container', ['コンテナ', '容器', '컨테이너', 'docker']],
|
|
662
|
+
['server', ['サーバー', 'サーバ', '服务器', '서버', 'سيرفر', 'שרת', 'سرور', 'सर्वर', 'сервер', 'sunucu', 'máy chủ']],
|
|
663
|
+
['database', ['データベース', '数据库', '데이터베이스', 'db', 'قاعدة البيانات', 'datenbank', 'پایگاه داده', 'डेटाबेس', 'base de données', 'base de datos', 'banco de dados']],
|
|
664
|
+
['cache', ['キャッシュ', '缓存', '캐시', 'كاش', 'מטמון', 'کش', 'कैश', 'кэш', 'önbellek', 'bộ đệm']],
|
|
665
|
+
['error', ['エラー', '错误', '오류', 'خطأ', 'שגיאה', 'خطا', 'त्रुटि', 'गलती', 'exception', 'err', 'fehler', 'erreur', 'fallo', 'errore', 'erro', 'ошибка', 'hata', 'hatalı', 'lỗi']],
|
|
666
|
+
['bug', ['バグ', '缺陷', '버그', 'issue', 'defect', 'خلل', 'اشکال', 'बग']],
|
|
667
|
+
['fix', ['修正', '修复', '수정', 'bugfix', 'patch', 'hotfix', 'إصلاح', 'תיקון', 'رفع', 'सुधार', 'beheben', 'behoben', 'réparer', 'correction', 'arreglar', 'corrección', 'correggere', 'corrigir', 'исправить', 'исправление', 'düzeltmek', 'düzeltme']],
|
|
668
|
+
['build', ['ビルド', '构建', '빌드', 'بناء', 'בנייה', 'ساخت', 'निर्माण', 'bauen', 'construire', 'construir', 'costruire', 'сборка', 'собрать', 'derleme', 'derlemek']],
|
|
669
|
+
['test', ['テスト', '测试', '테스트', 'اختبار', 'בדיקה', 'آزمایش', 'परीक्षण', 'testen', 'tester', 'probar', 'testare', 'testar', 'тест', 'тестирование', 'test', 'testlemek']],
|
|
670
|
+
['auth', ['認証', '认证', '인증', 'authentication', 'login', 'oauth', 'مصادقة', 'אימות', 'احراز هویت', 'प्रमाणीकरण', 'authentifizierung', 'authentification', 'autenticación', 'autenticazione', 'autenticação', 'аутентификация', 'авторизация', 'kimlik doğrulama', 'yetkilendirme']],
|
|
671
|
+
['authentication', ['auth', 'مصادقة', 'אימות', 'احراز هویت', 'प्रमाणीकरण', '認証', '认证', '인증', 'login', 'oauth', 'аутентификация', 'авторизация', 'kimlik doğrulama']],
|
|
672
|
+
['environment', ['環境', '环境', '환경', 'env', 'بيئة', 'סביבה', 'umgebung', 'environnement', 'entorno', 'ambiente', 'среда', 'окружение', 'ortam']],
|
|
673
|
+
['secret', ['key', 'token', 'password', 'jwt', 'مفتاح', 'מפתח', 'geheimnis', 'schlüssel', 'clave', 'clé', 'chiave', 'chave', 'секрет', 'ключ', 'gizli', 'anahtar']],
|
|
674
|
+
['problem', ['error', 'issue', 'bug', 'failure', 'مشكلة', '問題', '오류', 'fehler', 'problème', 'problema', 'проблема', 'ошибка', 'sorun', 'problem']],
|
|
675
|
+
['kubernetes', ['クベルネテス', 'k8s', 'kube']],
|
|
676
|
+
['network', ['ネットワーク', '网络', '네트워크', 'شبكة', 'רשת', 'شبکه', 'नेटवर्क', 'netzwerk', 'réseau', 'red', 'rete', 'rede', 'сеть', 'ağ']],
|
|
677
|
+
['timeout', ['タイムアウト', '超时', '타임아웃', 'مهلة']],
|
|
678
|
+
['memory', ['メモリ', '内存', '메모리', 'ram', 'ذاكرة', 'זיכרון', 'حافظه', 'मेमोरी', 'speicher', 'mémoire', 'memoria', 'memória', 'память', 'bellek', 'hafıza']],
|
|
679
|
+
['config', ['設定', '配置', '설정', 'configuration', 'settings', 'conf', 'إعدادات', 'הגדרות', 'پیکربندی', 'कॉन्फ़िग', 'konfiguration', 'einstellungen', 'configuración', 'configurazione', 'configuração', 'конфигурация', 'настройки', 'yapılandırma', 'ayarlar']],
|
|
680
|
+
['install', ['インストール', '安装', '설치', 'تثبيت', 'התקנה', 'نصب', 'इंस्टॉल', 'installieren', 'installer', 'instalar', 'installare', 'установить', 'установка', 'yüklemek', 'kurulum']],
|
|
681
|
+
['update', ['アップデート', '更新', '업데이트', 'upgrade', 'تحديث', 'עדכון', 'بهروزرسانی', 'अपडेट', 'aktualisieren', 'aktualisierung', 'actualizar', 'aggiornare', 'atualizar', 'обновление', 'обновить', 'güncelleme']],
|
|
682
|
+
['log', ['ログ', '日志', '로그', 'logging', 'سجل', 'יומן', 'ثبت', 'लॉग', 'protokoll', 'protokolle', 'journal', 'registro', 'журнал', 'лог', 'günlük']],
|
|
683
|
+
['port', ['ポート', '端口', '포트', 'منفذ', 'פורט']],
|
|
684
|
+
['file', ['ファイル', '文件', '파일', 'ملف', 'קובץ']],
|
|
685
|
+
['image', ['イメージ', '镜像', '이미지', 'صورة', 'תמונה', 'abbild', 'imagen', 'imagem']],
|
|
686
|
+
['volume', ['ボリューム', '卷', '볼륨']],
|
|
687
|
+
['cluster', ['クラスター', '集群', '클러스터']],
|
|
688
|
+
['node', ['ノード', '节点', '노드']],
|
|
689
|
+
['service', ['サービス', '服务', '서비스', 'svc', 'خدمة', 'שירות', 'dienst', 'servicio', 'servizio', 'serviço', 'сервис', 'служба', 'servis', 'hizmet']],
|
|
690
|
+
['certificate', ['証明書', '证书', '인증서', 'cert', 'ssl', 'tls', 'شهادة', 'תעודה']],
|
|
691
|
+
['password', ['パスワード', '密码', '비밀번호', 'passwd', 'pwd', 'كلمة المرور', 'סיסמה']],
|
|
692
|
+
['token', ['トークン', '令牌', '토큰', 'jwt', 'secret', 'رمز', 'אסימון']],
|
|
693
|
+
['health', ['ヘルス', '健康', '헬스', 'healthcheck', 'probe', 'صحة']],
|
|
694
|
+
['migration', ['マイグレーション', '迁移', '마이그레이션', 'migrate', 'هجرة']],
|
|
695
|
+
['backup', ['バックアップ', '备份', '백업', 'نسخ احتياطي', 'גיבוי', 'پشتیبانگیری', 'बैकअप', 'sicherung', 'sauvegarde', 'respaldo', 'cópia de segurança', 'резервная копия', 'резерв', 'yedekleme']],
|
|
696
|
+
['monitor', ['モニター', '监控', '모니터링', 'monitoring', 'metrics', 'alert', 'مراقبة', 'ניטור', 'نظارت', 'निगरानी', 'überwachung', 'surveiller', 'monitoreo', 'monitorare', 'monitorar', 'мониторинг', 'наблюдение', 'izleme']],
|
|
697
|
+
['performance', ['パフォーマンス', '性能', '성능', 'latency', 'throughput', 'عملکرد', 'प्रदर्शन', 'leistung', 'performances', 'rendimiento', 'prestazioni', 'desempenho', 'производительность', 'быстродействие', 'performans']],
|
|
698
|
+
['connection', ['接続', '连接', '연결', 'conn', 'socket', 'اتصال', 'חיבור', 'اتصال', 'कनेक्शन', 'verbindung', 'connexion', 'conexión', 'connessione', 'conexão']],
|
|
699
|
+
['queue', ['キュー', '队列', '큐', 'طابور', 'warteschlange', 'file d\'attente', 'cola', 'coda', 'fila']],
|
|
700
|
+
['redis', ['レディス', '레디스']],
|
|
701
|
+
['nginx', ['エンジンエックス']],
|
|
702
|
+
['linux', ['リナックス', 'لينكس']],
|
|
703
|
+
['api', ['エーピーアイ', '接口', 'endpoint', 'واجهة برمجية', 'ממשק']],
|
|
704
|
+
['healthcheck', ['ヘルスチェック', 'health', 'probe', 'فحص الصحة']],
|
|
705
|
+
['loadbalancer', ['ロードバランサー', '负载均衡', '로드밸런서', 'lb']],
|
|
706
|
+
['ssl', ['tls', 'https', 'certificate', 'cert']],
|
|
707
|
+
['tls', ['ssl', 'https', 'certificate', 'cert']],
|
|
708
|
+
['docker', ['container', 'コンテナ', '容器', '컨테이너', 'دوكر']],
|
|
709
|
+
['git', ['version control', 'repo', 'repository', 'commit', 'push', 'pull']],
|
|
710
|
+
['ci', ['pipeline', 'github actions', 'gitlab', 'jenkins', 'build']],
|
|
711
|
+
['cd', ['deploy', 'deployment', 'release']],
|
|
712
|
+
['debug', ['デバッグ', '调试', '디버그', 'debugging', 'breakpoint', 'تصحيح', 'איתור באגים', 'اشکالزدایی', 'डीबग', 'debuggen', 'déboguer', 'depurar', 'отладка', 'дебаг', 'hata ayıklama']],
|
|
713
|
+
['crash', ['クラッシュ', '崩溃', '크래시', 'panic', 'segfault', 'انهيار', 'קריסה', 'خرابی', 'क्रैश', 'absturz', 'panne', 'plantage', 'caída', 'falha', 'сбой', 'крэш', 'çökme', 'çöküş']],
|
|
714
|
+
['restart', ['再起動', '重启', '재시작', 'reboot', 'إعادة تشغيل', 'הפעלה מחדש', 'راهاندازی مجدد', 'पुنरारंभ', 'neustart', 'redémarrer', 'reiniciar', 'riavviare']],
|
|
715
|
+
['permission', ['権限', '权限', '권한', 'access', 'acl', 'chmod', 'صلاحية', 'הרשאה', 'مجوز', 'अनुमति', 'berechtigung', 'berechtigungen', 'permiso', 'permesso', 'permissão', 'разрешение', 'доступ', 'izin', 'yetki']],
|
|
716
|
+
// Japanese katakana → English
|
|
717
|
+
['デプロイ', ['deploy', 'deployment']],
|
|
718
|
+
['コンテナ', ['container', 'docker']],
|
|
719
|
+
['サーバー', ['server']],
|
|
720
|
+
['サーバ', ['server']],
|
|
721
|
+
['データベース', ['database', 'db']],
|
|
722
|
+
['キャッシュ', ['cache']],
|
|
723
|
+
['エラー', ['error', 'err']],
|
|
724
|
+
['バグ', ['bug', 'issue']],
|
|
725
|
+
['ビルド', ['build']],
|
|
726
|
+
['テスト', ['test']],
|
|
727
|
+
['認証', ['auth', 'authentication']],
|
|
728
|
+
['設定', ['config', 'configuration']],
|
|
729
|
+
['インストール', ['install']],
|
|
730
|
+
['アップデート', ['update', 'upgrade']],
|
|
731
|
+
['ログ', ['log', 'logs']],
|
|
732
|
+
['ポート', ['port']],
|
|
733
|
+
['ファイル', ['file']],
|
|
734
|
+
['イメージ', ['image']],
|
|
735
|
+
['クラスター', ['cluster']],
|
|
736
|
+
['ノード', ['node']],
|
|
737
|
+
['サービス', ['service']],
|
|
738
|
+
['パスワード', ['password']],
|
|
739
|
+
['トークン', ['token']],
|
|
740
|
+
['バックアップ', ['backup']],
|
|
741
|
+
['モニター', ['monitor', 'monitoring']],
|
|
742
|
+
['接続', ['connection']],
|
|
743
|
+
['キュー', ['queue']],
|
|
744
|
+
['ヘルスチェック', ['healthcheck', 'health check', 'health']],
|
|
745
|
+
['ロードバランサー', ['loadbalancer', 'load balancer']],
|
|
746
|
+
['デバッグ', ['debug']],
|
|
747
|
+
['クラッシュ', ['crash', 'panic']],
|
|
748
|
+
['再起動', ['restart', 'reboot']],
|
|
749
|
+
['権限', ['permission', 'access']],
|
|
750
|
+
// Chinese → English
|
|
751
|
+
['部署', ['deploy', 'deployment']],
|
|
752
|
+
['容器', ['container', 'docker']],
|
|
753
|
+
['服务器', ['server']],
|
|
754
|
+
['数据库', ['database']],
|
|
755
|
+
['缓存', ['cache']],
|
|
756
|
+
['错误', ['error']],
|
|
757
|
+
['修复', ['fix', 'bugfix']],
|
|
758
|
+
['构建', ['build']],
|
|
759
|
+
['测试', ['test']],
|
|
760
|
+
['认证', ['auth']],
|
|
761
|
+
['配置', ['config']],
|
|
762
|
+
['安装', ['install']],
|
|
763
|
+
['更新', ['update']],
|
|
764
|
+
['日志', ['log']],
|
|
765
|
+
['端口', ['port']],
|
|
766
|
+
['镜像', ['image']],
|
|
767
|
+
['集群', ['cluster']],
|
|
768
|
+
['节点', ['node']],
|
|
769
|
+
['服务', ['service']],
|
|
770
|
+
['密码', ['password']],
|
|
771
|
+
['令牌', ['token']],
|
|
772
|
+
['备份', ['backup']],
|
|
773
|
+
['监控', ['monitor']],
|
|
774
|
+
['连接', ['connection']],
|
|
775
|
+
['队列', ['queue']],
|
|
776
|
+
['性能', ['performance']],
|
|
777
|
+
['内存', ['memory']],
|
|
778
|
+
['调试', ['debug']],
|
|
779
|
+
['崩溃', ['crash']],
|
|
780
|
+
['重启', ['restart']],
|
|
781
|
+
['权限', ['permission']],
|
|
782
|
+
// Korean → English
|
|
783
|
+
['배포', ['deploy', 'deployment']],
|
|
784
|
+
['컨테이너', ['container']],
|
|
785
|
+
['서버', ['server']],
|
|
786
|
+
['데이터베이스', ['database']],
|
|
787
|
+
['캐시', ['cache']],
|
|
788
|
+
['오류', ['error']],
|
|
789
|
+
['수정', ['fix']],
|
|
790
|
+
['빌드', ['build']],
|
|
791
|
+
['테스트', ['test']],
|
|
792
|
+
['인증', ['auth']],
|
|
793
|
+
['설정', ['config']],
|
|
794
|
+
['설치', ['install']],
|
|
795
|
+
['업데이트', ['update']],
|
|
796
|
+
['로그', ['log']],
|
|
797
|
+
['포트', ['port']],
|
|
798
|
+
['이미지', ['image']],
|
|
799
|
+
['클러스터', ['cluster']],
|
|
800
|
+
['노드', ['node']],
|
|
801
|
+
['서비스', ['service']],
|
|
802
|
+
['토큰', ['token']],
|
|
803
|
+
['백업', ['backup']],
|
|
804
|
+
['모니터링', ['monitor']],
|
|
805
|
+
['연결', ['connection']],
|
|
806
|
+
['큐', ['queue']],
|
|
807
|
+
['성능', ['performance']],
|
|
808
|
+
['메모리', ['memory']],
|
|
809
|
+
['디버그', ['debug']],
|
|
810
|
+
['크래시', ['crash']],
|
|
811
|
+
['재시작', ['restart']],
|
|
812
|
+
['권한', ['permission']],
|
|
813
|
+
// Arabic → English
|
|
814
|
+
['خطأ', ['error']],
|
|
815
|
+
['إصلاح', ['fix']],
|
|
816
|
+
['بناء', ['build']],
|
|
817
|
+
['اختبار', ['test']],
|
|
818
|
+
['مصادقة', ['auth', 'authentication']],
|
|
819
|
+
['إعدادات', ['config', 'configuration']],
|
|
820
|
+
['تثبيت', ['install']],
|
|
821
|
+
['تحديث', ['update', 'upgrade']],
|
|
822
|
+
['سجل', ['log']],
|
|
823
|
+
['صورة', ['image']],
|
|
824
|
+
['خدمة', ['service']],
|
|
825
|
+
['نسخ احتياطي', ['backup']],
|
|
826
|
+
['مراقبة', ['monitor', 'monitoring']],
|
|
827
|
+
['اتصال', ['connection']],
|
|
828
|
+
['ذاكرة', ['memory']],
|
|
829
|
+
['تصحيح', ['debug', 'debugging']],
|
|
830
|
+
['شبكة', ['network']],
|
|
831
|
+
// Additional Arabic → English (covering blog post examples + common terms)
|
|
832
|
+
['نشر', ['deploy', 'deployment']],
|
|
833
|
+
['انهيار', ['crash', 'panic']],
|
|
834
|
+
['طابور', ['queue']],
|
|
835
|
+
['رمز', ['token', 'secret']],
|
|
836
|
+
['كاش', ['cache']],
|
|
837
|
+
['منفذ', ['port']],
|
|
838
|
+
['سيرفر', ['server']],
|
|
839
|
+
['قاعدة البيانات', ['database', 'db']],
|
|
840
|
+
['إعادة تشغيل', ['restart', 'reboot']],
|
|
841
|
+
['صلاحية', ['permission', 'access']],
|
|
842
|
+
['تطبيق', ['application', 'app']],
|
|
843
|
+
['مفتاح', ['key', 'token', 'secret']],
|
|
844
|
+
['واجهة برمجية', ['api', 'interface']],
|
|
845
|
+
['فحص الصحة', ['healthcheck', 'health']],
|
|
846
|
+
['حاوية', ['container', 'docker']],
|
|
847
|
+
['عقدة', ['node']],
|
|
848
|
+
['سرعة', ['performance', 'speed']],
|
|
849
|
+
['قرص', ['disk', 'storage']],
|
|
850
|
+
['خادم', ['server']],
|
|
851
|
+
['برنامج', ['application', 'software', 'app']],
|
|
852
|
+
// Arabic terms from blog examples
|
|
853
|
+
['مشكلة', ['error', 'issue', 'bug', 'problem', 'failure']],
|
|
854
|
+
['بيئة', ['environment', 'env']],
|
|
855
|
+
['متغيرات', ['variables', 'env', 'environment']],
|
|
856
|
+
['إنتاج', ['production', 'prod']],
|
|
857
|
+
['سري', ['secret', 'key', 'token']],
|
|
858
|
+
['مفتاح سري', ['secret', 'key']],
|
|
859
|
+
// Hebrew → English
|
|
860
|
+
['שגיאה', ['error']],
|
|
861
|
+
['תיקון', ['fix']],
|
|
862
|
+
['בנייה', ['build']],
|
|
863
|
+
['בדיקה', ['test']],
|
|
864
|
+
['אימות', ['auth', 'authentication']],
|
|
865
|
+
['הגדרות', ['config', 'configuration']],
|
|
866
|
+
['התקנה', ['install']],
|
|
867
|
+
['עדכון', ['update', 'upgrade']],
|
|
868
|
+
['יומן', ['log']],
|
|
869
|
+
['שרת', ['server']],
|
|
870
|
+
['שירות', ['service']],
|
|
871
|
+
['גיבוי', ['backup']],
|
|
872
|
+
['ניטור', ['monitor', 'monitoring']],
|
|
873
|
+
['חיבור', ['connection']],
|
|
874
|
+
['זיכרון', ['memory']],
|
|
875
|
+
['רשת', ['network']],
|
|
876
|
+
['תמונה', ['image']],
|
|
877
|
+
['סיסמה', ['password']],
|
|
878
|
+
['הרשאה', ['permission', 'access']],
|
|
879
|
+
// Additional Hebrew → English (covering blog post examples + common terms)
|
|
880
|
+
['קריסה', ['crash', 'panic']],
|
|
881
|
+
['תור', ['queue']],
|
|
882
|
+
['מטמון', ['cache']],
|
|
883
|
+
['אסימון', ['token', 'secret']],
|
|
884
|
+
['פורט', ['port']],
|
|
885
|
+
['איתור באגים', ['debug', 'debugging']],
|
|
886
|
+
['הפעלה מחדש', ['restart', 'reboot']],
|
|
887
|
+
['מיכל', ['container', 'docker']],
|
|
888
|
+
['יישום', ['application', 'app']],
|
|
889
|
+
['מפתח', ['key', 'token', 'secret']],
|
|
890
|
+
['בדיקת תקינות', ['healthcheck', 'health']],
|
|
891
|
+
['ממשק תכנות', ['api', 'interface']],
|
|
892
|
+
['ביצועים', ['performance']],
|
|
893
|
+
['אחסון', ['storage', 'disk']],
|
|
894
|
+
// Hebrew terms from blog examples
|
|
895
|
+
['פריסה', ['deploy', 'deployment']],
|
|
896
|
+
['בעיה', ['error', 'issue', 'bug', 'problem']],
|
|
897
|
+
['סביבה', ['environment', 'env']],
|
|
898
|
+
['ייצור', ['production', 'prod']],
|
|
899
|
+
['סודי', ['secret', 'key', 'token']],
|
|
900
|
+
['מפתח סודי', ['secret', 'key']],
|
|
901
|
+
// ── Farsi/Persian → English ───────────────────────────────────────────────
|
|
902
|
+
['استقرار', ['deploy', 'deployment']],
|
|
903
|
+
['خطا', ['error', 'err']], // Farsi: خطا vs Arabic: خطأ
|
|
904
|
+
['اشکال', ['bug', 'issue', 'error']],
|
|
905
|
+
['رفع اشکال', ['fix', 'debug', 'debugging']],
|
|
906
|
+
['ساخت', ['build']],
|
|
907
|
+
['آزمایش', ['test']],
|
|
908
|
+
['احراز هویت', ['auth', 'authentication']],
|
|
909
|
+
['پیکربندی', ['config', 'configuration']],
|
|
910
|
+
['نصب', ['install']],
|
|
911
|
+
['بهروزرسانی', ['update', 'upgrade']],
|
|
912
|
+
['ثبت', ['log']],
|
|
913
|
+
['سرور', ['server']],
|
|
914
|
+
['پایگاه داده', ['database', 'db']],
|
|
915
|
+
['شبکه', ['network']], // Farsi شبکه vs Arabic شبكة
|
|
916
|
+
['حافظه', ['memory', 'ram']], // Farsi حافظه vs Arabic ذاكرة
|
|
917
|
+
['کش', ['cache']],
|
|
918
|
+
['اشکالزدایی', ['debug', 'debugging']],
|
|
919
|
+
['خرابی', ['crash', 'failure', 'error']],
|
|
920
|
+
['راهاندازی مجدد', ['restart', 'reboot']],
|
|
921
|
+
['مجوز', ['permission', 'access']],
|
|
922
|
+
['پشتیبانگیری', ['backup']],
|
|
923
|
+
['عملکرد', ['performance']],
|
|
924
|
+
['مشکل', ['problem', 'issue', 'error']],
|
|
925
|
+
['سرویس', ['service']],
|
|
926
|
+
['کلید', ['key', 'token', 'secret']],
|
|
927
|
+
['محیط', ['environment', 'env']],
|
|
928
|
+
['تولید', ['production', 'prod']],
|
|
929
|
+
['رمز', ['secret', 'key', 'token']],
|
|
930
|
+
['اتصال', ['connection', 'conn']],
|
|
931
|
+
// ── Hindi (Devanagari) → English ──────────────────────────────────────────
|
|
932
|
+
['तैनाती', ['deploy', 'deployment']],
|
|
933
|
+
['तैना', ['deploy', 'deployment']], // stemmed form
|
|
934
|
+
['त्रुटि', ['error', 'err']],
|
|
935
|
+
['गलती', ['error', 'bug', 'issue']],
|
|
936
|
+
['सुधार', ['fix', 'bugfix']],
|
|
937
|
+
['निर्माण', ['build']],
|
|
938
|
+
['परीक्षण', ['test']],
|
|
939
|
+
['प्रमाणीकरण', ['auth', 'authentication']],
|
|
940
|
+
['विन्यास', ['config', 'configuration']],
|
|
941
|
+
['कॉन्फ़िगरेशन', ['config', 'configuration']],
|
|
942
|
+
['इंस्टॉल', ['install']],
|
|
943
|
+
['अपडेट', ['update', 'upgrade']],
|
|
944
|
+
['लॉग', ['log']],
|
|
945
|
+
['सर्वर', ['server']],
|
|
946
|
+
['डेटाबेस', ['database', 'db']],
|
|
947
|
+
['नेटवर्क', ['network']],
|
|
948
|
+
['मेमोरी', ['memory', 'ram']],
|
|
949
|
+
['कैश', ['cache']],
|
|
950
|
+
['डीबग', ['debug', 'debugging']],
|
|
951
|
+
['क्रैश', ['crash', 'panic']],
|
|
952
|
+
['पुनरारंभ', ['restart', 'reboot']],
|
|
953
|
+
['अनुमति', ['permission', 'access']],
|
|
954
|
+
['बैकअप', ['backup']],
|
|
955
|
+
['प्रदर्शन', ['performance']],
|
|
956
|
+
['समस्या', ['problem', 'issue', 'error']],
|
|
957
|
+
['सेवा', ['service']],
|
|
958
|
+
['कनेक्शन', ['connection', 'conn']],
|
|
959
|
+
['पासवर्ड', ['password']],
|
|
960
|
+
['टोकन', ['token', 'secret']],
|
|
961
|
+
['वातावरण', ['environment', 'env']],
|
|
962
|
+
['उत्पादन', ['production', 'prod']],
|
|
963
|
+
// ── European languages → English ─────────────────────────────────────────
|
|
964
|
+
// German (Deutsch) → English
|
|
965
|
+
['fehler', ['error', 'err']],
|
|
966
|
+
['absturz', ['crash', 'panic']],
|
|
967
|
+
['bug', ['bug', 'issue']], // same word, keep for completeness
|
|
968
|
+
['beheben', ['fix', 'bugfix']],
|
|
969
|
+
['behoben', ['fix', 'fixed']],
|
|
970
|
+
['lösung', ['solution', 'fix']],
|
|
971
|
+
['bauen', ['build']],
|
|
972
|
+
['testen', ['test']],
|
|
973
|
+
['authentifizierung', ['auth', 'authentication']],
|
|
974
|
+
['konfiguration', ['config', 'configuration']],
|
|
975
|
+
['einstellungen', ['config', 'settings']],
|
|
976
|
+
['installieren', ['install']],
|
|
977
|
+
['installiert', ['install', 'installed']],
|
|
978
|
+
['aktualisieren', ['update', 'upgrade']],
|
|
979
|
+
['aktualisierung', ['update', 'upgrade']],
|
|
980
|
+
['protokoll', ['log']],
|
|
981
|
+
['protokolle', ['log', 'logs']],
|
|
982
|
+
['abbild', ['image']],
|
|
983
|
+
['knoten', ['node']],
|
|
984
|
+
['dienst', ['service']],
|
|
985
|
+
['sicherung', ['backup']],
|
|
986
|
+
['überwachung', ['monitor', 'monitoring']],
|
|
987
|
+
['verbindung', ['connection', 'conn']],
|
|
988
|
+
['warteschlange', ['queue']],
|
|
989
|
+
['leistung', ['performance']],
|
|
990
|
+
['speicher', ['memory', 'storage']],
|
|
991
|
+
['debuggen', ['debug', 'debugging']],
|
|
992
|
+
['neustart', ['restart', 'reboot']],
|
|
993
|
+
['berechtigung', ['permission', 'access']],
|
|
994
|
+
['berechtigungen', ['permission', 'access', 'acl']],
|
|
995
|
+
['netzwerk', ['network']],
|
|
996
|
+
['datenbank', ['database', 'db']],
|
|
997
|
+
['bereitstellen', ['deploy', 'deployment']],
|
|
998
|
+
['bereitstellung', ['deploy', 'deployment']],
|
|
999
|
+
// 'container' and 'server' are the same in German/French/Spanish/Italian/Portuguese
|
|
1000
|
+
['kaputt', ['broken', 'error', 'crash']],
|
|
1001
|
+
['funktioniert', ['works', 'working']],
|
|
1002
|
+
['geht nicht', ['not working', 'broken', 'error']],
|
|
1003
|
+
['schlüssel', ['key', 'token', 'secret']],
|
|
1004
|
+
['zertifikat', ['certificate', 'cert', 'ssl', 'tls']],
|
|
1005
|
+
['umgebung', ['environment', 'env']],
|
|
1006
|
+
['variablen', ['variables', 'env', 'environment']],
|
|
1007
|
+
['aufgabe', ['task', 'job']],
|
|
1008
|
+
['wartung', ['maintenance']],
|
|
1009
|
+
['speicherleck', ['memory leak', 'memory']],
|
|
1010
|
+
['schnittstelle', ['interface', 'api']],
|
|
1011
|
+
// French (Français) → English
|
|
1012
|
+
['erreur', ['error', 'err']],
|
|
1013
|
+
['réparer', ['fix']],
|
|
1014
|
+
['correction', ['fix', 'bugfix']],
|
|
1015
|
+
['construire', ['build']],
|
|
1016
|
+
['tester', ['test']],
|
|
1017
|
+
['authentification', ['auth', 'authentication']],
|
|
1018
|
+
['configuration', ['config', 'configuration']],
|
|
1019
|
+
['installer', ['install']],
|
|
1020
|
+
['installation', ['install']],
|
|
1021
|
+
['mettre', ['update', 'upgrade']],
|
|
1022
|
+
['journal', ['log', 'logs']],
|
|
1023
|
+
['nœud', ['node']],
|
|
1024
|
+
['sauvegarde', ['backup']],
|
|
1025
|
+
['surveiller', ['monitor', 'monitoring']],
|
|
1026
|
+
['connexion', ['connection']],
|
|
1027
|
+
['mémoire', ['memory']],
|
|
1028
|
+
['déboguer', ['debug']],
|
|
1029
|
+
['redémarrer', ['restart', 'reboot']],
|
|
1030
|
+
['réseau', ['network']],
|
|
1031
|
+
['base de données', ['database', 'db']],
|
|
1032
|
+
['déployer', ['deploy']],
|
|
1033
|
+
['déploiement', ['deploy', 'deployment']],
|
|
1034
|
+
['clé', ['key', 'token']],
|
|
1035
|
+
['certificat', ['certificate', 'cert', 'ssl']],
|
|
1036
|
+
['environnement', ['environment', 'env']],
|
|
1037
|
+
['panne', ['crash', 'outage', 'failure']],
|
|
1038
|
+
['plantage', ['crash', 'panic']],
|
|
1039
|
+
['interface', ['interface', 'api']],
|
|
1040
|
+
['performances', ['performance']],
|
|
1041
|
+
['stockage', ['storage', 'disk']],
|
|
1042
|
+
['file attente', ['queue']],
|
|
1043
|
+
// Spanish (Español) → English
|
|
1044
|
+
['arreglar', ['fix']],
|
|
1045
|
+
['corrección', ['fix', 'bugfix']],
|
|
1046
|
+
['construir', ['build']],
|
|
1047
|
+
['probar', ['test']],
|
|
1048
|
+
['autenticación', ['auth', 'authentication']],
|
|
1049
|
+
['configuración', ['config', 'configuration']],
|
|
1050
|
+
['instalar', ['install']],
|
|
1051
|
+
['actualizar', ['update', 'upgrade']],
|
|
1052
|
+
['actualización', ['update', 'upgrade']],
|
|
1053
|
+
['registro', ['log', 'registry']],
|
|
1054
|
+
['nodo', ['node']],
|
|
1055
|
+
['servicio', ['service']],
|
|
1056
|
+
['contraseña', ['password']],
|
|
1057
|
+
['copia de seguridad', ['backup']],
|
|
1058
|
+
['respaldo', ['backup']],
|
|
1059
|
+
['monitorear', ['monitor']],
|
|
1060
|
+
['monitoreo', ['monitoring']],
|
|
1061
|
+
['conexión', ['connection']],
|
|
1062
|
+
['rendimiento', ['performance']],
|
|
1063
|
+
['memoria', ['memory']],
|
|
1064
|
+
['depurar', ['debug']],
|
|
1065
|
+
['fallo', ['crash', 'failure', 'error']],
|
|
1066
|
+
['caída', ['crash', 'outage']],
|
|
1067
|
+
['reiniciar', ['restart', 'reboot']],
|
|
1068
|
+
['permiso', ['permission']],
|
|
1069
|
+
['permisos', ['permission', 'access']],
|
|
1070
|
+
['red', ['network']],
|
|
1071
|
+
['base de datos', ['database', 'db']],
|
|
1072
|
+
['desplegar', ['deploy']],
|
|
1073
|
+
['despliegue', ['deploy', 'deployment']],
|
|
1074
|
+
['clave', ['key', 'token', 'secret']],
|
|
1075
|
+
['certificado', ['certificate', 'cert', 'ssl']],
|
|
1076
|
+
['entorno', ['environment', 'env']],
|
|
1077
|
+
['cola', ['queue']],
|
|
1078
|
+
['interfaz', ['interface', 'api']],
|
|
1079
|
+
['imagen', ['image']],
|
|
1080
|
+
// Italian (Italiano) → English
|
|
1081
|
+
['errore', ['error', 'err']],
|
|
1082
|
+
['correggere', ['fix']],
|
|
1083
|
+
['costruire', ['build']],
|
|
1084
|
+
['testare', ['test']],
|
|
1085
|
+
['autenticazione', ['auth', 'authentication']],
|
|
1086
|
+
['configurazione', ['config', 'configuration']],
|
|
1087
|
+
['installare', ['install']],
|
|
1088
|
+
['aggiornare', ['update', 'upgrade']],
|
|
1089
|
+
['aggiornamento', ['update', 'upgrade']],
|
|
1090
|
+
['registro', ['log']],
|
|
1091
|
+
['nodo', ['node']],
|
|
1092
|
+
['servizio', ['service']],
|
|
1093
|
+
['password', ['password']],
|
|
1094
|
+
['backup', ['backup']],
|
|
1095
|
+
['monitorare', ['monitor']],
|
|
1096
|
+
['connessione', ['connection']],
|
|
1097
|
+
['prestazioni', ['performance']],
|
|
1098
|
+
['memoria', ['memory']],
|
|
1099
|
+
['debug', ['debug']],
|
|
1100
|
+
['arresto anomalo', ['crash']],
|
|
1101
|
+
['riavviare', ['restart']],
|
|
1102
|
+
['permesso', ['permission']],
|
|
1103
|
+
['rete', ['network']],
|
|
1104
|
+
['database', ['database', 'db']],
|
|
1105
|
+
['distribuire', ['deploy']],
|
|
1106
|
+
['distribuzione', ['deploy', 'deployment']],
|
|
1107
|
+
['chiave', ['key', 'token', 'secret']],
|
|
1108
|
+
['certificato', ['certificate', 'cert']],
|
|
1109
|
+
['ambiente', ['environment', 'env']],
|
|
1110
|
+
['coda', ['queue']],
|
|
1111
|
+
['immagine', ['image']],
|
|
1112
|
+
// Portuguese (Português) → English
|
|
1113
|
+
['erro', ['error', 'err']],
|
|
1114
|
+
['corrigir', ['fix']],
|
|
1115
|
+
['construir', ['build']],
|
|
1116
|
+
['testar', ['test']],
|
|
1117
|
+
['autenticação', ['auth', 'authentication']],
|
|
1118
|
+
['configuração', ['config', 'configuration']],
|
|
1119
|
+
['instalar', ['install']],
|
|
1120
|
+
['atualizar', ['update', 'upgrade']],
|
|
1121
|
+
['atualização', ['update', 'upgrade']],
|
|
1122
|
+
['registro', ['log']],
|
|
1123
|
+
['nó', ['node']],
|
|
1124
|
+
['serviço', ['service']],
|
|
1125
|
+
['senha', ['password']],
|
|
1126
|
+
['cópia de segurança', ['backup']],
|
|
1127
|
+
['monitorar', ['monitor']],
|
|
1128
|
+
['conexão', ['connection']],
|
|
1129
|
+
['desempenho', ['performance']],
|
|
1130
|
+
['memória', ['memory']],
|
|
1131
|
+
['depurar', ['debug']],
|
|
1132
|
+
['falha', ['crash', 'failure', 'error']],
|
|
1133
|
+
['reiniciar', ['restart']],
|
|
1134
|
+
['permissão', ['permission']],
|
|
1135
|
+
['rede', ['network']],
|
|
1136
|
+
['banco de dados', ['database', 'db']],
|
|
1137
|
+
['implantar', ['deploy']],
|
|
1138
|
+
['implantação', ['deploy', 'deployment']],
|
|
1139
|
+
['chave', ['key', 'token', 'secret']],
|
|
1140
|
+
['certificado', ['certificate', 'cert']],
|
|
1141
|
+
['ambiente', ['environment', 'env']],
|
|
1142
|
+
['fila', ['queue']],
|
|
1143
|
+
['imagem', ['image']],
|
|
1144
|
+
// ── Russian (Cyrillic) → English ──────────────────────────────────────────
|
|
1145
|
+
['ошибка', ['error', 'err']],
|
|
1146
|
+
['ошибки', ['error', 'err']],
|
|
1147
|
+
['сбой', ['crash', 'failure', 'error']],
|
|
1148
|
+
['крэш', ['crash', 'panic']],
|
|
1149
|
+
['баг', ['bug', 'issue']],
|
|
1150
|
+
['исправить', ['fix', 'bugfix']],
|
|
1151
|
+
['исправление', ['fix', 'patch']],
|
|
1152
|
+
['сборка', ['build']],
|
|
1153
|
+
['собрать', ['build']],
|
|
1154
|
+
['тест', ['test']],
|
|
1155
|
+
['тестирование', ['test', 'testing']],
|
|
1156
|
+
['аутентификация', ['auth', 'authentication']],
|
|
1157
|
+
['авторизация', ['auth', 'authorization']],
|
|
1158
|
+
['конфигурация', ['config', 'configuration']],
|
|
1159
|
+
['настройки', ['config', 'settings']],
|
|
1160
|
+
['установить', ['install']],
|
|
1161
|
+
['установка', ['install']],
|
|
1162
|
+
['обновление', ['update', 'upgrade']],
|
|
1163
|
+
['обновить', ['update', 'upgrade']],
|
|
1164
|
+
['журнал', ['log']],
|
|
1165
|
+
['лог', ['log', 'logs']],
|
|
1166
|
+
['сервер', ['server']],
|
|
1167
|
+
['база данных', ['database', 'db']],
|
|
1168
|
+
['сеть', ['network']],
|
|
1169
|
+
['память', ['memory', 'ram']],
|
|
1170
|
+
['кэш', ['cache']],
|
|
1171
|
+
['отладка', ['debug', 'debugging']],
|
|
1172
|
+
['перезапуск', ['restart', 'reboot']],
|
|
1173
|
+
['перезагрузка', ['restart', 'reboot']],
|
|
1174
|
+
['разрешение', ['permission', 'access']],
|
|
1175
|
+
['доступ', ['permission', 'access']],
|
|
1176
|
+
['резерв', ['backup']],
|
|
1177
|
+
['производительность', ['performance']],
|
|
1178
|
+
['проблема', ['problem', 'issue', 'error']],
|
|
1179
|
+
['сервис', ['service']],
|
|
1180
|
+
['служба', ['service']],
|
|
1181
|
+
['подключение', ['connection', 'conn']],
|
|
1182
|
+
['соединение', ['connection', 'conn']],
|
|
1183
|
+
['развёртывание', ['deploy', 'deployment']],
|
|
1184
|
+
['развертывание', ['deploy', 'deployment']],
|
|
1185
|
+
['развёрт', ['deploy', 'deployment']], // stemmed form
|
|
1186
|
+
['разверт', ['deploy', 'deployment']], // stemmed form
|
|
1187
|
+
['контейнер', ['container', 'docker']],
|
|
1188
|
+
['образ', ['image']],
|
|
1189
|
+
['кластер', ['cluster']],
|
|
1190
|
+
['узел', ['node']],
|
|
1191
|
+
['секрет', ['secret', 'key', 'token']],
|
|
1192
|
+
['ключ', ['key', 'token', 'secret']],
|
|
1193
|
+
['пароль', ['password']],
|
|
1194
|
+
['токен', ['token', 'secret']],
|
|
1195
|
+
['мониторинг', ['monitor', 'monitoring']],
|
|
1196
|
+
['наблюдение', ['monitor', 'monitoring']],
|
|
1197
|
+
['мигрировать', ['migrate', 'migration']],
|
|
1198
|
+
['миграция', ['migrate', 'migration']],
|
|
1199
|
+
// ── Turkish (Latin + special chars ğ/ş/ı) → English ─────────────────────
|
|
1200
|
+
['hata', ['error', 'err']],
|
|
1201
|
+
['hatalı', ['error', 'err']],
|
|
1202
|
+
['çökme', ['crash', 'panic']],
|
|
1203
|
+
['çöküş', ['crash', 'failure']],
|
|
1204
|
+
['düzeltmek', ['fix', 'bugfix']],
|
|
1205
|
+
['düzeltme', ['fix', 'patch']],
|
|
1206
|
+
['derleme', ['build']],
|
|
1207
|
+
['derlemek', ['build']],
|
|
1208
|
+
['test', ['test']],
|
|
1209
|
+
['testlemek', ['test', 'testing']],
|
|
1210
|
+
['kimlik doğrulama', ['auth', 'authentication']],
|
|
1211
|
+
['yetkilendirme', ['auth', 'authorization']],
|
|
1212
|
+
['yapılandırma', ['config', 'configuration']],
|
|
1213
|
+
['ayarlar', ['config', 'settings']],
|
|
1214
|
+
['yüklemek', ['install']],
|
|
1215
|
+
['kurulum', ['install']],
|
|
1216
|
+
['güncelleme', ['update', 'upgrade']],
|
|
1217
|
+
['günlük', ['log']],
|
|
1218
|
+
['sunucu', ['server']],
|
|
1219
|
+
['veritabanı', ['database', 'db']],
|
|
1220
|
+
['ağ', ['network']],
|
|
1221
|
+
['bellek', ['memory', 'ram']],
|
|
1222
|
+
['önbellek', ['cache']],
|
|
1223
|
+
['hata ayıklama', ['debug', 'debugging']],
|
|
1224
|
+
['yeniden başlatma', ['restart', 'reboot']],
|
|
1225
|
+
['izin', ['permission', 'access']],
|
|
1226
|
+
['yetki', ['permission', 'access', 'acl']],
|
|
1227
|
+
['yedekleme', ['backup']],
|
|
1228
|
+
['performans', ['performance']],
|
|
1229
|
+
['sorun', ['problem', 'issue', 'error']],
|
|
1230
|
+
['servis', ['service']],
|
|
1231
|
+
['hizmet', ['service']],
|
|
1232
|
+
['bağlantı', ['connection', 'conn']],
|
|
1233
|
+
['dağıtım', ['deploy', 'deployment']],
|
|
1234
|
+
['dağıtmak', ['deploy']],
|
|
1235
|
+
['konteyner', ['container', 'docker']],
|
|
1236
|
+
['küme', ['cluster']],
|
|
1237
|
+
['düğüm', ['node']],
|
|
1238
|
+
['gizli', ['secret', 'key']],
|
|
1239
|
+
['anahtar', ['key', 'token', 'secret']],
|
|
1240
|
+
['şifre', ['password']],
|
|
1241
|
+
['izleme', ['monitor', 'monitoring']],
|
|
1242
|
+
['göç', ['migrate', 'migration']],
|
|
1243
|
+
// ── Vietnamese (Latin + Latin Extended Additional U+1EA0–U+1EF9) → English ─
|
|
1244
|
+
['triển khai', ['deploy', 'deployment']],
|
|
1245
|
+
['triển', ['deploy', 'deployment']], // split token form
|
|
1246
|
+
['lỗi', ['error', 'err']],
|
|
1247
|
+
['sửa lỗi', ['fix', 'bugfix', 'debug']],
|
|
1248
|
+
['xây dựng', ['build']],
|
|
1249
|
+
['kiểm tra', ['test']],
|
|
1250
|
+
['xác thực', ['auth', 'authentication']],
|
|
1251
|
+
['cấu hình', ['config', 'configuration']],
|
|
1252
|
+
['cài đặt', ['install']],
|
|
1253
|
+
['cập nhật', ['update', 'upgrade']],
|
|
1254
|
+
['nhật ký', ['log']],
|
|
1255
|
+
['máy chủ', ['server']],
|
|
1256
|
+
['cơ sở dữ liệu', ['database', 'db']],
|
|
1257
|
+
['mạng', ['network']],
|
|
1258
|
+
['bộ nhớ', ['memory', 'ram']],
|
|
1259
|
+
['bộ đệm', ['cache']],
|
|
1260
|
+
['gỡ lỗi', ['debug', 'debugging']],
|
|
1261
|
+
['sự cố', ['crash', 'failure', 'error']],
|
|
1262
|
+
['khởi động lại', ['restart', 'reboot']],
|
|
1263
|
+
['quyền', ['permission', 'access']],
|
|
1264
|
+
['sao lưu', ['backup']],
|
|
1265
|
+
['hiệu suất', ['performance']],
|
|
1266
|
+
['vấn đề', ['problem', 'issue', 'error']],
|
|
1267
|
+
['dịch vụ', ['service']],
|
|
1268
|
+
['kết nối', ['connection', 'conn']],
|
|
1269
|
+
['vùng chứa', ['container', 'docker']],
|
|
1270
|
+
['nút', ['node']],
|
|
1271
|
+
['khóa', ['key', 'token', 'secret']],
|
|
1272
|
+
['mật khẩu', ['password']],
|
|
1273
|
+
['giám sát', ['monitor', 'monitoring']],
|
|
1274
|
+
['di chuyển', ['migrate', 'migration']],
|
|
1275
|
+
]);
|
|
1276
|
+
/**
|
|
1277
|
+
* Expand a token to its cross-lingual synonyms.
|
|
1278
|
+
* Returns synonyms to be added to the token stream (pre-tokenized).
|
|
1279
|
+
* CJK synonyms are NOT pre-bigram'd here — caller handles that.
|
|
1280
|
+
*/
|
|
1281
|
+
function expandCrossLingual(token) {
|
|
1282
|
+
return CROSS_LINGUAL_MAP.get(token) ?? [];
|
|
1283
|
+
}
|
|
1284
|
+
/**
|
|
1285
|
+
* Tokenize text into meaningful keywords.
|
|
1286
|
+
* Handles accented Latin, RTL (Arabic, Hebrew), and CJK (Chinese, Japanese, Korean).
|
|
1287
|
+
*
|
|
1288
|
+
* Features:
|
|
1289
|
+
* • CJK → character bigrams
|
|
1290
|
+
* • Katakana → additionally emits Hepburn romaji tokens (enables romaji queries)
|
|
1291
|
+
* • Arabic → word tokenization + iterative prefix stemming (ال/و/ب/ل/ف/ك)
|
|
1292
|
+
* • Hebrew → word tokenization + iterative prefix stemming (ה/ו/ב/ל/מ/כ)
|
|
1293
|
+
* • Farsi → word tokenization + suffix/prefix stemming (ها/های plural, می prefix)
|
|
1294
|
+
* • Hindi → Devanagari word tokenization + light suffix stemming
|
|
1295
|
+
* • All scripts → cross-lingual synonym expansion (EN↔JA↔ZH↔KO↔AR↔HE↔FA↔HI↔DE↔FR↔ES↔IT↔PT)
|
|
1296
|
+
*/
|
|
1297
|
+
function tokenize(text) {
|
|
1298
|
+
const tokens = [];
|
|
1299
|
+
// Pre-process: split camelCase/PascalCase identifiers and snake_case before lowercasing
|
|
1300
|
+
const lower = preprocessText(text).toLowerCase();
|
|
1301
|
+
for (const [seg] of lower.matchAll(SEGMENT_RE)) {
|
|
1302
|
+
if (CJK_RE.test(seg)) {
|
|
1303
|
+
// CJK segment — extract overlapping character bigrams
|
|
1304
|
+
for (let i = 0; i < seg.length - 1; i++) {
|
|
1305
|
+
const bigram = seg[i] + seg[i + 1];
|
|
1306
|
+
if (!STOPWORDS.has(seg[i]) && !STOPWORDS.has(bigram)) {
|
|
1307
|
+
tokens.push(bigram);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
// Trailing single character (for 1-char CJK terms)
|
|
1311
|
+
if (seg.length >= 1) {
|
|
1312
|
+
const last = seg[seg.length - 1];
|
|
1313
|
+
if (!STOPWORDS.has(last))
|
|
1314
|
+
tokens.push(last);
|
|
1315
|
+
}
|
|
1316
|
+
// Whole-segment cross-lingual lookup (for known tech terms like デプロイ, 部署, 배포)
|
|
1317
|
+
// This ensures CJK full-words map to their English equivalents
|
|
1318
|
+
const segSyns = expandCrossLingual(seg);
|
|
1319
|
+
for (const syn of segSyns) {
|
|
1320
|
+
if (!CJK_RE.test(syn)) {
|
|
1321
|
+
syn.split(/\s+/).filter(w => w.length > 1 && !STOPWORDS.has(w)).forEach(w => tokens.push(w));
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
// Romanization: extract katakana-only portion and emit romaji tokens.
|
|
1325
|
+
// This allows users to search "kontena" or "depuroi" to find katakana docs.
|
|
1326
|
+
const kataOnly = seg.replace(/[^\u30a0-\u30ff]/g, '');
|
|
1327
|
+
if (kataOnly.length >= 2) {
|
|
1328
|
+
const romaji = katakanaToRomaji(kataOnly);
|
|
1329
|
+
if (romaji.length >= 2 && romaji.length <= 40 && !STOPWORDS.has(romaji)) {
|
|
1330
|
+
tokens.push(romaji);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
else if (RTL_RE.test(seg)) {
|
|
1335
|
+
// RTL segment — 3-way branch: Farsi / Hebrew / Arabic
|
|
1336
|
+
const words = seg.trim().split(/\s+/);
|
|
1337
|
+
let stemFn;
|
|
1338
|
+
if (FARSI_CHAR_RE.test(seg))
|
|
1339
|
+
stemFn = farsiLightStem;
|
|
1340
|
+
else if (HEBREW_CHAR_RE.test(seg))
|
|
1341
|
+
stemFn = hebrewLightStem;
|
|
1342
|
+
else
|
|
1343
|
+
stemFn = arabicLightStem;
|
|
1344
|
+
for (const w of words) {
|
|
1345
|
+
if (w.length <= 1 || STOPWORDS.has(w))
|
|
1346
|
+
continue;
|
|
1347
|
+
const stemmed = stemFn(w);
|
|
1348
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1349
|
+
tokens.push(stemmed);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
else if (DEVANAGARI_RE.test(seg)) {
|
|
1353
|
+
// Devanagari segment (Hindi / Marathi) — naturally space-separated
|
|
1354
|
+
const words = seg.trim().split(/[\s\u200c\u200d]+/); // handle zero-width joiners
|
|
1355
|
+
for (const w of words) {
|
|
1356
|
+
const deva = w.replace(/[^\u0900-\u097f0-9]/g, '');
|
|
1357
|
+
if (deva.length <= 1 || STOPWORDS.has(deva))
|
|
1358
|
+
continue;
|
|
1359
|
+
const stemmed = hindiLightStem(deva);
|
|
1360
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1361
|
+
tokens.push(stemmed);
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
else if (BENGALI_RE.test(seg)) {
|
|
1365
|
+
// Bengali (Bangla) segment — space-separated words, Bengali script
|
|
1366
|
+
const words = seg.trim().split(/[\s\u200c\u200d]+/);
|
|
1367
|
+
for (const w of words) {
|
|
1368
|
+
const bn = w.replace(/[^\u0980-\u09ff0-9]/g, '');
|
|
1369
|
+
if (bn.length <= 1 || STOPWORDS.has(bn))
|
|
1370
|
+
continue;
|
|
1371
|
+
const stemmed = bengaliLightStem(bn);
|
|
1372
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1373
|
+
tokens.push(stemmed);
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
else if (CYRILLIC_RE.test(seg)) {
|
|
1377
|
+
// Cyrillic segment (Russian, Bulgarian, Ukrainian, Serbian …)
|
|
1378
|
+
const words = seg.trim().split(/\s+/);
|
|
1379
|
+
for (const w of words) {
|
|
1380
|
+
const cyr = w.replace(/[^\u0400-\u04ff]/g, '');
|
|
1381
|
+
if (cyr.length <= 1 || STOPWORDS.has(cyr))
|
|
1382
|
+
continue;
|
|
1383
|
+
const stemmed = russianLightStem(cyr);
|
|
1384
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1385
|
+
tokens.push(stemmed);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
else {
|
|
1389
|
+
// Latin / other — includes Turkish (ğ/ş/ı), Polish (ą/ę/ł/ń/ó/ś/ź/ż/ć),
|
|
1390
|
+
// Czech (č/š/ž/ě/ů/ř), Hungarian (ő/ű), Romanian (ș/ț), Vietnamese (ắặầổợụừ…)
|
|
1391
|
+
const words = seg
|
|
1392
|
+
.replace(/[^a-záàâãäåæçéèêëíìîïñóòôõöúùûüýÿßœğşıİąćęłńśźżčšžěůřőűșț\u1ea0-\u1ef90-9:\-./]/g, ' ')
|
|
1393
|
+
.split(/\s+/)
|
|
1394
|
+
.filter(w => w.length > 1 && !STOPWORDS.has(w));
|
|
1395
|
+
// Turkish: apply light stemmer to words with Turkish-specific chars
|
|
1396
|
+
for (const w of words) {
|
|
1397
|
+
if (TURKISH_CHAR_RE.test(w)) {
|
|
1398
|
+
const stemmed = turkishLightStem(w);
|
|
1399
|
+
if (stemmed !== w && stemmed.length > 1 && !STOPWORDS.has(stemmed)) {
|
|
1400
|
+
tokens.push(stemmed);
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
tokens.push(w);
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
// ── Cross-lingual expansion ──────────────────────────────────────────────
|
|
1408
|
+
// For each token, look up synonyms in other languages and add them.
|
|
1409
|
+
// CJK synonyms are bigram-expanded inline; Latin/RTL synonyms added as-is.
|
|
1410
|
+
const expansions = [];
|
|
1411
|
+
const seen = new Set(tokens);
|
|
1412
|
+
for (const tok of tokens) {
|
|
1413
|
+
const syns = expandCrossLingual(tok);
|
|
1414
|
+
for (const syn of syns) {
|
|
1415
|
+
if (seen.has(syn))
|
|
1416
|
+
continue;
|
|
1417
|
+
seen.add(syn);
|
|
1418
|
+
if (CJK_RE.test(syn)) {
|
|
1419
|
+
// CJK synonym → expand to character bigrams
|
|
1420
|
+
for (let i = 0; i < syn.length - 1; i++) {
|
|
1421
|
+
const bg = syn[i] + syn[i + 1];
|
|
1422
|
+
if (!STOPWORDS.has(syn[i]) && !STOPWORDS.has(bg))
|
|
1423
|
+
expansions.push(bg);
|
|
1424
|
+
}
|
|
1425
|
+
if (syn.length >= 1) {
|
|
1426
|
+
const last = syn[syn.length - 1];
|
|
1427
|
+
if (!STOPWORDS.has(last))
|
|
1428
|
+
expansions.push(last);
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
else if (RTL_RE.test(syn)) {
|
|
1432
|
+
// RTL synonym — apply correct stemmer per language
|
|
1433
|
+
const rtlStem = FARSI_CHAR_RE.test(syn) ? farsiLightStem
|
|
1434
|
+
: HEBREW_CHAR_RE.test(syn) ? hebrewLightStem
|
|
1435
|
+
: arabicLightStem;
|
|
1436
|
+
const stemmed = rtlStem(syn);
|
|
1437
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1438
|
+
expansions.push(stemmed);
|
|
1439
|
+
}
|
|
1440
|
+
else if (DEVANAGARI_RE.test(syn)) {
|
|
1441
|
+
// Devanagari synonym (Hindi)
|
|
1442
|
+
const stemmed = hindiLightStem(syn);
|
|
1443
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1444
|
+
expansions.push(stemmed);
|
|
1445
|
+
}
|
|
1446
|
+
else if (BENGALI_RE.test(syn)) {
|
|
1447
|
+
// Bengali synonym
|
|
1448
|
+
const stemmed = bengaliLightStem(syn);
|
|
1449
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1450
|
+
expansions.push(stemmed);
|
|
1451
|
+
}
|
|
1452
|
+
else if (CYRILLIC_RE.test(syn)) {
|
|
1453
|
+
// Cyrillic synonym (Russian)
|
|
1454
|
+
const cyr = syn.replace(/[^\u0400-\u04ff]/g, '');
|
|
1455
|
+
const stemmed = russianLightStem(cyr);
|
|
1456
|
+
if (stemmed.length > 1 && !STOPWORDS.has(stemmed))
|
|
1457
|
+
expansions.push(stemmed);
|
|
1458
|
+
}
|
|
1459
|
+
else {
|
|
1460
|
+
// Latin synonym (includes Turkish) — split in case of multi-word
|
|
1461
|
+
const parts = syn.split(/\s+/).filter(w => {
|
|
1462
|
+
if (w.length <= 1 || STOPWORDS.has(w))
|
|
1463
|
+
return false;
|
|
1464
|
+
return true;
|
|
1465
|
+
});
|
|
1466
|
+
for (const p of parts) {
|
|
1467
|
+
expansions.push(p);
|
|
1468
|
+
// Apply Turkish stemmer if it has Turkish-specific characters
|
|
1469
|
+
if (TURKISH_CHAR_RE.test(p)) {
|
|
1470
|
+
const stemmed = turkishLightStem(p);
|
|
1471
|
+
if (stemmed !== p && stemmed.length > 1 && !STOPWORDS.has(stemmed)) {
|
|
1472
|
+
expansions.push(stemmed);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
tokens.push(...expansions);
|
|
1480
|
+
return tokens;
|
|
1481
|
+
}
|
|
1482
|
+
/**
|
|
1483
|
+
* Split a multi-topic query into sub-queries.
|
|
1484
|
+
* Detects numbered lists, semicolons, "and also", line breaks, etc.
|
|
1485
|
+
*
|
|
1486
|
+
* Example: "deploy API fix routing and also check auth" →
|
|
1487
|
+
* ["deploy API fix routing", "check auth"]
|
|
1488
|
+
*
|
|
1489
|
+
* Example: "1. deploy 2. fix routing 3. auth" →
|
|
1490
|
+
* ["deploy", "fix routing", "auth"]
|
|
1491
|
+
*/
|
|
1492
|
+
function splitMultiQuery(query) {
|
|
1493
|
+
// Numbered list: "1. foo 2. bar 3. baz"
|
|
1494
|
+
const numberedParts = query.split(/\d+[.)]\s*/g).filter(s => s.trim().length > 2);
|
|
1495
|
+
if (numberedParts.length >= 2)
|
|
1496
|
+
return numberedParts.map(s => s.trim());
|
|
1497
|
+
// Semicolons or newlines
|
|
1498
|
+
const semiParts = query.split(/[;\n]+/).filter(s => s.trim().length > 2);
|
|
1499
|
+
if (semiParts.length >= 2)
|
|
1500
|
+
return semiParts.map(s => s.trim());
|
|
1501
|
+
// Conjunctions (EN + DE + FR + ES + IT + PT + RU + TR)
|
|
1502
|
+
const conjParts = query.split(/\b(?:and also|also noch|außerdem|plus|additionally|furthermore|de plus|además|inoltre|além disso|а также|и также|кроме того|плюс|ayrıca|bunun yanı sıra|همچنین|وأيضاً|وكذلك|وهمچنین|और भी|इसके अलावा)\b/i).filter(s => s.trim().length > 2);
|
|
1503
|
+
if (conjParts.length >= 2)
|
|
1504
|
+
return conjParts.map(s => s.trim());
|
|
1505
|
+
// Comma-separated with 3+ parts (likely a list)
|
|
1506
|
+
const commaParts = query.split(/,\s*/).filter(s => s.trim().length > 2);
|
|
1507
|
+
if (commaParts.length >= 3)
|
|
1508
|
+
return commaParts.map(s => s.trim());
|
|
1509
|
+
// Single query
|
|
1510
|
+
return [query];
|
|
1511
|
+
}
|
|
1512
|
+
// ── BM25+ Scoring ─────────────────────────────────────────────────────────────
|
|
1513
|
+
//
|
|
1514
|
+
// BM25+ fixes the "long document under-scoring" bug in classic BM25.
|
|
1515
|
+
// Paper: Lv & Zhai (2011) "Lower-Bounding Term Frequency Normalization"
|
|
1516
|
+
//
|
|
1517
|
+
// Improvements over standard BM25:
|
|
1518
|
+
// 1. δ=1 additive term → guarantees TF>0 terms always contribute positively
|
|
1519
|
+
// 2. Bigram proximity boost → adjacent query terms in doc get 2× weight
|
|
1520
|
+
// 3. Recency boost → entries with timestamps get exp-decay bonus (7d half-life)
|
|
1521
|
+
// 4. Levenshtein fuzzy match → typo-tolerant (distance ≤ 2)
|
|
1522
|
+
//
|
|
1523
|
+
/** BM25+ parameters */
|
|
1524
|
+
const BM25_K1 = 1.2; // term frequency saturation
|
|
1525
|
+
const BM25_B = 0.75; // length normalization
|
|
1526
|
+
const BM25_DELTA = 1.0; // BM25+ lower-bound guarantee (0 = classic BM25)
|
|
1527
|
+
/** Recency boost: half-life in days. Entry from 7 days ago gets 0.5× boost. */
|
|
1528
|
+
const RECENCY_HALF_LIFE_DAYS = 7;
|
|
1529
|
+
const ZERO_RESULTS_LOG = [];
|
|
1530
|
+
const ZERO_RESULTS_MAX = 500;
|
|
1531
|
+
let zeroResultsTotal = 0;
|
|
1532
|
+
function logZeroResult(query) {
|
|
1533
|
+
zeroResultsTotal++;
|
|
1534
|
+
if (ZERO_RESULTS_LOG.length >= ZERO_RESULTS_MAX)
|
|
1535
|
+
ZERO_RESULTS_LOG.shift();
|
|
1536
|
+
ZERO_RESULTS_LOG.push({ query, ts: Date.now() });
|
|
1537
|
+
}
|
|
1538
|
+
/** Global index vocabulary — rebuilt on each keywordSearch call from the doc set. */
|
|
1539
|
+
let _indexVocab = new Set();
|
|
1540
|
+
/**
|
|
1541
|
+
* Levenshtein distance — edit distance between two strings.
|
|
1542
|
+
* Used for typo-tolerant fuzzy matching (distance ≤ 2 = match).
|
|
1543
|
+
* O(n*m) but strings are short (tokens), so this is fast.
|
|
1544
|
+
*/
|
|
1545
|
+
function levenshtein(a, b) {
|
|
1546
|
+
if (a === b)
|
|
1547
|
+
return 0;
|
|
1548
|
+
if (a.length === 0)
|
|
1549
|
+
return b.length;
|
|
1550
|
+
if (b.length === 0)
|
|
1551
|
+
return a.length;
|
|
1552
|
+
// Skip if length diff > 2 (can't be ≤ 2 edits)
|
|
1553
|
+
if (Math.abs(a.length - b.length) > 2)
|
|
1554
|
+
return 3;
|
|
1555
|
+
const prev = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
1556
|
+
for (let i = 1; i <= a.length; i++) {
|
|
1557
|
+
let corner = i - 1;
|
|
1558
|
+
prev[0] = i;
|
|
1559
|
+
for (let j = 1; j <= b.length; j++) {
|
|
1560
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
1561
|
+
const cur = Math.min(prev[j] + 1, prev[j - 1] + 1, corner + cost);
|
|
1562
|
+
corner = prev[j];
|
|
1563
|
+
prev[j] = cur;
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
return prev[b.length];
|
|
1567
|
+
}
|
|
1568
|
+
/** Extract a timestamp from JSON content (looks for "ts" or "created" fields). */
|
|
1569
|
+
function extractTimestamp(content) {
|
|
1570
|
+
// Fast regex for ISO dates in JSON: "ts":"2026-04-17T..." or "created":"..."
|
|
1571
|
+
const match = content.match(/"(?:ts|created|created_at|timestamp)"\s*:\s*"([^"]+)"/);
|
|
1572
|
+
if (match) {
|
|
1573
|
+
const ms = Date.parse(match[1]);
|
|
1574
|
+
if (!isNaN(ms))
|
|
1575
|
+
return ms;
|
|
1576
|
+
}
|
|
1577
|
+
return undefined;
|
|
1578
|
+
}
|
|
1579
|
+
/** Recency multiplier: 1.0 for now, 0.5 after half-life days, exponential decay. */
|
|
1580
|
+
function recencyBoost(timestampMs) {
|
|
1581
|
+
if (!timestampMs)
|
|
1582
|
+
return 1.0; // no timestamp → neutral
|
|
1583
|
+
const ageDays = (Date.now() - timestampMs) / (1000 * 60 * 60 * 24);
|
|
1584
|
+
if (ageDays <= 0)
|
|
1585
|
+
return 1.5; // future/just-now → max boost
|
|
1586
|
+
return Math.pow(0.5, ageDays / RECENCY_HALF_LIFE_DAYS) + 0.5; // range: [0.5, 1.5]
|
|
1587
|
+
}
|
|
1588
|
+
/**
|
|
1589
|
+
* BM25+ with Bigram Proximity, Recency Boost, and Fuzzy Matching.
|
|
1590
|
+
*
|
|
1591
|
+
* Architecture:
|
|
1592
|
+
* ┌─────────────────────────────────────────────────────────────────┐
|
|
1593
|
+
* │ Query: "deploy API fix routing; check auth" │
|
|
1594
|
+
* │ ↓ splitMultiQuery │
|
|
1595
|
+
* │ Sub-queries: ["deploy API fix routing", "check auth"] │
|
|
1596
|
+
* │ ↓ for each sub-query │
|
|
1597
|
+
* │ Tokenize → BM25+ score per doc → Bigram boost → Fuzzy match │
|
|
1598
|
+
* │ ↓ merge & deduplicate │
|
|
1599
|
+
* │ Recency boost → Sort → Top-K │
|
|
1600
|
+
* └─────────────────────────────────────────────────────────────────┘
|
|
1601
|
+
*/
|
|
1602
|
+
async function keywordSearch(redis, patterns, query, topK = 10) {
|
|
1603
|
+
// ── Step 1: Collect all matching keys — patterns scanned in parallel ──
|
|
1604
|
+
const keyBatches = await Promise.all(patterns.map(pattern => {
|
|
1605
|
+
const keys = [];
|
|
1606
|
+
const stream = redis.scanStream({ match: pattern, count: 500 });
|
|
1607
|
+
return new Promise((resolve, reject) => {
|
|
1608
|
+
stream.on('data', (batch) => {
|
|
1609
|
+
keys.push(...batch.filter((k) => !k.endsWith(':meta')));
|
|
1610
|
+
});
|
|
1611
|
+
stream.on('end', () => resolve(keys));
|
|
1612
|
+
stream.on('error', reject);
|
|
1613
|
+
});
|
|
1614
|
+
}));
|
|
1615
|
+
// Deduplicate across patterns (a key may match multiple globs)
|
|
1616
|
+
const allKeys = [...new Set(keyBatches.flat())];
|
|
1617
|
+
if (allKeys.length === 0)
|
|
1618
|
+
return [];
|
|
1619
|
+
// Pipeline GET for speed
|
|
1620
|
+
const pipeline = redis.pipeline();
|
|
1621
|
+
for (const key of allKeys)
|
|
1622
|
+
pipeline.get(key);
|
|
1623
|
+
const results = await pipeline.exec();
|
|
1624
|
+
const docs = [];
|
|
1625
|
+
let totalTokens = 0;
|
|
1626
|
+
for (let i = 0; i < allKeys.length; i++) {
|
|
1627
|
+
const content = results?.[i]?.[1];
|
|
1628
|
+
if (!content)
|
|
1629
|
+
continue;
|
|
1630
|
+
const tokens = tokenize(`${allKeys[i]} ${content}`);
|
|
1631
|
+
if (tokens.length === 0)
|
|
1632
|
+
continue;
|
|
1633
|
+
// Term frequency map
|
|
1634
|
+
const tokenFreq = new Map();
|
|
1635
|
+
for (const t of tokens) {
|
|
1636
|
+
tokenFreq.set(t, (tokenFreq.get(t) ?? 0) + 1);
|
|
1637
|
+
}
|
|
1638
|
+
// Bigrams — adjacent token pairs for proximity detection
|
|
1639
|
+
const bigrams = new Set();
|
|
1640
|
+
for (let j = 0; j < tokens.length - 1; j++) {
|
|
1641
|
+
bigrams.add(`${tokens[j]}|${tokens[j + 1]}`);
|
|
1642
|
+
}
|
|
1643
|
+
const timestamp = extractTimestamp(content);
|
|
1644
|
+
// Key tokens for title-boost: terms appearing in the Redis key get extra weight
|
|
1645
|
+
const keyTokens = new Set(tokenize(allKeys[i]));
|
|
1646
|
+
docs.push({ key: allKeys[i], content, tokens, tokenFreq, bigrams, keyTokens, timestamp });
|
|
1647
|
+
totalTokens += tokens.length;
|
|
1648
|
+
}
|
|
1649
|
+
if (docs.length === 0)
|
|
1650
|
+
return [];
|
|
1651
|
+
const avgDL = totalTokens / docs.length;
|
|
1652
|
+
// ── Step 2: IDF (inverse document frequency) ──
|
|
1653
|
+
const docFreq = new Map();
|
|
1654
|
+
for (const doc of docs) {
|
|
1655
|
+
const seen = new Set();
|
|
1656
|
+
for (const t of doc.tokens) {
|
|
1657
|
+
if (!seen.has(t)) {
|
|
1658
|
+
docFreq.set(t, (docFreq.get(t) ?? 0) + 1);
|
|
1659
|
+
seen.add(t);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
const N = docs.length;
|
|
1664
|
+
function idf(term) {
|
|
1665
|
+
const df = docFreq.get(term) ?? 0;
|
|
1666
|
+
return Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
1667
|
+
}
|
|
1668
|
+
/**
|
|
1669
|
+
* BM25+ score for a term in a document.
|
|
1670
|
+
* Classic BM25: TF_norm = tf*(k1+1) / (tf + k1*(1 - b + b*dl/avgdl))
|
|
1671
|
+
* BM25+ adds: + δ (guarantees long docs with the term still score positively)
|
|
1672
|
+
*/
|
|
1673
|
+
function bm25PlusTerm(term, doc) {
|
|
1674
|
+
const tf = doc.tokenFreq.get(term) ?? 0;
|
|
1675
|
+
if (tf === 0)
|
|
1676
|
+
return 0;
|
|
1677
|
+
const dl = doc.tokens.length;
|
|
1678
|
+
const tfNorm = (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * (1 - BM25_B + BM25_B * (dl / avgDL)));
|
|
1679
|
+
return idf(term) * (tfNorm + BM25_DELTA);
|
|
1680
|
+
}
|
|
1681
|
+
/**
|
|
1682
|
+
* Fuzzy match: tries exact → prefix → substring → Levenshtein ≤ 2.
|
|
1683
|
+
* Returns [matchedTerm, weight] or null.
|
|
1684
|
+
*
|
|
1685
|
+
* Weights:
|
|
1686
|
+
* 1.0 — exact match
|
|
1687
|
+
* 0.85 — doc token starts with query (e.g. "dockerf" → "dockerfile")
|
|
1688
|
+
* 0.75 — query starts with doc token (e.g. "kubernetes" → "kube")
|
|
1689
|
+
* 0.6 — substring (either direction)
|
|
1690
|
+
* 0.4 — Levenshtein ≤ 2 (typo tolerance)
|
|
1691
|
+
*/
|
|
1692
|
+
function fuzzyMatch(qt, docTermSet) {
|
|
1693
|
+
// Exact
|
|
1694
|
+
if (docTermSet.has(qt))
|
|
1695
|
+
return [qt, 1.0];
|
|
1696
|
+
// Prefix: query is prefix of a doc token (user typed partial word)
|
|
1697
|
+
if (qt.length >= 4) {
|
|
1698
|
+
for (const dt of docTermSet) {
|
|
1699
|
+
if (dt.length > qt.length && dt.startsWith(qt))
|
|
1700
|
+
return [dt, 0.85];
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
// Reverse-prefix: doc token is prefix of query (doc has abbreviated form)
|
|
1704
|
+
if (qt.length >= 4) {
|
|
1705
|
+
for (const dt of docTermSet) {
|
|
1706
|
+
if (dt.length >= 4 && dt.length < qt.length && qt.startsWith(dt))
|
|
1707
|
+
return [dt, 0.75];
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
// Substring (partial)
|
|
1711
|
+
for (const dt of docTermSet) {
|
|
1712
|
+
if (dt.length > 3 && qt.length > 3 && (dt.includes(qt) || qt.includes(dt))) {
|
|
1713
|
+
return [dt, 0.6];
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
// Levenshtein ≤ 2 (typo tolerance) — only for tokens ≥ 4 chars
|
|
1717
|
+
if (qt.length >= 4) {
|
|
1718
|
+
for (const dt of docTermSet) {
|
|
1719
|
+
if (dt.length >= 4 && levenshtein(qt, dt) <= 2) {
|
|
1720
|
+
return [dt, 0.4];
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
return null;
|
|
1725
|
+
}
|
|
1726
|
+
// ── Step 3: Score each sub-query independently ──
|
|
1727
|
+
const subQueries = splitMultiQuery(query);
|
|
1728
|
+
const allMatches = new Map();
|
|
1729
|
+
for (const sq of subQueries) {
|
|
1730
|
+
const queryTokens = tokenize(sq);
|
|
1731
|
+
if (queryTokens.length === 0)
|
|
1732
|
+
continue;
|
|
1733
|
+
// Pre-compute query bigrams for proximity boost
|
|
1734
|
+
const queryBigrams = new Set();
|
|
1735
|
+
for (let j = 0; j < queryTokens.length - 1; j++) {
|
|
1736
|
+
queryBigrams.add(`${queryTokens[j]}|${queryTokens[j + 1]}`);
|
|
1737
|
+
}
|
|
1738
|
+
for (const doc of docs) {
|
|
1739
|
+
let score = 0;
|
|
1740
|
+
const matchedWords = [];
|
|
1741
|
+
const docTermSet = new Set(doc.tokens);
|
|
1742
|
+
for (const qt of queryTokens) {
|
|
1743
|
+
// Try exact BM25+ first
|
|
1744
|
+
const exactScore = bm25PlusTerm(qt, doc);
|
|
1745
|
+
if (exactScore > 0) {
|
|
1746
|
+
score += exactScore;
|
|
1747
|
+
matchedWords.push(qt);
|
|
1748
|
+
continue;
|
|
1749
|
+
}
|
|
1750
|
+
// Fuzzy match (substring or Levenshtein)
|
|
1751
|
+
const fuzz = fuzzyMatch(qt, docTermSet);
|
|
1752
|
+
if (fuzz) {
|
|
1753
|
+
score += bm25PlusTerm(fuzz[0], doc) * fuzz[1];
|
|
1754
|
+
matchedWords.push(`~${qt}`);
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
// Bigram proximity boost: +50% for each adjacent query term pair found in doc
|
|
1758
|
+
if (queryBigrams.size > 0) {
|
|
1759
|
+
let bigramHits = 0;
|
|
1760
|
+
for (const bg of queryBigrams) {
|
|
1761
|
+
if (doc.bigrams.has(bg))
|
|
1762
|
+
bigramHits++;
|
|
1763
|
+
}
|
|
1764
|
+
if (bigramHits > 0) {
|
|
1765
|
+
score *= 1 + 0.5 * (bigramHits / queryBigrams.size);
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
// Phrase-match boost: if the raw (lowercased) query appears verbatim in content, 2× boost
|
|
1769
|
+
// This rewards docs where the exact phrase exists (vs scattered tokens).
|
|
1770
|
+
if (sq.length >= 4 && doc.content.toLowerCase().includes(sq.toLowerCase())) {
|
|
1771
|
+
score *= 2.0;
|
|
1772
|
+
}
|
|
1773
|
+
// Key/title boost: if query terms appear in the Redis key (=title), 1.5× boost
|
|
1774
|
+
// Keys often encode the primary topic (e.g. "deploy:api:server"), so key hits are high-precision.
|
|
1775
|
+
if (score > 0) {
|
|
1776
|
+
let keyHits = 0;
|
|
1777
|
+
for (const qt of queryTokens) {
|
|
1778
|
+
if (doc.keyTokens.has(qt))
|
|
1779
|
+
keyHits++;
|
|
1780
|
+
}
|
|
1781
|
+
if (keyHits > 0) {
|
|
1782
|
+
score *= 1 + 0.5 * (keyHits / queryTokens.length);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
// Recency boost
|
|
1786
|
+
score *= recencyBoost(doc.timestamp);
|
|
1787
|
+
if (score > 0) {
|
|
1788
|
+
const existing = allMatches.get(doc.key);
|
|
1789
|
+
if (!existing || score > existing.score) {
|
|
1790
|
+
allMatches.set(doc.key, {
|
|
1791
|
+
key: doc.key,
|
|
1792
|
+
content: doc.content,
|
|
1793
|
+
score,
|
|
1794
|
+
matchedWords: [...new Set(matchedWords)],
|
|
1795
|
+
subQuery: subQueries.length > 1 ? sq : undefined,
|
|
1796
|
+
});
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
// ── Step 4: Sort by score, return top-K ──
|
|
1802
|
+
const sorted = [...allMatches.values()].sort((a, b) => b.score - a.score);
|
|
1803
|
+
const topResults = sorted.slice(0, topK);
|
|
1804
|
+
// ── Step 5: Zero-results logging + Did-You-Mean ───────────────────────────
|
|
1805
|
+
if (topResults.length === 0) {
|
|
1806
|
+
logZeroResult(query);
|
|
1807
|
+
// Rebuild index vocab for Did-You-Mean suggestions
|
|
1808
|
+
_indexVocab = new Set();
|
|
1809
|
+
for (const doc of docs)
|
|
1810
|
+
for (const t of doc.tokens)
|
|
1811
|
+
_indexVocab.add(t);
|
|
1812
|
+
}
|
|
1813
|
+
return topResults;
|
|
1814
|
+
}
|
|
1815
|
+
// ── Exported for testing ──────────────────────────────────────────────────────
|
|
1816
|
+
export { tokenize, splitMultiQuery, levenshtein, recencyBoost, extractTimestamp, STOPWORDS, katakanaToRomaji, arabicLightStem, expandCrossLingual, CROSS_LINGUAL_MAP };
|
|
1817
|
+
export { keywordSearch, ZERO_RESULTS_LOG, logZeroResult, _indexVocab as indexVocab, zeroResultsTotal };
|
|
1818
|
+
//# sourceMappingURL=search.js.map
|