@delali/narsil 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/dist/adapters/filesystem.d.ts +8 -0
- package/dist/adapters/filesystem.mjs +107 -0
- package/dist/adapters/indexeddb.d.ts +9 -0
- package/dist/adapters/indexeddb.mjs +153 -0
- package/dist/adapters/memory.d.ts +5 -0
- package/dist/adapters/memory.mjs +30 -0
- package/dist/adapters-XODT2F3h.d.ts +30 -0
- package/dist/chunk-5HXMVMSM.mjs +3147 -0
- package/dist/chunk-JKIBUPAY.mjs +42 -0
- package/dist/chunk-N37VMC62.mjs +18 -0
- package/dist/chunk-WYC635GI.mjs +335 -0
- package/dist/direct-executor-MQD7JRD4.mjs +3 -0
- package/dist/index.d.ts +386 -0
- package/dist/index.mjs +1099 -0
- package/dist/invalidation/broadcast-channel.d.ts +8 -0
- package/dist/invalidation/broadcast-channel.mjs +46 -0
- package/dist/invalidation/filesystem.d.ts +10 -0
- package/dist/invalidation/filesystem.mjs +159 -0
- package/dist/invalidation/noop.d.ts +5 -0
- package/dist/invalidation/noop.mjs +13 -0
- package/dist/language-ClUGLuoC.d.ts +13 -0
- package/dist/languages/arabic.d.ts +5 -0
- package/dist/languages/arabic.mjs +631 -0
- package/dist/languages/armenian.d.ts +5 -0
- package/dist/languages/armenian.mjs +306 -0
- package/dist/languages/bulgarian.d.ts +5 -0
- package/dist/languages/bulgarian.mjs +330 -0
- package/dist/languages/chinese.d.ts +5 -0
- package/dist/languages/chinese.mjs +250 -0
- package/dist/languages/dagbani.d.ts +5 -0
- package/dist/languages/dagbani.mjs +85 -0
- package/dist/languages/danish.d.ts +5 -0
- package/dist/languages/danish.mjs +224 -0
- package/dist/languages/dutch.d.ts +5 -0
- package/dist/languages/dutch.mjs +296 -0
- package/dist/languages/english.d.ts +5 -0
- package/dist/languages/english.mjs +1 -0
- package/dist/languages/ewe.d.ts +5 -0
- package/dist/languages/ewe.mjs +85 -0
- package/dist/languages/finnish.d.ts +5 -0
- package/dist/languages/finnish.mjs +536 -0
- package/dist/languages/french.d.ts +5 -0
- package/dist/languages/french.mjs +635 -0
- package/dist/languages/ga.d.ts +5 -0
- package/dist/languages/ga.mjs +69 -0
- package/dist/languages/german.d.ts +5 -0
- package/dist/languages/german.mjs +414 -0
- package/dist/languages/greek.d.ts +5 -0
- package/dist/languages/greek.mjs +888 -0
- package/dist/languages/hausa.d.ts +5 -0
- package/dist/languages/hausa.mjs +243 -0
- package/dist/languages/hindi.d.ts +5 -0
- package/dist/languages/hindi.mjs +333 -0
- package/dist/languages/hungarian.d.ts +5 -0
- package/dist/languages/hungarian.mjs +600 -0
- package/dist/languages/igbo.d.ts +5 -0
- package/dist/languages/igbo.mjs +96 -0
- package/dist/languages/indonesian.d.ts +5 -0
- package/dist/languages/indonesian.mjs +896 -0
- package/dist/languages/irish.d.ts +5 -0
- package/dist/languages/irish.mjs +268 -0
- package/dist/languages/italian.d.ts +5 -0
- package/dist/languages/italian.mjs +713 -0
- package/dist/languages/japanese.d.ts +5 -0
- package/dist/languages/japanese.mjs +210 -0
- package/dist/languages/nepali.d.ts +5 -0
- package/dist/languages/nepali.mjs +262 -0
- package/dist/languages/norwegian.d.ts +5 -0
- package/dist/languages/norwegian.mjs +274 -0
- package/dist/languages/portuguese.d.ts +5 -0
- package/dist/languages/portuguese.mjs +606 -0
- package/dist/languages/romanian.d.ts +5 -0
- package/dist/languages/romanian.mjs +634 -0
- package/dist/languages/russian.d.ts +5 -0
- package/dist/languages/russian.mjs +424 -0
- package/dist/languages/sanskrit.d.ts +5 -0
- package/dist/languages/sanskrit.mjs +155 -0
- package/dist/languages/serbian.d.ts +5 -0
- package/dist/languages/serbian.mjs +277 -0
- package/dist/languages/slovenian.d.ts +5 -0
- package/dist/languages/slovenian.mjs +522 -0
- package/dist/languages/spanish.d.ts +5 -0
- package/dist/languages/spanish.mjs +606 -0
- package/dist/languages/swahili.d.ts +5 -0
- package/dist/languages/swahili.mjs +212 -0
- package/dist/languages/swedish.d.ts +5 -0
- package/dist/languages/swedish.mjs +224 -0
- package/dist/languages/tamil.d.ts +5 -0
- package/dist/languages/tamil.mjs +186 -0
- package/dist/languages/turkish.d.ts +5 -0
- package/dist/languages/turkish.mjs +675 -0
- package/dist/languages/twi.d.ts +5 -0
- package/dist/languages/twi.mjs +84 -0
- package/dist/languages/ukrainian.d.ts +5 -0
- package/dist/languages/ukrainian.mjs +444 -0
- package/dist/languages/yoruba.d.ts +5 -0
- package/dist/languages/yoruba.mjs +103 -0
- package/dist/languages/zulu.d.ts +5 -0
- package/dist/languages/zulu.mjs +113 -0
- package/dist/workers/entry.d.ts +3 -0
- package/dist/workers/entry.mjs +106 -0
- package/package.json +87 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// src/languages/swahili.ts
|
|
2
|
+
var SUBJECT_PREFIXES = ["ni", "tu", "wa", "vi", "zi", "li", "ya", "ku"];
|
|
3
|
+
var SHORT_SUBJECT_PREFIXES = ["u", "a", "i", "m"];
|
|
4
|
+
var TENSE_MARKERS = ["na", "li", "ta", "me", "ki"];
|
|
5
|
+
var RELATIVE_MARKERS = ["cho", "vyo", "ye", "zo", "lo", "yo", "ko", "po", "mo"];
|
|
6
|
+
var NEGATIVE_PREFIXES = ["ha", "si"];
|
|
7
|
+
var NOUN_CLASS_PREFIXES = ["wa", "mi", "ki", "vi", "ma"];
|
|
8
|
+
var SHORT_NOUN_PREFIXES = ["m", "u"];
|
|
9
|
+
var LONG_SUFFIXES = ["isha", "ika", "ana", "iwa", "ewa", "aje", "ane", "eni", "ini"];
|
|
10
|
+
var SHORT_SUFFIXES = ["wa", "ia", "ea"];
|
|
11
|
+
var FINAL_VOWELS = ["a", "i", "e", "u"];
|
|
12
|
+
function tryRemovePrefix(word, prefix, minRemaining) {
|
|
13
|
+
if (word.startsWith(prefix) && word.length - prefix.length >= minRemaining) {
|
|
14
|
+
return word.slice(prefix.length);
|
|
15
|
+
}
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
function removePrefixes(word) {
|
|
19
|
+
let current = word;
|
|
20
|
+
for (const prefix of NEGATIVE_PREFIXES) {
|
|
21
|
+
const result = tryRemovePrefix(current, prefix, 3);
|
|
22
|
+
if (result !== null) {
|
|
23
|
+
current = result;
|
|
24
|
+
break;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
let subjectRemoved = false;
|
|
28
|
+
for (const prefix of SUBJECT_PREFIXES) {
|
|
29
|
+
const result = tryRemovePrefix(current, prefix, 3);
|
|
30
|
+
if (result !== null) {
|
|
31
|
+
current = result;
|
|
32
|
+
subjectRemoved = true;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (!subjectRemoved) {
|
|
37
|
+
for (const prefix of SHORT_SUBJECT_PREFIXES) {
|
|
38
|
+
const result = tryRemovePrefix(current, prefix, 3);
|
|
39
|
+
if (result !== null) {
|
|
40
|
+
current = result;
|
|
41
|
+
subjectRemoved = true;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (subjectRemoved) {
|
|
47
|
+
for (const marker of TENSE_MARKERS) {
|
|
48
|
+
const result = tryRemovePrefix(current, marker, 3);
|
|
49
|
+
if (result !== null) {
|
|
50
|
+
current = result;
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
for (const marker of RELATIVE_MARKERS) {
|
|
55
|
+
const result = tryRemovePrefix(current, marker, 3);
|
|
56
|
+
if (result !== null) {
|
|
57
|
+
current = result;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (!subjectRemoved) {
|
|
63
|
+
for (const prefix of NOUN_CLASS_PREFIXES) {
|
|
64
|
+
const result = tryRemovePrefix(current, prefix, 3);
|
|
65
|
+
if (result !== null) {
|
|
66
|
+
current = result;
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (current === word) {
|
|
71
|
+
for (const prefix of SHORT_NOUN_PREFIXES) {
|
|
72
|
+
const result = tryRemovePrefix(current, prefix, 3);
|
|
73
|
+
if (result !== null) {
|
|
74
|
+
current = result;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return current;
|
|
81
|
+
}
|
|
82
|
+
function removeSuffixes(word) {
|
|
83
|
+
for (const suffix of LONG_SUFFIXES) {
|
|
84
|
+
if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
|
|
85
|
+
return word.slice(0, -suffix.length);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const suffix of SHORT_SUFFIXES) {
|
|
89
|
+
if (word.endsWith(suffix) && word.length - suffix.length >= 3) {
|
|
90
|
+
return word.slice(0, -suffix.length);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
for (const vowel of FINAL_VOWELS) {
|
|
94
|
+
if (word.endsWith(vowel) && word.length - 1 >= 3) {
|
|
95
|
+
return word.slice(0, -1);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return word;
|
|
99
|
+
}
|
|
100
|
+
function stem(word) {
|
|
101
|
+
if (word.length < 4) return word;
|
|
102
|
+
let result = removePrefixes(word);
|
|
103
|
+
result = removeSuffixes(result);
|
|
104
|
+
return result;
|
|
105
|
+
}
|
|
106
|
+
var stopWords = /* @__PURE__ */ new Set([
|
|
107
|
+
"wa",
|
|
108
|
+
"na",
|
|
109
|
+
"ya",
|
|
110
|
+
"kwa",
|
|
111
|
+
"la",
|
|
112
|
+
"za",
|
|
113
|
+
"ni",
|
|
114
|
+
"au",
|
|
115
|
+
"katika",
|
|
116
|
+
"kama",
|
|
117
|
+
"lakini",
|
|
118
|
+
"pia",
|
|
119
|
+
"hii",
|
|
120
|
+
"yake",
|
|
121
|
+
"wake",
|
|
122
|
+
"yangu",
|
|
123
|
+
"yetu",
|
|
124
|
+
"yao",
|
|
125
|
+
"ile",
|
|
126
|
+
"hizo",
|
|
127
|
+
"hiyo",
|
|
128
|
+
"hivi",
|
|
129
|
+
"hivyo",
|
|
130
|
+
"huku",
|
|
131
|
+
"kila",
|
|
132
|
+
"bila",
|
|
133
|
+
"ama",
|
|
134
|
+
"bali",
|
|
135
|
+
"hata",
|
|
136
|
+
"ingawa",
|
|
137
|
+
"kwamba",
|
|
138
|
+
"kuwa",
|
|
139
|
+
"ili",
|
|
140
|
+
"mpaka",
|
|
141
|
+
"tangu",
|
|
142
|
+
"ndiyo",
|
|
143
|
+
"hapana",
|
|
144
|
+
"hapa",
|
|
145
|
+
"pale",
|
|
146
|
+
"sana",
|
|
147
|
+
"tu",
|
|
148
|
+
"hasa",
|
|
149
|
+
"kabisa",
|
|
150
|
+
"ndio",
|
|
151
|
+
"nini",
|
|
152
|
+
"gani",
|
|
153
|
+
"wapi",
|
|
154
|
+
"lini",
|
|
155
|
+
"vipi",
|
|
156
|
+
"nani",
|
|
157
|
+
"kwani",
|
|
158
|
+
"basi",
|
|
159
|
+
"sasa",
|
|
160
|
+
"tena",
|
|
161
|
+
"bado",
|
|
162
|
+
"tayari",
|
|
163
|
+
"labda",
|
|
164
|
+
"pengine",
|
|
165
|
+
"yote",
|
|
166
|
+
"wote",
|
|
167
|
+
"zote",
|
|
168
|
+
"kote",
|
|
169
|
+
"nyingi",
|
|
170
|
+
"mengi",
|
|
171
|
+
"wengi",
|
|
172
|
+
"chache",
|
|
173
|
+
"kidogo",
|
|
174
|
+
"zaidi",
|
|
175
|
+
"sawa",
|
|
176
|
+
"kweli",
|
|
177
|
+
"si",
|
|
178
|
+
"siyo",
|
|
179
|
+
"hakuna",
|
|
180
|
+
"wala",
|
|
181
|
+
"halafu",
|
|
182
|
+
"kisha",
|
|
183
|
+
"baada",
|
|
184
|
+
"kabla",
|
|
185
|
+
"wakati",
|
|
186
|
+
"mara",
|
|
187
|
+
"daima",
|
|
188
|
+
"kamwe",
|
|
189
|
+
"ikiwa",
|
|
190
|
+
"iwapo",
|
|
191
|
+
"endapo",
|
|
192
|
+
"ijapokuwa",
|
|
193
|
+
"kutoka",
|
|
194
|
+
"hadi",
|
|
195
|
+
"kuhusu",
|
|
196
|
+
"juu",
|
|
197
|
+
"chini",
|
|
198
|
+
"ndani",
|
|
199
|
+
"nje",
|
|
200
|
+
"mbele",
|
|
201
|
+
"nyuma",
|
|
202
|
+
"karibu",
|
|
203
|
+
"mbali"
|
|
204
|
+
]);
|
|
205
|
+
var swahili = {
|
|
206
|
+
name: "swahili",
|
|
207
|
+
stemmer: stem,
|
|
208
|
+
stopWords,
|
|
209
|
+
tokenizer: { splitPattern: /[^a-z0-9]+/gi }
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
export { swahili };
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// src/languages/swedish.ts
|
|
2
|
+
var VOWELS = "aeiouy\xE4\xE5\xF6";
|
|
3
|
+
function isVowel(ch) {
|
|
4
|
+
return VOWELS.includes(ch);
|
|
5
|
+
}
|
|
6
|
+
function computeR1(word) {
|
|
7
|
+
let i = 0;
|
|
8
|
+
while (i < word.length && !isVowel(word[i])) i++;
|
|
9
|
+
while (i < word.length && isVowel(word[i])) i++;
|
|
10
|
+
const r1 = i < word.length ? i + 1 : word.length;
|
|
11
|
+
return Math.max(r1, 3);
|
|
12
|
+
}
|
|
13
|
+
var S_ENDINGS = "bcdfghjklmnoprtvy";
|
|
14
|
+
var STEP1_SUFFIXES = [
|
|
15
|
+
"heterna",
|
|
16
|
+
"hetens",
|
|
17
|
+
"arnas",
|
|
18
|
+
"ernas",
|
|
19
|
+
"ornas",
|
|
20
|
+
"anden",
|
|
21
|
+
"heten",
|
|
22
|
+
"heter",
|
|
23
|
+
"andes",
|
|
24
|
+
"arens",
|
|
25
|
+
"andet",
|
|
26
|
+
"arna",
|
|
27
|
+
"erna",
|
|
28
|
+
"orna",
|
|
29
|
+
"ande",
|
|
30
|
+
"arne",
|
|
31
|
+
"aste",
|
|
32
|
+
"aren",
|
|
33
|
+
"ades",
|
|
34
|
+
"erns",
|
|
35
|
+
"ade",
|
|
36
|
+
"are",
|
|
37
|
+
"ern",
|
|
38
|
+
"ens",
|
|
39
|
+
"het",
|
|
40
|
+
"ast",
|
|
41
|
+
"ad",
|
|
42
|
+
"en",
|
|
43
|
+
"ar",
|
|
44
|
+
"er",
|
|
45
|
+
"or",
|
|
46
|
+
"as",
|
|
47
|
+
"es",
|
|
48
|
+
"at",
|
|
49
|
+
"a",
|
|
50
|
+
"e",
|
|
51
|
+
"s"
|
|
52
|
+
];
|
|
53
|
+
function step1(word, r1) {
|
|
54
|
+
for (const suffix of STEP1_SUFFIXES) {
|
|
55
|
+
if (!word.endsWith(suffix)) continue;
|
|
56
|
+
const boundary = word.length - suffix.length;
|
|
57
|
+
if (boundary < r1) continue;
|
|
58
|
+
if (suffix === "s") {
|
|
59
|
+
if (boundary > 0 && S_ENDINGS.includes(word[boundary - 1])) {
|
|
60
|
+
return word.slice(0, boundary);
|
|
61
|
+
}
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
return word.slice(0, boundary);
|
|
65
|
+
}
|
|
66
|
+
return word;
|
|
67
|
+
}
|
|
68
|
+
var STEP2_ENDINGS = ["dd", "gd", "nn", "dt", "gt", "kt", "tt"];
|
|
69
|
+
function step2(word, r1) {
|
|
70
|
+
for (const ending of STEP2_ENDINGS) {
|
|
71
|
+
if (!word.endsWith(ending)) continue;
|
|
72
|
+
if (word.length - 2 < r1) continue;
|
|
73
|
+
return word.slice(0, -1);
|
|
74
|
+
}
|
|
75
|
+
return word;
|
|
76
|
+
}
|
|
77
|
+
var STEP3_SUFFIXES = [
|
|
78
|
+
["fullt", "full"],
|
|
79
|
+
["l\xF6st", "l\xF6s"],
|
|
80
|
+
["lig", null],
|
|
81
|
+
["els", null],
|
|
82
|
+
["ig", null]
|
|
83
|
+
];
|
|
84
|
+
function step3(word, r1) {
|
|
85
|
+
for (const [suffix, replacement] of STEP3_SUFFIXES) {
|
|
86
|
+
if (!word.endsWith(suffix)) continue;
|
|
87
|
+
const boundary = word.length - suffix.length;
|
|
88
|
+
if (boundary < r1) continue;
|
|
89
|
+
return replacement ? word.slice(0, boundary) + replacement : word.slice(0, boundary);
|
|
90
|
+
}
|
|
91
|
+
return word;
|
|
92
|
+
}
|
|
93
|
+
function stem(word) {
|
|
94
|
+
if (word.length < 3) return word;
|
|
95
|
+
const r1 = computeR1(word);
|
|
96
|
+
word = step1(word, r1);
|
|
97
|
+
word = step2(word, r1);
|
|
98
|
+
word = step3(word, r1);
|
|
99
|
+
return word;
|
|
100
|
+
}
|
|
101
|
+
var stopWords = /* @__PURE__ */ new Set([
|
|
102
|
+
"och",
|
|
103
|
+
"det",
|
|
104
|
+
"att",
|
|
105
|
+
"i",
|
|
106
|
+
"en",
|
|
107
|
+
"jag",
|
|
108
|
+
"hon",
|
|
109
|
+
"som",
|
|
110
|
+
"han",
|
|
111
|
+
"p\xE5",
|
|
112
|
+
"den",
|
|
113
|
+
"med",
|
|
114
|
+
"var",
|
|
115
|
+
"sig",
|
|
116
|
+
"f\xF6r",
|
|
117
|
+
"s\xE5",
|
|
118
|
+
"till",
|
|
119
|
+
"\xE4r",
|
|
120
|
+
"men",
|
|
121
|
+
"ett",
|
|
122
|
+
"om",
|
|
123
|
+
"hade",
|
|
124
|
+
"de",
|
|
125
|
+
"av",
|
|
126
|
+
"icke",
|
|
127
|
+
"mig",
|
|
128
|
+
"du",
|
|
129
|
+
"henne",
|
|
130
|
+
"d\xE5",
|
|
131
|
+
"sin",
|
|
132
|
+
"nu",
|
|
133
|
+
"har",
|
|
134
|
+
"inte",
|
|
135
|
+
"hans",
|
|
136
|
+
"honom",
|
|
137
|
+
"skulle",
|
|
138
|
+
"hennes",
|
|
139
|
+
"d\xE4r",
|
|
140
|
+
"min",
|
|
141
|
+
"man",
|
|
142
|
+
"ej",
|
|
143
|
+
"vid",
|
|
144
|
+
"kunde",
|
|
145
|
+
"n\xE5got",
|
|
146
|
+
"fr\xE5n",
|
|
147
|
+
"ut",
|
|
148
|
+
"n\xE4r",
|
|
149
|
+
"efter",
|
|
150
|
+
"upp",
|
|
151
|
+
"vi",
|
|
152
|
+
"dem",
|
|
153
|
+
"vara",
|
|
154
|
+
"vad",
|
|
155
|
+
"\xF6ver",
|
|
156
|
+
"\xE4n",
|
|
157
|
+
"dig",
|
|
158
|
+
"kan",
|
|
159
|
+
"sina",
|
|
160
|
+
"h\xE4r",
|
|
161
|
+
"ha",
|
|
162
|
+
"mot",
|
|
163
|
+
"alla",
|
|
164
|
+
"under",
|
|
165
|
+
"n\xE5gon",
|
|
166
|
+
"eller",
|
|
167
|
+
"allt",
|
|
168
|
+
"mycket",
|
|
169
|
+
"sedan",
|
|
170
|
+
"ju",
|
|
171
|
+
"denna",
|
|
172
|
+
"sj\xE4lv",
|
|
173
|
+
"detta",
|
|
174
|
+
"\xE5t",
|
|
175
|
+
"utan",
|
|
176
|
+
"varit",
|
|
177
|
+
"hur",
|
|
178
|
+
"ingen",
|
|
179
|
+
"mitt",
|
|
180
|
+
"ni",
|
|
181
|
+
"bli",
|
|
182
|
+
"blev",
|
|
183
|
+
"oss",
|
|
184
|
+
"din",
|
|
185
|
+
"dessa",
|
|
186
|
+
"n\xE5gra",
|
|
187
|
+
"deras",
|
|
188
|
+
"blir",
|
|
189
|
+
"mina",
|
|
190
|
+
"samma",
|
|
191
|
+
"vilken",
|
|
192
|
+
"er",
|
|
193
|
+
"s\xE5dan",
|
|
194
|
+
"v\xE5r",
|
|
195
|
+
"blivit",
|
|
196
|
+
"dess",
|
|
197
|
+
"inom",
|
|
198
|
+
"mellan",
|
|
199
|
+
"s\xE5dant",
|
|
200
|
+
"varf\xF6r",
|
|
201
|
+
"varje",
|
|
202
|
+
"vilka",
|
|
203
|
+
"ditt",
|
|
204
|
+
"vem",
|
|
205
|
+
"vilket",
|
|
206
|
+
"sitta",
|
|
207
|
+
"s\xE5dana",
|
|
208
|
+
"vart",
|
|
209
|
+
"dina",
|
|
210
|
+
"vars",
|
|
211
|
+
"v\xE5rt",
|
|
212
|
+
"v\xE5ra",
|
|
213
|
+
"ert",
|
|
214
|
+
"era",
|
|
215
|
+
"vilkas"
|
|
216
|
+
]);
|
|
217
|
+
var swedish = {
|
|
218
|
+
name: "swedish",
|
|
219
|
+
stemmer: stem,
|
|
220
|
+
stopWords,
|
|
221
|
+
tokenizer: { splitPattern: /[^a-z0-9åäöü-]+/gi }
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
export { swedish };
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// src/languages/tamil.ts
|
|
2
|
+
var VERB_SUFFIXES = [
|
|
3
|
+
"\u0B95\u0BBF\u0BB1\u0BC0\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
4
|
+
"\u0B95\u0BBF\u0BB1\u0BBE\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
5
|
+
"\u0BA8\u0BCD\u0BA4\u0BC0\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
6
|
+
"\u0BA8\u0BCD\u0BA4\u0BBE\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
7
|
+
"\u0BB5\u0BC0\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
8
|
+
"\u0BB5\u0BBE\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
9
|
+
"\u0B95\u0BBF\u0BB1\u0BC7\u0BA9\u0BCD",
|
|
10
|
+
"\u0B95\u0BBF\u0BB1\u0BBE\u0BAF\u0BCD",
|
|
11
|
+
"\u0B95\u0BBF\u0BB1\u0BBE\u0BA9\u0BCD",
|
|
12
|
+
"\u0B95\u0BBF\u0BB1\u0BBE\u0BB3\u0BCD",
|
|
13
|
+
"\u0B95\u0BBF\u0BB1\u0BCB\u0BAE\u0BCD",
|
|
14
|
+
"\u0B95\u0BBF\u0BA9\u0BCD\u0BB1\u0BA9",
|
|
15
|
+
"\u0BA8\u0BCD\u0BA4\u0BC7\u0BA9\u0BCD",
|
|
16
|
+
"\u0BA8\u0BCD\u0BA4\u0BBE\u0BAF\u0BCD",
|
|
17
|
+
"\u0BA8\u0BCD\u0BA4\u0BBE\u0BA9\u0BCD",
|
|
18
|
+
"\u0BA8\u0BCD\u0BA4\u0BBE\u0BB3\u0BCD",
|
|
19
|
+
"\u0BA8\u0BCD\u0BA4\u0BCB\u0BAE\u0BCD",
|
|
20
|
+
"\u0B95\u0BBF\u0BB1\u0BA4\u0BC1",
|
|
21
|
+
"\u0BA8\u0BCD\u0BA4\u0BA4\u0BC1",
|
|
22
|
+
"\u0BB5\u0BC7\u0BA9\u0BCD",
|
|
23
|
+
"\u0BB5\u0BBE\u0BAF\u0BCD",
|
|
24
|
+
"\u0BB5\u0BBE\u0BA9\u0BCD",
|
|
25
|
+
"\u0BB5\u0BBE\u0BB3\u0BCD",
|
|
26
|
+
"\u0BB5\u0BCB\u0BAE\u0BCD",
|
|
27
|
+
"\u0BA8\u0BCD\u0BA4\u0BC1",
|
|
28
|
+
"\u0BA4\u0BCD\u0BA4\u0BC1",
|
|
29
|
+
"\u0BB5\u0BA4\u0BC1",
|
|
30
|
+
"\u0B95\u0BCD\u0B95",
|
|
31
|
+
"\u0BAF"
|
|
32
|
+
];
|
|
33
|
+
var CASE_SUFFIXES = [
|
|
34
|
+
"\u0B87\u0B9F\u0BAE\u0BBF\u0BB0\u0BC1\u0BA8\u0BCD\u0BA4\u0BC1",
|
|
35
|
+
"\u0B87\u0BB0\u0BC1\u0BA8\u0BCD\u0BA4\u0BC1",
|
|
36
|
+
"\u0B89\u0B95\u0BCD\u0B95\u0BC1",
|
|
37
|
+
"\u0B89\u0B9F\u0BC8\u0BAF",
|
|
38
|
+
"\u0B95\u0BCD\u0B95\u0BC1",
|
|
39
|
+
"\u0B87\u0B9F\u0BAE\u0BCD",
|
|
40
|
+
"\u0B89\u0B9F\u0BA9\u0BCD",
|
|
41
|
+
"\u0B86\u0BB2\u0BCD",
|
|
42
|
+
"\u0B92\u0B9F\u0BC1",
|
|
43
|
+
"\u0B87\u0BA9\u0BCD",
|
|
44
|
+
"\u0B85\u0BA4\u0BC1",
|
|
45
|
+
"\u0B87\u0BB2\u0BCD",
|
|
46
|
+
"\u0B95\u0BA3\u0BCD",
|
|
47
|
+
"\u0B93\u0B9F\u0BC1",
|
|
48
|
+
"\u0B90"
|
|
49
|
+
];
|
|
50
|
+
var POST_POSITIONS = ["\u0BAA\u0BB1\u0BCD\u0BB1\u0BBF", "\u0BAE\u0BC7\u0BB2\u0BCD", "\u0B95\u0BC0\u0BB4\u0BCD", "\u0BAE\u0BC0\u0BA4\u0BC1", "\u0BB5\u0BB0\u0BC8"];
|
|
51
|
+
function removeLongestSuffix(word, suffixes, minLength) {
|
|
52
|
+
for (const suffix of suffixes) {
|
|
53
|
+
if (word.endsWith(suffix) && word.length - suffix.length >= minLength) {
|
|
54
|
+
return word.slice(0, word.length - suffix.length);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return word;
|
|
58
|
+
}
|
|
59
|
+
function stem(word) {
|
|
60
|
+
if (word.length < 4) return word;
|
|
61
|
+
if (word.endsWith("\u0B95\u0BB3\u0BCD") && word.length - 3 >= 2) {
|
|
62
|
+
word = word.slice(0, word.length - 3);
|
|
63
|
+
}
|
|
64
|
+
word = removeLongestSuffix(word, CASE_SUFFIXES, 2);
|
|
65
|
+
word = removeLongestSuffix(word, VERB_SUFFIXES, 2);
|
|
66
|
+
word = removeLongestSuffix(word, POST_POSITIONS, 2);
|
|
67
|
+
return word;
|
|
68
|
+
}
|
|
69
|
+
var stopWords = /* @__PURE__ */ new Set([
|
|
70
|
+
"\u0B92\u0BB0\u0BC1",
|
|
71
|
+
"\u0B8E\u0BA9\u0BCD\u0BB1\u0BC1",
|
|
72
|
+
"\u0BAE\u0BB1\u0BCD\u0BB1\u0BC1\u0BAE\u0BCD",
|
|
73
|
+
"\u0B87\u0BA8\u0BCD\u0BA4",
|
|
74
|
+
"\u0B87\u0BA4\u0BC1",
|
|
75
|
+
"\u0B8E\u0BA9\u0BCD\u0BB1",
|
|
76
|
+
"\u0B95\u0BCA\u0BA3\u0BCD\u0B9F\u0BC1",
|
|
77
|
+
"\u0B8E\u0BA9\u0BCD\u0BAA\u0BA4\u0BC1",
|
|
78
|
+
"\u0BAA\u0BB2",
|
|
79
|
+
"\u0B86\u0B95\u0BC1\u0BAE\u0BCD",
|
|
80
|
+
"\u0B85\u0BB2\u0BCD\u0BB2\u0BA4\u0BC1",
|
|
81
|
+
"\u0B85\u0BB5\u0BB0\u0BCD",
|
|
82
|
+
"\u0BA8\u0BBE\u0BA9\u0BCD",
|
|
83
|
+
"\u0B89\u0BB3\u0BCD\u0BB3",
|
|
84
|
+
"\u0B85\u0BA8\u0BCD\u0BA4",
|
|
85
|
+
"\u0B87\u0BB5\u0BB0\u0BCD",
|
|
86
|
+
"\u0B8E\u0BA9",
|
|
87
|
+
"\u0BAE\u0BC1\u0BA4\u0BB2\u0BCD",
|
|
88
|
+
"\u0B8E\u0BA9\u0BCD\u0BA9",
|
|
89
|
+
"\u0B87\u0BB0\u0BC1\u0BA8\u0BCD\u0BA4\u0BC1",
|
|
90
|
+
"\u0B9A\u0BBF\u0BB2",
|
|
91
|
+
"\u0B8E\u0BA9\u0BCD",
|
|
92
|
+
"\u0BAA\u0BCB\u0BA9\u0BCD\u0BB1",
|
|
93
|
+
"\u0BB5\u0BC7\u0BA3\u0BCD\u0B9F\u0BC1\u0BAE\u0BCD",
|
|
94
|
+
"\u0BB5\u0BA8\u0BCD\u0BA4\u0BC1",
|
|
95
|
+
"\u0B87\u0BA4\u0BA9\u0BCD",
|
|
96
|
+
"\u0B85\u0BA4\u0BC1",
|
|
97
|
+
"\u0B85\u0BB5\u0BA9\u0BCD",
|
|
98
|
+
"\u0BA4\u0BBE\u0BA9\u0BCD",
|
|
99
|
+
"\u0B8E\u0BA9\u0BCD\u0BA9\u0BC1\u0BAE\u0BCD",
|
|
100
|
+
"\u0BAE\u0BC7\u0BB2\u0BC1\u0BAE\u0BCD",
|
|
101
|
+
"\u0BAA\u0BBF\u0BA9\u0BCD\u0BA9\u0BB0\u0BCD",
|
|
102
|
+
"\u0B95\u0BCA\u0BA3\u0BCD\u0B9F",
|
|
103
|
+
"\u0B87\u0BB0\u0BC1\u0B95\u0BCD\u0B95\u0BC1\u0BAE\u0BCD",
|
|
104
|
+
"\u0BA4\u0BA9\u0BA4\u0BC1",
|
|
105
|
+
"\u0B89\u0BB3\u0BCD\u0BB3\u0BA4\u0BC1",
|
|
106
|
+
"\u0BAA\u0BCB\u0BA4\u0BC1",
|
|
107
|
+
"\u0B8E\u0BA9\u0BCD\u0BB1\u0BC1\u0BAE\u0BCD",
|
|
108
|
+
"\u0B85\u0BA4\u0BA9\u0BCD",
|
|
109
|
+
"\u0BA4\u0BA9\u0BCD",
|
|
110
|
+
"\u0BAA\u0BBF\u0BB1\u0B95\u0BC1",
|
|
111
|
+
"\u0B85\u0BB5\u0BB0\u0BCD\u0B95\u0BB3\u0BCD",
|
|
112
|
+
"\u0BB5\u0BB0\u0BC8",
|
|
113
|
+
"\u0B85\u0BB5\u0BB3\u0BCD",
|
|
114
|
+
"\u0BA8\u0BC0",
|
|
115
|
+
"\u0B86\u0B95\u0BBF\u0BAF",
|
|
116
|
+
"\u0B89\u0BB3\u0BCD\u0BB3\u0BA9",
|
|
117
|
+
"\u0BB5\u0BA8\u0BCD\u0BA4",
|
|
118
|
+
"\u0B87\u0BB0\u0BC1\u0BA8\u0BCD\u0BA4",
|
|
119
|
+
"\u0BAE\u0BBF\u0B95\u0BB5\u0BC1\u0BAE\u0BCD",
|
|
120
|
+
"\u0B87\u0B99\u0BCD\u0B95\u0BC1",
|
|
121
|
+
"\u0BAE\u0BC0\u0BA4\u0BC1",
|
|
122
|
+
"\u0B93\u0BB0\u0BCD",
|
|
123
|
+
"\u0B87\u0BB5\u0BC8",
|
|
124
|
+
"\u0BAA\u0BB1\u0BCD\u0BB1\u0BBF",
|
|
125
|
+
"\u0BB5\u0BB0\u0BC1\u0BAE\u0BCD",
|
|
126
|
+
"\u0BB5\u0BC7\u0BB1\u0BC1",
|
|
127
|
+
"\u0B87\u0BB0\u0BC1",
|
|
128
|
+
"\u0B87\u0BA4\u0BBF\u0BB2\u0BCD",
|
|
129
|
+
"\u0BAA\u0BCB\u0BB2\u0BCD",
|
|
130
|
+
"\u0B87\u0BAA\u0BCD\u0BAA\u0BCB\u0BA4\u0BC1",
|
|
131
|
+
"\u0B85\u0BB5\u0BB0\u0BA4\u0BC1",
|
|
132
|
+
"\u0BAE\u0B9F\u0BCD\u0B9F\u0BC1\u0BAE\u0BCD",
|
|
133
|
+
"\u0B8E\u0BA9\u0BC1\u0BAE\u0BCD",
|
|
134
|
+
"\u0BAE\u0BC7\u0BB2\u0BCD",
|
|
135
|
+
"\u0BAA\u0BBF\u0BA9\u0BCD",
|
|
136
|
+
"\u0B86\u0B95\u0BBF\u0BAF\u0BCB\u0BB0\u0BCD",
|
|
137
|
+
"\u0B87\u0BA9\u0BCD\u0BA9\u0BC1\u0BAE\u0BCD",
|
|
138
|
+
"\u0B85\u0BA9\u0BCD\u0BB1\u0BC1",
|
|
139
|
+
"\u0B92\u0BB0\u0BC7",
|
|
140
|
+
"\u0BAE\u0BBF\u0B95",
|
|
141
|
+
"\u0B85\u0B99\u0BCD\u0B95\u0BC1",
|
|
142
|
+
"\u0BAA\u0BB2\u0BCD\u0BB5\u0BC7\u0BB1\u0BC1",
|
|
143
|
+
"\u0B85\u0BA4\u0BC8",
|
|
144
|
+
"\u0BAA\u0BB1\u0BCD\u0BB1\u0BBF\u0BAF",
|
|
145
|
+
"\u0B89\u0BA9\u0BCD",
|
|
146
|
+
"\u0B85\u0BA4\u0BBF\u0B95",
|
|
147
|
+
"\u0BAA\u0BC7\u0BB0\u0BCD",
|
|
148
|
+
"\u0B87\u0BA4\u0BA9\u0BBE\u0BB2\u0BCD",
|
|
149
|
+
"\u0B85\u0BB5\u0BC8",
|
|
150
|
+
"\u0B85\u0BA4\u0BC7",
|
|
151
|
+
"\u0B8F\u0BA9\u0BCD",
|
|
152
|
+
"\u0BAE\u0BC1\u0BB1\u0BC8",
|
|
153
|
+
"\u0BAF\u0BBE\u0BB0\u0BCD",
|
|
154
|
+
"\u0B8E\u0BA9\u0BCD\u0BAA\u0BA4\u0BC8",
|
|
155
|
+
"\u0B8E\u0BB2\u0BCD\u0BB2\u0BBE\u0BAE\u0BCD",
|
|
156
|
+
"\u0BAE\u0B9F\u0BCD\u0B9F\u0BC1\u0BAE\u0BC7",
|
|
157
|
+
"\u0B87\u0B99\u0BCD\u0B95\u0BC7",
|
|
158
|
+
"\u0B85\u0B99\u0BCD\u0B95\u0BC7",
|
|
159
|
+
"\u0B87\u0B9F\u0BAE\u0BCD",
|
|
160
|
+
"\u0B85\u0BA4\u0BBF\u0BB2\u0BCD",
|
|
161
|
+
"\u0BA8\u0BBE\u0BAE\u0BCD",
|
|
162
|
+
"\u0B85\u0BA4\u0BB1\u0BCD\u0B95\u0BC1",
|
|
163
|
+
"\u0B8E\u0BA9\u0BB5\u0BC7",
|
|
164
|
+
"\u0BAA\u0BBF\u0BB1",
|
|
165
|
+
"\u0B9A\u0BBF\u0BB1\u0BC1",
|
|
166
|
+
"\u0BAE\u0BB1\u0BCD\u0BB1",
|
|
167
|
+
"\u0BB5\u0BBF\u0B9F",
|
|
168
|
+
"\u0B8E\u0BA8\u0BCD\u0BA4",
|
|
169
|
+
"\u0B85\u0B9F\u0BC1\u0BA4\u0BCD\u0BA4",
|
|
170
|
+
"\u0B87\u0BA4\u0BA9\u0BC8",
|
|
171
|
+
"\u0B87\u0BA4\u0BC8",
|
|
172
|
+
"\u0B95\u0BCA\u0BB3\u0BCD\u0BB3",
|
|
173
|
+
"\u0B87\u0BA4\u0BB1\u0BCD\u0B95\u0BC1",
|
|
174
|
+
"\u0B85\u0BA4\u0BA9\u0BBE\u0BB2\u0BCD",
|
|
175
|
+
"\u0BA4\u0BB5\u0BBF\u0BB0",
|
|
176
|
+
"\u0BAA\u0BCB\u0BB2",
|
|
177
|
+
"\u0B9A\u0BB1\u0BCD\u0BB1\u0BC1"
|
|
178
|
+
]);
|
|
179
|
+
var tamil = {
|
|
180
|
+
name: "tamil",
|
|
181
|
+
stemmer: stem,
|
|
182
|
+
stopWords,
|
|
183
|
+
tokenizer: { splitPattern: /[^\u0B80-\u0BFFa-z0-9]+/gi }
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
export { tamil };
|