natural 7.0.3 → 7.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/npm-publish.yml +2 -0
- package/.nycrc +9 -0
- package/package.json +3 -4
- package/tsconfig.json +1 -1
- package/ts_spec/MaxEntClassifier_spec.ts +0 -130
- package/ts_spec/SentimentAnalyzer_spec.ts +0 -232
- package/ts_spec/WordPunctTokenizer_spec.ts +0 -57
- package/ts_spec/aggressive_tokenizer_de_spec.ts +0 -36
- package/ts_spec/aggressive_tokenizer_es_spec.ts +0 -39
- package/ts_spec/aggressive_tokenizer_fr_spec.ts +0 -111
- package/ts_spec/aggressive_tokenizer_hi_spec.ts +0 -70
- package/ts_spec/aggressive_tokenizer_nl_spec.ts +0 -39
- package/ts_spec/aggressive_tokenizer_pt_spec.ts +0 -48
- package/ts_spec/aggressive_tokenizer_spec.ts +0 -57
- package/ts_spec/aggressive_tokenizer_sv_spec.ts +0 -52
- package/ts_spec/aggressive_tokenizer_vi_spec.ts +0 -48
- package/ts_spec/bayes_classifier_spec.ts +0 -165
- package/ts_spec/brill_pos_tagger_spec.ts +0 -79
- package/ts_spec/brill_pos_trainer_spec.ts +0 -107
- package/ts_spec/carry_stemmer_fr_spec.ts +0 -97
- package/ts_spec/classifier_spec.ts +0 -104
- package/ts_spec/count_inflector_fr_spec.ts +0 -47
- package/ts_spec/count_inflector_spec.ts +0 -76
- package/ts_spec/damerau_levenshtein_spec.ts +0 -53
- package/ts_spec/dice_coefficient_spec.ts +0 -37
- package/ts_spec/double_metaphone_spec.ts +0 -739
- package/ts_spec/hamming_distance_spec.ts +0 -66
- package/ts_spec/jaro-winkler_spec.ts +0 -79
- package/ts_spec/lancaster_stemmer_spec.ts +0 -63
- package/ts_spec/levenshtein_spec.ts +0 -107
- package/ts_spec/logistic_regression_classifier_spec.ts +0 -93
- package/ts_spec/longest_path_tree_spec.ts +0 -111
- package/ts_spec/metaphone_spec.ts +0 -258
- package/ts_spec/ngram_spec.ts +0 -145
- package/ts_spec/ngram_zh_spec.ts +0 -124
- package/ts_spec/normalizer_ja_spec.ts +0 -165
- package/ts_spec/normalizer_no_spec.ts +0 -112
- package/ts_spec/normalizer_spec.ts +0 -141
- package/ts_spec/noun_inflector_fr_spec.ts +0 -472
- package/ts_spec/noun_inflector_ja_spec.ts +0 -116
- package/ts_spec/noun_inflector_spec.ts +0 -369
- package/ts_spec/orthography_tokenizer_spec.ts +0 -47
- package/ts_spec/porter_stemmer_de_spec.ts +0 -53
- package/ts_spec/porter_stemmer_es_spec.ts +0 -63
- package/ts_spec/porter_stemmer_fr_spec.ts +0 -103
- package/ts_spec/porter_stemmer_it_spec.ts +0 -56
- package/ts_spec/porter_stemmer_nl_spec.ts +0 -73
- package/ts_spec/porter_stemmer_no_spec.ts +0 -148
- package/ts_spec/porter_stemmer_pt_spec.ts +0 -50
- package/ts_spec/porter_stemmer_ru_spec.ts +0 -67
- package/ts_spec/porter_stemmer_spec.ts +0 -202
- package/ts_spec/porter_stemmer_sv_spec.ts +0 -41
- package/ts_spec/present_verb_inflector_spec.ts +0 -127
- package/ts_spec/remove_diacritics_spec.ts +0 -30
- package/ts_spec/sentence_analyzer_spec.ts +0 -293
- package/ts_spec/sentence_tokenizer_parser_spec.ts +0 -200
- package/ts_spec/sentence_tokenizer_spec.ts +0 -189
- package/ts_spec/shortest_path_tree_spec.ts +0 -141
- package/ts_spec/soundex_spec.ts +0 -162
- package/ts_spec/spellcheck_spec.ts +0 -112
- package/ts_spec/stemmer_id_spec.ts +0 -374
- package/ts_spec/stemmer_ja_spec.ts +0 -69
- package/ts_spec/stemmer_token_spec.ts +0 -163
- package/ts_spec/tfidf_spec.ts +0 -275
- package/ts_spec/tokenizer_case_spec.ts +0 -242
- package/ts_spec/tokenizer_ja_spec.ts +0 -63
- package/ts_spec/transliterator_ja_spec.ts +0 -105
- package/ts_spec/treebank_word_tokenizer_spec.ts +0 -38
- package/ts_spec/trie_spec.ts +0 -197
|
@@ -1,374 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
Copyright (c) 2018
|
|
3
|
-
|
|
4
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
5
|
-
of this softwé and associated documentation files (the "Softwé"), to deal
|
|
6
|
-
in the Softwé without restriction, including without limitation the rights
|
|
7
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
-
copies of the Softwé, and to permit persons to whom the Softwé is
|
|
9
|
-
furnished to do so, subject to the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be included in
|
|
12
|
-
all copies or substantial portions of the Softwé.
|
|
13
|
-
|
|
14
|
-
THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
19
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
|
|
20
|
-
THE SOFTWARE.
|
|
21
|
-
*/
|
|
22
|
-
|
|
23
|
-
'use strict'
|
|
24
|
-
|
|
25
|
-
// Adapted from original Mocha test script in tests folder
|
|
26
|
-
|
|
27
|
-
import { StemmerId as Stemmer } from 'lib/natural'
|
|
28
|
-
|
|
29
|
-
declare type Tests = Array<[string, string]>
|
|
30
|
-
const data: Tests = []
|
|
31
|
-
|
|
32
|
-
// THE TESTCASES
|
|
33
|
-
// don't stem short words
|
|
34
|
-
data.push(['mei', 'mei'])
|
|
35
|
-
data.push(['bui', 'bui'])
|
|
36
|
-
|
|
37
|
-
// lookup up the dictionary, to prevent overstemming
|
|
38
|
-
// don't stem nilai to nila
|
|
39
|
-
data.push(['nilai', 'nilai'])
|
|
40
|
-
|
|
41
|
-
// lah|kah|tah|pun
|
|
42
|
-
data.push(['hancurlah', 'hancur'])
|
|
43
|
-
// data.push(['benarkah', 'benar']);
|
|
44
|
-
// data.push(['apatah', 'apa']);
|
|
45
|
-
// data.push(['siapapun', 'siapa']);
|
|
46
|
-
|
|
47
|
-
// ku|mu|nya
|
|
48
|
-
data.push(['jubahku', 'jubah'])
|
|
49
|
-
data.push(['bajumu', 'baju'])
|
|
50
|
-
data.push(['celananya', 'celana'])
|
|
51
|
-
|
|
52
|
-
// i|kan|an
|
|
53
|
-
data.push(['hantui', 'hantu'])
|
|
54
|
-
// data.push(['belikan', 'beli']);
|
|
55
|
-
data.push(['jualan', 'jual'])
|
|
56
|
-
|
|
57
|
-
// combination of suffixes
|
|
58
|
-
data.push(['bukumukah', 'buku']) // gagal karena -ku dianggap suffix dan dihilangkan
|
|
59
|
-
data.push(['miliknyalah', 'milik'])
|
|
60
|
-
data.push(['kulitkupun', 'kulit']) // gagal karena -ku dianggap suffix dan dihilangkan
|
|
61
|
-
data.push(['berikanku', 'beri'])
|
|
62
|
-
data.push(['sakitimu', 'sakit'])
|
|
63
|
-
data.push(['beriannya', 'beri'])
|
|
64
|
-
data.push(['kasihilah', 'kasih'])
|
|
65
|
-
|
|
66
|
-
// plain prefix
|
|
67
|
-
data.push(['dibuang', 'buang'])
|
|
68
|
-
data.push(['kesakitan', 'sakit'])
|
|
69
|
-
data.push(['sesuap', 'suap'])
|
|
70
|
-
|
|
71
|
-
// data.push(['teriakanmu', 'teriak']); // wtf? kok jadi ria?
|
|
72
|
-
// teriakanmu -> te-ria-kan-mu
|
|
73
|
-
|
|
74
|
-
// template formulas for derivation prefix rules (disambiguation) //
|
|
75
|
-
|
|
76
|
-
// rule 1a : berV -> ber-V
|
|
77
|
-
data.push(['beradu', 'adu'])
|
|
78
|
-
|
|
79
|
-
// rule 1b : berV -> be-rV
|
|
80
|
-
data.push(['berambut', 'rambut'])
|
|
81
|
-
|
|
82
|
-
// rule 2 : berCAP -> ber-CAP
|
|
83
|
-
data.push(['bersuara', 'suara'])
|
|
84
|
-
|
|
85
|
-
// rule 3 : berCAerV -> ber-CAerV where C != 'r'
|
|
86
|
-
data.push(['berdaerah', 'daerah'])
|
|
87
|
-
|
|
88
|
-
// rule 4 : belajar -> bel-ajar
|
|
89
|
-
data.push(['belajar', 'ajar'])
|
|
90
|
-
|
|
91
|
-
// rule 5 : beC1erC2 -> be-C1erC2 where C1 != {'r'|'l'}
|
|
92
|
-
// data.push(['bekerja', 'kerja']);
|
|
93
|
-
data.push(['beternak', 'ternak'])
|
|
94
|
-
|
|
95
|
-
// rule 6a : terV -> ter-V
|
|
96
|
-
data.push(['terasing', 'asing'])
|
|
97
|
-
|
|
98
|
-
// rule 6b : terV -> te-rV
|
|
99
|
-
data.push(['teraup', 'raup'])
|
|
100
|
-
|
|
101
|
-
// rule 7 : terCerV -> ter-CerV where C != 'r'
|
|
102
|
-
data.push(['tergerak', 'gerak'])
|
|
103
|
-
|
|
104
|
-
// rule 8 : terCP -> ter-CP where C != 'r' and P != 'er'
|
|
105
|
-
data.push(['terpuruk', 'puruk'])
|
|
106
|
-
|
|
107
|
-
// rule 9 : teC1erC2 -> te-C1erC2 where C1 != 'r'
|
|
108
|
-
data.push(['teterbang', 'terbang'])
|
|
109
|
-
|
|
110
|
-
// rule 10 : me{l|r|w|y}V -> me-{l|r|w|y}V
|
|
111
|
-
data.push(['melipat', 'lipat'])
|
|
112
|
-
data.push(['meringkas', 'ringkas'])
|
|
113
|
-
data.push(['mewarnai', 'warna'])
|
|
114
|
-
// data.push(['meyakinkan', 'yakin']);
|
|
115
|
-
|
|
116
|
-
// rule 11 : mem{b|f|v} -> mem-{b|f|v}
|
|
117
|
-
data.push(['membangun', 'bangun'])
|
|
118
|
-
data.push(['memfitnah', 'fitnah'])
|
|
119
|
-
data.push(['memvonis', 'vonis'])
|
|
120
|
-
|
|
121
|
-
// rule 12 : mempe{r|l} -> mem-pe
|
|
122
|
-
data.push(['memperbarui', 'baru'])
|
|
123
|
-
data.push(['mempelajari', 'ajar'])
|
|
124
|
-
|
|
125
|
-
// rule 13a : mem{rV|V} -> mem{rV|V}
|
|
126
|
-
data.push(['meminum', 'minum'])
|
|
127
|
-
|
|
128
|
-
// rule 13b : mem{rV|V} -> me-p{rV|V}
|
|
129
|
-
data.push(['memukul', 'pukul'])
|
|
130
|
-
|
|
131
|
-
// rule 14 : men{c|d|j|z} -> men-{c|d|j|z}
|
|
132
|
-
data.push(['mencinta', 'cinta'])
|
|
133
|
-
data.push(['mendua', 'dua'])
|
|
134
|
-
data.push(['menjauh', 'jauh'])
|
|
135
|
-
data.push(['menziarah', 'ziarah'])
|
|
136
|
-
|
|
137
|
-
// rule 15a : men{V} -> me-n{V}
|
|
138
|
-
data.push(['menuklir', 'nuklir'])
|
|
139
|
-
|
|
140
|
-
// rule 15b : men{V} -> me-t{V}
|
|
141
|
-
data.push(['menangkap', 'tangkap'])
|
|
142
|
-
|
|
143
|
-
// rule 16 : meng{g|h|q} -> meng-{g|h|q}
|
|
144
|
-
data.push(['menggila', 'gila'])
|
|
145
|
-
data.push(['menghajar', 'hajar'])
|
|
146
|
-
data.push(['mengqasar', 'qasar'])
|
|
147
|
-
|
|
148
|
-
// rule 17a : mengV -> meng-V
|
|
149
|
-
data.push(['mengudara', 'udara'])
|
|
150
|
-
|
|
151
|
-
// rule 17b : mengV -> meng-kV
|
|
152
|
-
data.push(['mengupas', 'kupas'])
|
|
153
|
-
|
|
154
|
-
// rule 18 : menyV -> meny-sV
|
|
155
|
-
data.push(['menyuarakan', 'suara'])
|
|
156
|
-
|
|
157
|
-
// rule 19 : mempV -> mem-pV where V != 'e'
|
|
158
|
-
data.push(['mempopulerkan', 'populer'])
|
|
159
|
-
|
|
160
|
-
// rule 20 : pe{w|y}V -> pe-{w|y}V
|
|
161
|
-
data.push(['pewarna', 'warna'])
|
|
162
|
-
data.push(['peyoga', 'yoga'])
|
|
163
|
-
|
|
164
|
-
// rule 21a : perV -> per-V
|
|
165
|
-
data.push(['peradilan', 'adil'])
|
|
166
|
-
|
|
167
|
-
// rule 21b : perV -> pe-rV
|
|
168
|
-
data.push(['perumahan', 'rumah'])
|
|
169
|
-
|
|
170
|
-
// rule 22 is missing in the document?
|
|
171
|
-
|
|
172
|
-
// rule 23 : perCAP -> per-CAP where C != 'r' and P != 'er'
|
|
173
|
-
data.push(['permuka', 'muka'])
|
|
174
|
-
|
|
175
|
-
// rule 24 : perCAerV -> per-CAerV where C != 'r'
|
|
176
|
-
data.push(['perdaerah', 'daerah'])
|
|
177
|
-
|
|
178
|
-
// rule 25 : pem{b|f|v} -> pem-{b|f|v}
|
|
179
|
-
data.push(['pembangun', 'bangun'])
|
|
180
|
-
data.push(['pemfitnah', 'fitnah'])
|
|
181
|
-
data.push(['pemvonis', 'vonis'])
|
|
182
|
-
|
|
183
|
-
// rule 26a : pem{rV|V} -> pe-m{rV|V}
|
|
184
|
-
data.push(['peminum', 'minum'])
|
|
185
|
-
|
|
186
|
-
// rule 26b : pem{rV|V} -> pe-p{rV|V}
|
|
187
|
-
data.push(['pemukul', 'pukul'])
|
|
188
|
-
|
|
189
|
-
// rule 27 : men{c|d|j|z} -> men-{c|d|j|z}
|
|
190
|
-
data.push(['pencinta', 'cinta'])
|
|
191
|
-
data.push(['pendahulu', 'dahulu'])
|
|
192
|
-
data.push(['penjarah', 'jarah'])
|
|
193
|
-
data.push(['penziarah', 'ziarah'])
|
|
194
|
-
|
|
195
|
-
// rule 28a : pen{V} -> pe-n{V}
|
|
196
|
-
data.push(['penasihat', 'nasihat'])
|
|
197
|
-
|
|
198
|
-
// rule 28b : pen{V} -> pe-t{V}
|
|
199
|
-
data.push(['penangkap', 'tangkap'])
|
|
200
|
-
|
|
201
|
-
// rule 29 : peng{g|h|q} -> peng-{g|h|q}
|
|
202
|
-
data.push(['penggila', 'gila'])
|
|
203
|
-
data.push(['penghajar', 'hajar'])
|
|
204
|
-
data.push(['pengqasar', 'qasar'])
|
|
205
|
-
|
|
206
|
-
// rule 30a : pengV -> peng-V
|
|
207
|
-
data.push(['pengudara', 'udara'])
|
|
208
|
-
|
|
209
|
-
// rule 30b : pengV -> peng-kV
|
|
210
|
-
data.push(['pengupas', 'kupas'])
|
|
211
|
-
|
|
212
|
-
// rule 31 : penyV -> peny-sV
|
|
213
|
-
data.push(['penyuara', 'suara'])
|
|
214
|
-
|
|
215
|
-
// rule 32 : pelV -> pe-lV except pelajar -> ajar
|
|
216
|
-
data.push(['pelajar', 'ajar'])
|
|
217
|
-
data.push(['pelabuhan', 'labuh'])
|
|
218
|
-
|
|
219
|
-
// rule 33 : peCerV -> per-erV where C != {r|w|y|l|m|n}
|
|
220
|
-
// TODO : find the examples
|
|
221
|
-
|
|
222
|
-
// rule 34 : peCP -> pe-CP where C != {r|w|y|l|m|n} and P != 'er'
|
|
223
|
-
data.push(['petarung', 'tarung'])
|
|
224
|
-
|
|
225
|
-
// CS additional rules
|
|
226
|
-
|
|
227
|
-
// rule 35 : terC1erC2 -> ter-C1erC2 where C1 != 'r'
|
|
228
|
-
data.push(['terpercaya', 'percaya'])
|
|
229
|
-
|
|
230
|
-
// rule 36 : peC1erC2 -> pe-C1erC2 where C1 != {r|w|y|l|m|n}
|
|
231
|
-
data.push(['pekerja', 'kerja'])
|
|
232
|
-
data.push(['peserta', 'serta'])
|
|
233
|
-
|
|
234
|
-
// CS modify rule 12
|
|
235
|
-
data.push(['mempengaruhi', 'pengaruh'])
|
|
236
|
-
|
|
237
|
-
// CS modify rule 16
|
|
238
|
-
data.push(['mengkritik', 'kritik'])
|
|
239
|
-
|
|
240
|
-
// CS adjusting rule precedence
|
|
241
|
-
data.push(['bersekolah', 'sekolah']) // gagal sekolah -> seko why?
|
|
242
|
-
data.push(['bertahan', 'tahan'])
|
|
243
|
-
data.push(['mencapai', 'capai']) // gagal mencapai -> capa
|
|
244
|
-
// data.push(['dimulai', 'mulai']);
|
|
245
|
-
data.push(['petani', 'tani']) // gagal petani -> petan
|
|
246
|
-
data.push(['terabai', 'abai']) // gagal terabai -> aba
|
|
247
|
-
|
|
248
|
-
// ECS
|
|
249
|
-
data.push(['mensyaratkan', 'syarat'])
|
|
250
|
-
data.push(['mensyukuri', 'syukur'])
|
|
251
|
-
data.push(['mengebom', 'bom'])
|
|
252
|
-
data.push(['mempromosikan', 'promosi'])
|
|
253
|
-
data.push(['memproteksi', 'proteksi'])
|
|
254
|
-
data.push(['memprediksi', 'prediksi'])
|
|
255
|
-
data.push(['pengkajian', 'kaji'])
|
|
256
|
-
data.push(['pengebom', 'bom'])
|
|
257
|
-
|
|
258
|
-
// ECS loop pengembalian akhiran
|
|
259
|
-
data.push(['bersembunyi', 'sembunyi'])
|
|
260
|
-
data.push(['bersembunyilah', 'sembunyi'])
|
|
261
|
-
data.push(['pelanggan', 'langgan'])
|
|
262
|
-
data.push(['pelaku', 'laku'])
|
|
263
|
-
data.push(['pelangganmukah', 'langgan'])
|
|
264
|
-
data.push(['pelakunyalah', 'laku'])
|
|
265
|
-
|
|
266
|
-
data.push(['perbaikan', 'baik'])
|
|
267
|
-
data.push(['kebaikannya', 'baik'])
|
|
268
|
-
data.push(['bisikan', 'bisik'])
|
|
269
|
-
// data.push(['menerangi', 'terang']);
|
|
270
|
-
// data.push(['berimanlah', 'iman']);
|
|
271
|
-
|
|
272
|
-
// data.push(['memuaskan', 'puas']);
|
|
273
|
-
data.push(['berpelanggan', 'langgan'])
|
|
274
|
-
data.push(['bermakanan', 'makan'])
|
|
275
|
-
|
|
276
|
-
// CC (Modified ECS)
|
|
277
|
-
data.push(['menyala', 'nyala'])
|
|
278
|
-
data.push(['menyanyikan', 'nyanyi'])
|
|
279
|
-
data.push(['menyatakannya', 'nyata'])
|
|
280
|
-
|
|
281
|
-
data.push(['penyanyi', 'nyanyi'])
|
|
282
|
-
data.push(['penyawaan', 'nyawa'])
|
|
283
|
-
|
|
284
|
-
// CC infix
|
|
285
|
-
// data.push(['rerata', 'rata']);
|
|
286
|
-
// data.push(['lelembut', 'lembut']);
|
|
287
|
-
data.push(['lemigas', 'ligas'])
|
|
288
|
-
data.push(['kinerja', 'kerja'])
|
|
289
|
-
|
|
290
|
-
// plurals
|
|
291
|
-
data.push(['buku-buku', 'buku'])
|
|
292
|
-
data.push(['berbalas-balasan', 'balas'])
|
|
293
|
-
data.push(['bolak-balik', 'bolak-balik'])
|
|
294
|
-
|
|
295
|
-
// combination of prefix + suffix
|
|
296
|
-
data.push(['bertebaran', 'tebar'])
|
|
297
|
-
data.push(['terasingkan', 'asing'])
|
|
298
|
-
data.push(['membangunkan', 'bangun'])
|
|
299
|
-
data.push(['mencintai', 'cinta'])
|
|
300
|
-
data.push(['menduakan', 'dua'])
|
|
301
|
-
data.push(['menjauhi', 'jauh'])
|
|
302
|
-
data.push(['menggilai', 'gila'])
|
|
303
|
-
data.push(['pembangunan', 'bangun'])
|
|
304
|
-
|
|
305
|
-
// return the word if not found in the dictionary
|
|
306
|
-
data.push(['marwan', 'marwan'])
|
|
307
|
-
data.push(['subarkah', 'subarkah'])
|
|
308
|
-
|
|
309
|
-
// recursively remove prefix
|
|
310
|
-
data.push(['memberdayakan', 'daya'])
|
|
311
|
-
data.push(['persemakmuran', 'makmur'])
|
|
312
|
-
data.push(['keberuntunganmu', 'untung'])
|
|
313
|
-
data.push(['kesepersepuluhnya', 'sepuluh'])
|
|
314
|
-
|
|
315
|
-
// test stem sentence
|
|
316
|
-
// data.push(['siapakah memberdayakan pembangunan', 'siapa daya bangun']);
|
|
317
|
-
|
|
318
|
-
// issues
|
|
319
|
-
data.push(['Perekonomian', 'ekonomi'])
|
|
320
|
-
data.push(['menahan', 'tahan'])
|
|
321
|
-
|
|
322
|
-
// test stem multiple sentences
|
|
323
|
-
// var multipleSentence1 = 'Cinta telah bertebaran.Keduanya saling mencintai.';
|
|
324
|
-
// var multipleSentence2 = "(Cinta telah bertebaran)\n\n\n\nKeduanya saling mencintai.";
|
|
325
|
-
// data.push([multipleSentence1, 'cinta telah tebar dua saling cinta']);
|
|
326
|
-
// data.push([multipleSentence2, 'cinta telah tebar dua saling cinta']);
|
|
327
|
-
|
|
328
|
-
// failed on other method / algorithm but we should succeed
|
|
329
|
-
data.push(['peranan', 'peran'])
|
|
330
|
-
// data.push(['memberikan', 'beri']);
|
|
331
|
-
data.push(['medannya', 'medan'])
|
|
332
|
-
|
|
333
|
-
// TODO:
|
|
334
|
-
// data.push(['sebagai', 'bagai']);
|
|
335
|
-
// data.push(['bagian', 'bagian']);
|
|
336
|
-
data.push(['berbadan', 'badan'])
|
|
337
|
-
data.push(['abdullah', 'abdullah'])
|
|
338
|
-
|
|
339
|
-
// adopted foreign suffixes
|
|
340
|
-
// data.push(['budayawan', 'budaya']);
|
|
341
|
-
// data.push(['karyawati', 'karya']);
|
|
342
|
-
// data.push(['idealis', 'ideal']);
|
|
343
|
-
// data.push(['idealisme', 'ideal']);
|
|
344
|
-
data.push(['finalisasi', 'final'])
|
|
345
|
-
|
|
346
|
-
// sastrawi additional rules
|
|
347
|
-
data.push(['penstabilan', 'stabil'])
|
|
348
|
-
data.push(['pentranskripsi', 'transkripsi'])
|
|
349
|
-
|
|
350
|
-
data.push(['mentaati', 'taat'])
|
|
351
|
-
data.push(['meniru-nirukan', 'tiru'])
|
|
352
|
-
data.push(['menyepak-nyepak', 'sepak'])
|
|
353
|
-
|
|
354
|
-
data.push(['melewati', 'lewat'])
|
|
355
|
-
data.push(['menganga', 'nganga'])
|
|
356
|
-
|
|
357
|
-
data.push(['kupukul', 'pukul'])
|
|
358
|
-
data.push(['kauhajar', 'hajar'])
|
|
359
|
-
|
|
360
|
-
data.push(['kuasa-Mu', 'kuasa'])
|
|
361
|
-
data.push(['malaikat-malaikat-Nya', 'malaikat'])
|
|
362
|
-
data.push(['nikmat-Ku', 'nikmat'])
|
|
363
|
-
data.push(['allah-lah', 'allah'])
|
|
364
|
-
|
|
365
|
-
describe('Indonesian stemmer', function () {
|
|
366
|
-
data.forEach(function (testPair) {
|
|
367
|
-
it('should correctly tokenize and stem ' + testPair[0] + ' to ' + testPair[1], function () {
|
|
368
|
-
const output = Stemmer.tokenizeAndStem(testPair[0])
|
|
369
|
-
const outputConcat = output.join(' ')
|
|
370
|
-
// console.log(output);
|
|
371
|
-
expect(outputConcat).toEqual(testPair[1])
|
|
372
|
-
})
|
|
373
|
-
})
|
|
374
|
-
})
|
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
Copyright (c) 2012, Guillaume Marty
|
|
3
|
-
|
|
4
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
5
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
6
|
-
in the Software without restriction, including without limitation the rights
|
|
7
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
9
|
-
furnished to do so, subject to the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be included in
|
|
12
|
-
all copies or substantial portions of the Software.
|
|
13
|
-
|
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
19
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
20
|
-
THE SOFTWARE.
|
|
21
|
-
*/
|
|
22
|
-
|
|
23
|
-
'use strict'
|
|
24
|
-
|
|
25
|
-
import { StemmerJa as stemmer } from 'lib/natural'
|
|
26
|
-
|
|
27
|
-
const test = ['コピー', 'コーヒー', 'タクシー', 'パーティー', 'パーティ', 'ヘルプ・センター']
|
|
28
|
-
const testResult = ['コピー', 'コーヒ', 'タクシ', 'パーティ', 'パーティ', 'ヘルプ・センタ']
|
|
29
|
-
const text = '明後日パーティーに行く予定がある。図書館で資料をコピーしました。'
|
|
30
|
-
|
|
31
|
-
describe('StemmerJa', function () {
|
|
32
|
-
it('should stem words', function () {
|
|
33
|
-
for (let i = 0; i < test.length; i++) {
|
|
34
|
-
expect(stemmer.stem(test[i])).toBe(testResult[i])
|
|
35
|
-
}
|
|
36
|
-
})
|
|
37
|
-
|
|
38
|
-
it('should not tokenize halfwidth katakana', function () {
|
|
39
|
-
expect(stemmer.stem('タクシー')).toEqual('タクシー')
|
|
40
|
-
})
|
|
41
|
-
|
|
42
|
-
it('should tokenize, stem and exclude stop words (default behavior)', function () {
|
|
43
|
-
const tokens = [
|
|
44
|
-
'明後日', 'パーティ', '行く', '予定', // パーティー should be stemmed
|
|
45
|
-
'図書館', '資料', 'コピー', 'まし']
|
|
46
|
-
expect(stemmer.tokenizeAndStem(text)).toEqual(tokens)
|
|
47
|
-
expect(stemmer.tokenizeAndStem(text, false)).toEqual(tokens)
|
|
48
|
-
})
|
|
49
|
-
|
|
50
|
-
it('should tokenize, stem and keep stop words', function () {
|
|
51
|
-
expect(stemmer.tokenizeAndStem(text, true)).toEqual([
|
|
52
|
-
'明後日', 'パーティ', 'に', '行く', '予定', 'が', 'ある',
|
|
53
|
-
'図書館', 'で', '資料', 'を', 'コピー', 'し', 'まし', 'た'])
|
|
54
|
-
})
|
|
55
|
-
|
|
56
|
-
/*
|
|
57
|
-
it('should attach new methods to String', function() {
|
|
58
|
-
stemmer.attach();
|
|
59
|
-
expect('コーヒー'.stem()).toEqual('コーヒ');
|
|
60
|
-
expect('コピー'.stem()).toEqual('コピー');
|
|
61
|
-
expect('図書館で資料をコピーしました。'.tokenizeAndStem()).toEqual([
|
|
62
|
-
'図書館', '資料', 'コピー', 'まし']);
|
|
63
|
-
expect('図書館で資料をコピーしました。'.tokenizeAndStem(false)).toEqual([
|
|
64
|
-
'図書館', '資料', 'コピー', 'まし']);
|
|
65
|
-
expect('図書館で資料をコピーしました。'.tokenizeAndStem(true)).toEqual([
|
|
66
|
-
'図書館', 'で', '資料', 'を', 'コピー', 'し', 'まし', 'た']);
|
|
67
|
-
});
|
|
68
|
-
*/
|
|
69
|
-
})
|
|
@@ -1,163 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
Copyright (c) 2014, Luís Rodrigues
|
|
3
|
-
|
|
4
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
5
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
6
|
-
in the Software without restriction, including without limitation the rights
|
|
7
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
9
|
-
furnished to do so, subject to the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be included in
|
|
12
|
-
all copies or substantial portions of the Software.
|
|
13
|
-
|
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
19
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
20
|
-
THE SOFTWARE.
|
|
21
|
-
*/
|
|
22
|
-
|
|
23
|
-
'use strict'
|
|
24
|
-
|
|
25
|
-
import { Token } from 'lib/natural'
|
|
26
|
-
|
|
27
|
-
describe('stemmer token', function () {
|
|
28
|
-
it('should receive a string', function () {
|
|
29
|
-
const string = 'test'
|
|
30
|
-
const token = new Token(string)
|
|
31
|
-
|
|
32
|
-
expect(token.string).toBe(string)
|
|
33
|
-
})
|
|
34
|
-
|
|
35
|
-
it('should hold the original token string', function () {
|
|
36
|
-
const string = 'test'
|
|
37
|
-
const token = new Token(string)
|
|
38
|
-
|
|
39
|
-
expect(token.original).toBe(string)
|
|
40
|
-
})
|
|
41
|
-
|
|
42
|
-
it('should replace all instances of a string', function () {
|
|
43
|
-
const string = 'tester'
|
|
44
|
-
const token = new Token(string)
|
|
45
|
-
|
|
46
|
-
token.replaceAll('e', 'a')
|
|
47
|
-
expect(token.string).toBe('tastar')
|
|
48
|
-
|
|
49
|
-
token.replaceAll('ar', 'er')
|
|
50
|
-
expect(token.string).toBe('taster')
|
|
51
|
-
|
|
52
|
-
token.replaceAll('r', '')
|
|
53
|
-
expect(token.string).toBe('taste')
|
|
54
|
-
})
|
|
55
|
-
|
|
56
|
-
it('should allow vowels to be set', function () {
|
|
57
|
-
const vowels = 'aeiou'
|
|
58
|
-
const vowelsArray = vowels.split('')
|
|
59
|
-
const token = new Token('')
|
|
60
|
-
|
|
61
|
-
token.usingVowels(vowels)
|
|
62
|
-
expect(token.vowels).toBe(vowels)
|
|
63
|
-
|
|
64
|
-
token.usingVowels(vowelsArray)
|
|
65
|
-
expect(token.vowels).toBe(vowelsArray)
|
|
66
|
-
})
|
|
67
|
-
|
|
68
|
-
it('should check for vowels', function () {
|
|
69
|
-
const vowels = 'aeiou'
|
|
70
|
-
const token = new Token('test')
|
|
71
|
-
|
|
72
|
-
token.usingVowels(vowels)
|
|
73
|
-
|
|
74
|
-
expect(token.hasVowelAtIndex(0)).toBe(false)
|
|
75
|
-
expect(token.hasVowelAtIndex(1)).toBe(true)
|
|
76
|
-
expect(token.hasVowelAtIndex(99)).toBe(false)
|
|
77
|
-
|
|
78
|
-
token.usingVowels(vowels.split(''))
|
|
79
|
-
|
|
80
|
-
expect(token.hasVowelAtIndex(0)).toBe(false)
|
|
81
|
-
expect(token.hasVowelAtIndex(1)).toBe(true)
|
|
82
|
-
expect(token.hasVowelAtIndex(99)).toBe(false)
|
|
83
|
-
})
|
|
84
|
-
|
|
85
|
-
it('should find the next vowel', function () {
|
|
86
|
-
const token = new Token('tester').usingVowels('aeiou')
|
|
87
|
-
|
|
88
|
-
expect(token.nextVowelIndex(0)).toBe(1)
|
|
89
|
-
expect(token.nextVowelIndex(1)).toBe(1)
|
|
90
|
-
expect(token.nextVowelIndex(2)).toBe(4)
|
|
91
|
-
expect(token.nextVowelIndex(5)).toBe(6)
|
|
92
|
-
expect(token.nextVowelIndex(99)).toBe(6)
|
|
93
|
-
expect(token.nextVowelIndex(-1)).toBe(6)
|
|
94
|
-
})
|
|
95
|
-
|
|
96
|
-
it('should find the next consonant', function () {
|
|
97
|
-
const token = new Token('testee').usingVowels('aeiou')
|
|
98
|
-
|
|
99
|
-
expect(token.nextConsonantIndex(0)).toBe(0)
|
|
100
|
-
expect(token.nextConsonantIndex(1)).toBe(2)
|
|
101
|
-
expect(token.nextConsonantIndex(5)).toBe(6)
|
|
102
|
-
expect(token.nextConsonantIndex(99)).toBe(6)
|
|
103
|
-
expect(token.nextConsonantIndex(-1)).toBe(6)
|
|
104
|
-
})
|
|
105
|
-
|
|
106
|
-
it('should mark regions', function () {
|
|
107
|
-
const token = new Token('tester')
|
|
108
|
-
|
|
109
|
-
token.markRegion('test', 1)
|
|
110
|
-
expect(token.regions.test).toBe(1)
|
|
111
|
-
})
|
|
112
|
-
|
|
113
|
-
it('should mark regions with a callback', function () {
|
|
114
|
-
const token = new Token('tester')
|
|
115
|
-
const context = { value: 99 }
|
|
116
|
-
|
|
117
|
-
token.markRegion('test', 1, function (arg) { return arg })
|
|
118
|
-
expect(token.regions.test).toBe(1)
|
|
119
|
-
|
|
120
|
-
token.markRegion('test', [1], function (arg) { return arg })
|
|
121
|
-
expect(token.regions.test).toBe(1)
|
|
122
|
-
|
|
123
|
-
token.markRegion('test', [1, 1], function (a1, a2) { return a1 + a2 })
|
|
124
|
-
expect(token.regions.test).toBe(2)
|
|
125
|
-
|
|
126
|
-
token.markRegion('test', null, function (this: Token) {
|
|
127
|
-
return this.string.length
|
|
128
|
-
})
|
|
129
|
-
expect(token.regions.test).toBe(6)
|
|
130
|
-
|
|
131
|
-
token.markRegion('test', null, function (this: any) { return this.value }, context)
|
|
132
|
-
expect(token.regions.test).toBe(99)
|
|
133
|
-
})
|
|
134
|
-
|
|
135
|
-
it('should check for suffixes', function () {
|
|
136
|
-
const token = new Token('tester')
|
|
137
|
-
|
|
138
|
-
expect(token.hasSuffix('er')).toBeTruthy()
|
|
139
|
-
expect(token.hasSuffix('st')).toBeFalsy()
|
|
140
|
-
})
|
|
141
|
-
|
|
142
|
-
it('should check for suffixes within a region', function () {
|
|
143
|
-
const token = new Token('tester').markRegion('region', 2)
|
|
144
|
-
|
|
145
|
-
expect(token.hasSuffixInRegion('st', 'region')).toBe(false)
|
|
146
|
-
expect(token.hasSuffixInRegion('ster', 'region')).toBe(true)
|
|
147
|
-
expect(token.hasSuffixInRegion('ester', 'region')).toBe(false)
|
|
148
|
-
})
|
|
149
|
-
|
|
150
|
-
it('should replace the suffix within a region', function () {
|
|
151
|
-
const t1 = new Token('tester').markRegion('region', 4)
|
|
152
|
-
const t2 = new Token('tester').markRegion('region', 0)
|
|
153
|
-
|
|
154
|
-
t1.replaceSuffixInRegion('ter', '<s>', 'region')
|
|
155
|
-
expect(t1.string).toBe('tester')
|
|
156
|
-
|
|
157
|
-
t1.replaceSuffixInRegion('er', '<s>', 'region')
|
|
158
|
-
expect(t1.string).toBe('test<s>')
|
|
159
|
-
|
|
160
|
-
t2.replaceSuffixInRegion('protester', '<s>', 'region')
|
|
161
|
-
expect(t2.string).toBe('tester')
|
|
162
|
-
})
|
|
163
|
-
})
|