natural 7.0.3 → 7.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/.github/workflows/npm-publish.yml +2 -0
  2. package/.nycrc +9 -0
  3. package/package.json +3 -4
  4. package/tsconfig.json +1 -1
  5. package/ts_spec/MaxEntClassifier_spec.ts +0 -130
  6. package/ts_spec/SentimentAnalyzer_spec.ts +0 -232
  7. package/ts_spec/WordPunctTokenizer_spec.ts +0 -57
  8. package/ts_spec/aggressive_tokenizer_de_spec.ts +0 -36
  9. package/ts_spec/aggressive_tokenizer_es_spec.ts +0 -39
  10. package/ts_spec/aggressive_tokenizer_fr_spec.ts +0 -111
  11. package/ts_spec/aggressive_tokenizer_hi_spec.ts +0 -70
  12. package/ts_spec/aggressive_tokenizer_nl_spec.ts +0 -39
  13. package/ts_spec/aggressive_tokenizer_pt_spec.ts +0 -48
  14. package/ts_spec/aggressive_tokenizer_spec.ts +0 -57
  15. package/ts_spec/aggressive_tokenizer_sv_spec.ts +0 -52
  16. package/ts_spec/aggressive_tokenizer_vi_spec.ts +0 -48
  17. package/ts_spec/bayes_classifier_spec.ts +0 -165
  18. package/ts_spec/brill_pos_tagger_spec.ts +0 -79
  19. package/ts_spec/brill_pos_trainer_spec.ts +0 -107
  20. package/ts_spec/carry_stemmer_fr_spec.ts +0 -97
  21. package/ts_spec/classifier_spec.ts +0 -104
  22. package/ts_spec/count_inflector_fr_spec.ts +0 -47
  23. package/ts_spec/count_inflector_spec.ts +0 -76
  24. package/ts_spec/damerau_levenshtein_spec.ts +0 -53
  25. package/ts_spec/dice_coefficient_spec.ts +0 -37
  26. package/ts_spec/double_metaphone_spec.ts +0 -739
  27. package/ts_spec/hamming_distance_spec.ts +0 -66
  28. package/ts_spec/jaro-winkler_spec.ts +0 -79
  29. package/ts_spec/lancaster_stemmer_spec.ts +0 -63
  30. package/ts_spec/levenshtein_spec.ts +0 -107
  31. package/ts_spec/logistic_regression_classifier_spec.ts +0 -93
  32. package/ts_spec/longest_path_tree_spec.ts +0 -111
  33. package/ts_spec/metaphone_spec.ts +0 -258
  34. package/ts_spec/ngram_spec.ts +0 -145
  35. package/ts_spec/ngram_zh_spec.ts +0 -124
  36. package/ts_spec/normalizer_ja_spec.ts +0 -165
  37. package/ts_spec/normalizer_no_spec.ts +0 -112
  38. package/ts_spec/normalizer_spec.ts +0 -141
  39. package/ts_spec/noun_inflector_fr_spec.ts +0 -472
  40. package/ts_spec/noun_inflector_ja_spec.ts +0 -116
  41. package/ts_spec/noun_inflector_spec.ts +0 -369
  42. package/ts_spec/orthography_tokenizer_spec.ts +0 -47
  43. package/ts_spec/porter_stemmer_de_spec.ts +0 -53
  44. package/ts_spec/porter_stemmer_es_spec.ts +0 -63
  45. package/ts_spec/porter_stemmer_fr_spec.ts +0 -103
  46. package/ts_spec/porter_stemmer_it_spec.ts +0 -56
  47. package/ts_spec/porter_stemmer_nl_spec.ts +0 -73
  48. package/ts_spec/porter_stemmer_no_spec.ts +0 -148
  49. package/ts_spec/porter_stemmer_pt_spec.ts +0 -50
  50. package/ts_spec/porter_stemmer_ru_spec.ts +0 -67
  51. package/ts_spec/porter_stemmer_spec.ts +0 -202
  52. package/ts_spec/porter_stemmer_sv_spec.ts +0 -41
  53. package/ts_spec/present_verb_inflector_spec.ts +0 -127
  54. package/ts_spec/remove_diacritics_spec.ts +0 -30
  55. package/ts_spec/sentence_analyzer_spec.ts +0 -293
  56. package/ts_spec/sentence_tokenizer_parser_spec.ts +0 -200
  57. package/ts_spec/sentence_tokenizer_spec.ts +0 -189
  58. package/ts_spec/shortest_path_tree_spec.ts +0 -141
  59. package/ts_spec/soundex_spec.ts +0 -162
  60. package/ts_spec/spellcheck_spec.ts +0 -112
  61. package/ts_spec/stemmer_id_spec.ts +0 -374
  62. package/ts_spec/stemmer_ja_spec.ts +0 -69
  63. package/ts_spec/stemmer_token_spec.ts +0 -163
  64. package/ts_spec/tfidf_spec.ts +0 -275
  65. package/ts_spec/tokenizer_case_spec.ts +0 -242
  66. package/ts_spec/tokenizer_ja_spec.ts +0 -63
  67. package/ts_spec/transliterator_ja_spec.ts +0 -105
  68. package/ts_spec/treebank_word_tokenizer_spec.ts +0 -38
  69. package/ts_spec/trie_spec.ts +0 -197
@@ -1,374 +0,0 @@
1
- /*
2
- Copyright (c) 2018
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this softwé and associated documentation files (the "Softwé"), to deal
6
- in the Softwé without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Softwé, and to permit persons to whom the Softwé is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Softwé.
13
-
14
- THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- // Adapted from original Mocha test script in tests folder
26
-
27
- import { StemmerId as Stemmer } from 'lib/natural'
28
-
29
- declare type Tests = Array<[string, string]>
30
- const data: Tests = []
31
-
32
- // THE TESTCASES
33
- // don't stem short words
34
- data.push(['mei', 'mei'])
35
- data.push(['bui', 'bui'])
36
-
37
- // lookup up the dictionary, to prevent overstemming
38
- // don't stem nilai to nila
39
- data.push(['nilai', 'nilai'])
40
-
41
- // lah|kah|tah|pun
42
- data.push(['hancurlah', 'hancur'])
43
- // data.push(['benarkah', 'benar']);
44
- // data.push(['apatah', 'apa']);
45
- // data.push(['siapapun', 'siapa']);
46
-
47
- // ku|mu|nya
48
- data.push(['jubahku', 'jubah'])
49
- data.push(['bajumu', 'baju'])
50
- data.push(['celananya', 'celana'])
51
-
52
- // i|kan|an
53
- data.push(['hantui', 'hantu'])
54
- // data.push(['belikan', 'beli']);
55
- data.push(['jualan', 'jual'])
56
-
57
- // combination of suffixes
58
- data.push(['bukumukah', 'buku']) // gagal karena -ku dianggap suffix dan dihilangkan
59
- data.push(['miliknyalah', 'milik'])
60
- data.push(['kulitkupun', 'kulit']) // gagal karena -ku dianggap suffix dan dihilangkan
61
- data.push(['berikanku', 'beri'])
62
- data.push(['sakitimu', 'sakit'])
63
- data.push(['beriannya', 'beri'])
64
- data.push(['kasihilah', 'kasih'])
65
-
66
- // plain prefix
67
- data.push(['dibuang', 'buang'])
68
- data.push(['kesakitan', 'sakit'])
69
- data.push(['sesuap', 'suap'])
70
-
71
- // data.push(['teriakanmu', 'teriak']); // wtf? kok jadi ria?
72
- // teriakanmu -> te-ria-kan-mu
73
-
74
- // template formulas for derivation prefix rules (disambiguation) //
75
-
76
- // rule 1a : berV -> ber-V
77
- data.push(['beradu', 'adu'])
78
-
79
- // rule 1b : berV -> be-rV
80
- data.push(['berambut', 'rambut'])
81
-
82
- // rule 2 : berCAP -> ber-CAP
83
- data.push(['bersuara', 'suara'])
84
-
85
- // rule 3 : berCAerV -> ber-CAerV where C != 'r'
86
- data.push(['berdaerah', 'daerah'])
87
-
88
- // rule 4 : belajar -> bel-ajar
89
- data.push(['belajar', 'ajar'])
90
-
91
- // rule 5 : beC1erC2 -> be-C1erC2 where C1 != {'r'|'l'}
92
- // data.push(['bekerja', 'kerja']);
93
- data.push(['beternak', 'ternak'])
94
-
95
- // rule 6a : terV -> ter-V
96
- data.push(['terasing', 'asing'])
97
-
98
- // rule 6b : terV -> te-rV
99
- data.push(['teraup', 'raup'])
100
-
101
- // rule 7 : terCerV -> ter-CerV where C != 'r'
102
- data.push(['tergerak', 'gerak'])
103
-
104
- // rule 8 : terCP -> ter-CP where C != 'r' and P != 'er'
105
- data.push(['terpuruk', 'puruk'])
106
-
107
- // rule 9 : teC1erC2 -> te-C1erC2 where C1 != 'r'
108
- data.push(['teterbang', 'terbang'])
109
-
110
- // rule 10 : me{l|r|w|y}V -> me-{l|r|w|y}V
111
- data.push(['melipat', 'lipat'])
112
- data.push(['meringkas', 'ringkas'])
113
- data.push(['mewarnai', 'warna'])
114
- // data.push(['meyakinkan', 'yakin']);
115
-
116
- // rule 11 : mem{b|f|v} -> mem-{b|f|v}
117
- data.push(['membangun', 'bangun'])
118
- data.push(['memfitnah', 'fitnah'])
119
- data.push(['memvonis', 'vonis'])
120
-
121
- // rule 12 : mempe{r|l} -> mem-pe
122
- data.push(['memperbarui', 'baru'])
123
- data.push(['mempelajari', 'ajar'])
124
-
125
- // rule 13a : mem{rV|V} -> mem{rV|V}
126
- data.push(['meminum', 'minum'])
127
-
128
- // rule 13b : mem{rV|V} -> me-p{rV|V}
129
- data.push(['memukul', 'pukul'])
130
-
131
- // rule 14 : men{c|d|j|z} -> men-{c|d|j|z}
132
- data.push(['mencinta', 'cinta'])
133
- data.push(['mendua', 'dua'])
134
- data.push(['menjauh', 'jauh'])
135
- data.push(['menziarah', 'ziarah'])
136
-
137
- // rule 15a : men{V} -> me-n{V}
138
- data.push(['menuklir', 'nuklir'])
139
-
140
- // rule 15b : men{V} -> me-t{V}
141
- data.push(['menangkap', 'tangkap'])
142
-
143
- // rule 16 : meng{g|h|q} -> meng-{g|h|q}
144
- data.push(['menggila', 'gila'])
145
- data.push(['menghajar', 'hajar'])
146
- data.push(['mengqasar', 'qasar'])
147
-
148
- // rule 17a : mengV -> meng-V
149
- data.push(['mengudara', 'udara'])
150
-
151
- // rule 17b : mengV -> meng-kV
152
- data.push(['mengupas', 'kupas'])
153
-
154
- // rule 18 : menyV -> meny-sV
155
- data.push(['menyuarakan', 'suara'])
156
-
157
- // rule 19 : mempV -> mem-pV where V != 'e'
158
- data.push(['mempopulerkan', 'populer'])
159
-
160
- // rule 20 : pe{w|y}V -> pe-{w|y}V
161
- data.push(['pewarna', 'warna'])
162
- data.push(['peyoga', 'yoga'])
163
-
164
- // rule 21a : perV -> per-V
165
- data.push(['peradilan', 'adil'])
166
-
167
- // rule 21b : perV -> pe-rV
168
- data.push(['perumahan', 'rumah'])
169
-
170
- // rule 22 is missing in the document?
171
-
172
- // rule 23 : perCAP -> per-CAP where C != 'r' and P != 'er'
173
- data.push(['permuka', 'muka'])
174
-
175
- // rule 24 : perCAerV -> per-CAerV where C != 'r'
176
- data.push(['perdaerah', 'daerah'])
177
-
178
- // rule 25 : pem{b|f|v} -> pem-{b|f|v}
179
- data.push(['pembangun', 'bangun'])
180
- data.push(['pemfitnah', 'fitnah'])
181
- data.push(['pemvonis', 'vonis'])
182
-
183
- // rule 26a : pem{rV|V} -> pe-m{rV|V}
184
- data.push(['peminum', 'minum'])
185
-
186
- // rule 26b : pem{rV|V} -> pe-p{rV|V}
187
- data.push(['pemukul', 'pukul'])
188
-
189
- // rule 27 : men{c|d|j|z} -> men-{c|d|j|z}
190
- data.push(['pencinta', 'cinta'])
191
- data.push(['pendahulu', 'dahulu'])
192
- data.push(['penjarah', 'jarah'])
193
- data.push(['penziarah', 'ziarah'])
194
-
195
- // rule 28a : pen{V} -> pe-n{V}
196
- data.push(['penasihat', 'nasihat'])
197
-
198
- // rule 28b : pen{V} -> pe-t{V}
199
- data.push(['penangkap', 'tangkap'])
200
-
201
- // rule 29 : peng{g|h|q} -> peng-{g|h|q}
202
- data.push(['penggila', 'gila'])
203
- data.push(['penghajar', 'hajar'])
204
- data.push(['pengqasar', 'qasar'])
205
-
206
- // rule 30a : pengV -> peng-V
207
- data.push(['pengudara', 'udara'])
208
-
209
- // rule 30b : pengV -> peng-kV
210
- data.push(['pengupas', 'kupas'])
211
-
212
- // rule 31 : penyV -> peny-sV
213
- data.push(['penyuara', 'suara'])
214
-
215
- // rule 32 : pelV -> pe-lV except pelajar -> ajar
216
- data.push(['pelajar', 'ajar'])
217
- data.push(['pelabuhan', 'labuh'])
218
-
219
- // rule 33 : peCerV -> per-erV where C != {r|w|y|l|m|n}
220
- // TODO : find the examples
221
-
222
- // rule 34 : peCP -> pe-CP where C != {r|w|y|l|m|n} and P != 'er'
223
- data.push(['petarung', 'tarung'])
224
-
225
- // CS additional rules
226
-
227
- // rule 35 : terC1erC2 -> ter-C1erC2 where C1 != 'r'
228
- data.push(['terpercaya', 'percaya'])
229
-
230
- // rule 36 : peC1erC2 -> pe-C1erC2 where C1 != {r|w|y|l|m|n}
231
- data.push(['pekerja', 'kerja'])
232
- data.push(['peserta', 'serta'])
233
-
234
- // CS modify rule 12
235
- data.push(['mempengaruhi', 'pengaruh'])
236
-
237
- // CS modify rule 16
238
- data.push(['mengkritik', 'kritik'])
239
-
240
- // CS adjusting rule precedence
241
- data.push(['bersekolah', 'sekolah']) // gagal sekolah -> seko why?
242
- data.push(['bertahan', 'tahan'])
243
- data.push(['mencapai', 'capai']) // gagal mencapai -> capa
244
- // data.push(['dimulai', 'mulai']);
245
- data.push(['petani', 'tani']) // gagal petani -> petan
246
- data.push(['terabai', 'abai']) // gagal terabai -> aba
247
-
248
- // ECS
249
- data.push(['mensyaratkan', 'syarat'])
250
- data.push(['mensyukuri', 'syukur'])
251
- data.push(['mengebom', 'bom'])
252
- data.push(['mempromosikan', 'promosi'])
253
- data.push(['memproteksi', 'proteksi'])
254
- data.push(['memprediksi', 'prediksi'])
255
- data.push(['pengkajian', 'kaji'])
256
- data.push(['pengebom', 'bom'])
257
-
258
- // ECS loop pengembalian akhiran
259
- data.push(['bersembunyi', 'sembunyi'])
260
- data.push(['bersembunyilah', 'sembunyi'])
261
- data.push(['pelanggan', 'langgan'])
262
- data.push(['pelaku', 'laku'])
263
- data.push(['pelangganmukah', 'langgan'])
264
- data.push(['pelakunyalah', 'laku'])
265
-
266
- data.push(['perbaikan', 'baik'])
267
- data.push(['kebaikannya', 'baik'])
268
- data.push(['bisikan', 'bisik'])
269
- // data.push(['menerangi', 'terang']);
270
- // data.push(['berimanlah', 'iman']);
271
-
272
- // data.push(['memuaskan', 'puas']);
273
- data.push(['berpelanggan', 'langgan'])
274
- data.push(['bermakanan', 'makan'])
275
-
276
- // CC (Modified ECS)
277
- data.push(['menyala', 'nyala'])
278
- data.push(['menyanyikan', 'nyanyi'])
279
- data.push(['menyatakannya', 'nyata'])
280
-
281
- data.push(['penyanyi', 'nyanyi'])
282
- data.push(['penyawaan', 'nyawa'])
283
-
284
- // CC infix
285
- // data.push(['rerata', 'rata']);
286
- // data.push(['lelembut', 'lembut']);
287
- data.push(['lemigas', 'ligas'])
288
- data.push(['kinerja', 'kerja'])
289
-
290
- // plurals
291
- data.push(['buku-buku', 'buku'])
292
- data.push(['berbalas-balasan', 'balas'])
293
- data.push(['bolak-balik', 'bolak-balik'])
294
-
295
- // combination of prefix + suffix
296
- data.push(['bertebaran', 'tebar'])
297
- data.push(['terasingkan', 'asing'])
298
- data.push(['membangunkan', 'bangun'])
299
- data.push(['mencintai', 'cinta'])
300
- data.push(['menduakan', 'dua'])
301
- data.push(['menjauhi', 'jauh'])
302
- data.push(['menggilai', 'gila'])
303
- data.push(['pembangunan', 'bangun'])
304
-
305
- // return the word if not found in the dictionary
306
- data.push(['marwan', 'marwan'])
307
- data.push(['subarkah', 'subarkah'])
308
-
309
- // recursively remove prefix
310
- data.push(['memberdayakan', 'daya'])
311
- data.push(['persemakmuran', 'makmur'])
312
- data.push(['keberuntunganmu', 'untung'])
313
- data.push(['kesepersepuluhnya', 'sepuluh'])
314
-
315
- // test stem sentence
316
- // data.push(['siapakah memberdayakan pembangunan', 'siapa daya bangun']);
317
-
318
- // issues
319
- data.push(['Perekonomian', 'ekonomi'])
320
- data.push(['menahan', 'tahan'])
321
-
322
- // test stem multiple sentences
323
- // var multipleSentence1 = 'Cinta telah bertebaran.Keduanya saling mencintai.';
324
- // var multipleSentence2 = "(Cinta telah bertebaran)\n\n\n\nKeduanya saling mencintai.";
325
- // data.push([multipleSentence1, 'cinta telah tebar dua saling cinta']);
326
- // data.push([multipleSentence2, 'cinta telah tebar dua saling cinta']);
327
-
328
- // failed on other method / algorithm but we should succeed
329
- data.push(['peranan', 'peran'])
330
- // data.push(['memberikan', 'beri']);
331
- data.push(['medannya', 'medan'])
332
-
333
- // TODO:
334
- // data.push(['sebagai', 'bagai']);
335
- // data.push(['bagian', 'bagian']);
336
- data.push(['berbadan', 'badan'])
337
- data.push(['abdullah', 'abdullah'])
338
-
339
- // adopted foreign suffixes
340
- // data.push(['budayawan', 'budaya']);
341
- // data.push(['karyawati', 'karya']);
342
- // data.push(['idealis', 'ideal']);
343
- // data.push(['idealisme', 'ideal']);
344
- data.push(['finalisasi', 'final'])
345
-
346
- // sastrawi additional rules
347
- data.push(['penstabilan', 'stabil'])
348
- data.push(['pentranskripsi', 'transkripsi'])
349
-
350
- data.push(['mentaati', 'taat'])
351
- data.push(['meniru-nirukan', 'tiru'])
352
- data.push(['menyepak-nyepak', 'sepak'])
353
-
354
- data.push(['melewati', 'lewat'])
355
- data.push(['menganga', 'nganga'])
356
-
357
- data.push(['kupukul', 'pukul'])
358
- data.push(['kauhajar', 'hajar'])
359
-
360
- data.push(['kuasa-Mu', 'kuasa'])
361
- data.push(['malaikat-malaikat-Nya', 'malaikat'])
362
- data.push(['nikmat-Ku', 'nikmat'])
363
- data.push(['allah-lah', 'allah'])
364
-
365
- describe('Indonesian stemmer', function () {
366
- data.forEach(function (testPair) {
367
- it('should correctly tokenize and stem ' + testPair[0] + ' to ' + testPair[1], function () {
368
- const output = Stemmer.tokenizeAndStem(testPair[0])
369
- const outputConcat = output.join(' ')
370
- // console.log(output);
371
- expect(outputConcat).toEqual(testPair[1])
372
- })
373
- })
374
- })
@@ -1,69 +0,0 @@
1
- /*
2
- Copyright (c) 2012, Guillaume Marty
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { StemmerJa as stemmer } from 'lib/natural'
26
-
27
- const test = ['コピー', 'コーヒー', 'タクシー', 'パーティー', 'パーティ', 'ヘルプ・センター']
28
- const testResult = ['コピー', 'コーヒ', 'タクシ', 'パーティ', 'パーティ', 'ヘルプ・センタ']
29
- const text = '明後日パーティーに行く予定がある。図書館で資料をコピーしました。'
30
-
31
- describe('StemmerJa', function () {
32
- it('should stem words', function () {
33
- for (let i = 0; i < test.length; i++) {
34
- expect(stemmer.stem(test[i])).toBe(testResult[i])
35
- }
36
- })
37
-
38
- it('should not tokenize halfwidth katakana', function () {
39
- expect(stemmer.stem('タクシー')).toEqual('タクシー')
40
- })
41
-
42
- it('should tokenize, stem and exclude stop words (default behavior)', function () {
43
- const tokens = [
44
- '明後日', 'パーティ', '行く', '予定', // パーティー should be stemmed
45
- '図書館', '資料', 'コピー', 'まし']
46
- expect(stemmer.tokenizeAndStem(text)).toEqual(tokens)
47
- expect(stemmer.tokenizeAndStem(text, false)).toEqual(tokens)
48
- })
49
-
50
- it('should tokenize, stem and keep stop words', function () {
51
- expect(stemmer.tokenizeAndStem(text, true)).toEqual([
52
- '明後日', 'パーティ', 'に', '行く', '予定', 'が', 'ある',
53
- '図書館', 'で', '資料', 'を', 'コピー', 'し', 'まし', 'た'])
54
- })
55
-
56
- /*
57
- it('should attach new methods to String', function() {
58
- stemmer.attach();
59
- expect('コーヒー'.stem()).toEqual('コーヒ');
60
- expect('コピー'.stem()).toEqual('コピー');
61
- expect('図書館で資料をコピーしました。'.tokenizeAndStem()).toEqual([
62
- '図書館', '資料', 'コピー', 'まし']);
63
- expect('図書館で資料をコピーしました。'.tokenizeAndStem(false)).toEqual([
64
- '図書館', '資料', 'コピー', 'まし']);
65
- expect('図書館で資料をコピーしました。'.tokenizeAndStem(true)).toEqual([
66
- '図書館', 'で', '資料', 'を', 'コピー', 'し', 'まし', 'た']);
67
- });
68
- */
69
- })
@@ -1,163 +0,0 @@
1
- /*
2
- Copyright (c) 2014, Luís Rodrigues
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { Token } from 'lib/natural'
26
-
27
- describe('stemmer token', function () {
28
- it('should receive a string', function () {
29
- const string = 'test'
30
- const token = new Token(string)
31
-
32
- expect(token.string).toBe(string)
33
- })
34
-
35
- it('should hold the original token string', function () {
36
- const string = 'test'
37
- const token = new Token(string)
38
-
39
- expect(token.original).toBe(string)
40
- })
41
-
42
- it('should replace all instances of a string', function () {
43
- const string = 'tester'
44
- const token = new Token(string)
45
-
46
- token.replaceAll('e', 'a')
47
- expect(token.string).toBe('tastar')
48
-
49
- token.replaceAll('ar', 'er')
50
- expect(token.string).toBe('taster')
51
-
52
- token.replaceAll('r', '')
53
- expect(token.string).toBe('taste')
54
- })
55
-
56
- it('should allow vowels to be set', function () {
57
- const vowels = 'aeiou'
58
- const vowelsArray = vowels.split('')
59
- const token = new Token('')
60
-
61
- token.usingVowels(vowels)
62
- expect(token.vowels).toBe(vowels)
63
-
64
- token.usingVowels(vowelsArray)
65
- expect(token.vowels).toBe(vowelsArray)
66
- })
67
-
68
- it('should check for vowels', function () {
69
- const vowels = 'aeiou'
70
- const token = new Token('test')
71
-
72
- token.usingVowels(vowels)
73
-
74
- expect(token.hasVowelAtIndex(0)).toBe(false)
75
- expect(token.hasVowelAtIndex(1)).toBe(true)
76
- expect(token.hasVowelAtIndex(99)).toBe(false)
77
-
78
- token.usingVowels(vowels.split(''))
79
-
80
- expect(token.hasVowelAtIndex(0)).toBe(false)
81
- expect(token.hasVowelAtIndex(1)).toBe(true)
82
- expect(token.hasVowelAtIndex(99)).toBe(false)
83
- })
84
-
85
- it('should find the next vowel', function () {
86
- const token = new Token('tester').usingVowels('aeiou')
87
-
88
- expect(token.nextVowelIndex(0)).toBe(1)
89
- expect(token.nextVowelIndex(1)).toBe(1)
90
- expect(token.nextVowelIndex(2)).toBe(4)
91
- expect(token.nextVowelIndex(5)).toBe(6)
92
- expect(token.nextVowelIndex(99)).toBe(6)
93
- expect(token.nextVowelIndex(-1)).toBe(6)
94
- })
95
-
96
- it('should find the next consonant', function () {
97
- const token = new Token('testee').usingVowels('aeiou')
98
-
99
- expect(token.nextConsonantIndex(0)).toBe(0)
100
- expect(token.nextConsonantIndex(1)).toBe(2)
101
- expect(token.nextConsonantIndex(5)).toBe(6)
102
- expect(token.nextConsonantIndex(99)).toBe(6)
103
- expect(token.nextConsonantIndex(-1)).toBe(6)
104
- })
105
-
106
- it('should mark regions', function () {
107
- const token = new Token('tester')
108
-
109
- token.markRegion('test', 1)
110
- expect(token.regions.test).toBe(1)
111
- })
112
-
113
- it('should mark regions with a callback', function () {
114
- const token = new Token('tester')
115
- const context = { value: 99 }
116
-
117
- token.markRegion('test', 1, function (arg) { return arg })
118
- expect(token.regions.test).toBe(1)
119
-
120
- token.markRegion('test', [1], function (arg) { return arg })
121
- expect(token.regions.test).toBe(1)
122
-
123
- token.markRegion('test', [1, 1], function (a1, a2) { return a1 + a2 })
124
- expect(token.regions.test).toBe(2)
125
-
126
- token.markRegion('test', null, function (this: Token) {
127
- return this.string.length
128
- })
129
- expect(token.regions.test).toBe(6)
130
-
131
- token.markRegion('test', null, function (this: any) { return this.value }, context)
132
- expect(token.regions.test).toBe(99)
133
- })
134
-
135
- it('should check for suffixes', function () {
136
- const token = new Token('tester')
137
-
138
- expect(token.hasSuffix('er')).toBeTruthy()
139
- expect(token.hasSuffix('st')).toBeFalsy()
140
- })
141
-
142
- it('should check for suffixes within a region', function () {
143
- const token = new Token('tester').markRegion('region', 2)
144
-
145
- expect(token.hasSuffixInRegion('st', 'region')).toBe(false)
146
- expect(token.hasSuffixInRegion('ster', 'region')).toBe(true)
147
- expect(token.hasSuffixInRegion('ester', 'region')).toBe(false)
148
- })
149
-
150
- it('should replace the suffix within a region', function () {
151
- const t1 = new Token('tester').markRegion('region', 4)
152
- const t2 = new Token('tester').markRegion('region', 0)
153
-
154
- t1.replaceSuffixInRegion('ter', '<s>', 'region')
155
- expect(t1.string).toBe('tester')
156
-
157
- t1.replaceSuffixInRegion('er', '<s>', 'region')
158
- expect(t1.string).toBe('test<s>')
159
-
160
- t2.replaceSuffixInRegion('protester', '<s>', 'region')
161
- expect(t2.string).toBe('tester')
162
- })
163
- })