natural 7.0.3 → 7.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.github/workflows/npm-publish.yml +2 -0
  2. package/.nycrc +9 -0
  3. package/lib/natural/classifiers/logistic_regression_classifier.js +1 -1
  4. package/package.json +3 -4
  5. package/tsconfig.json +1 -1
  6. package/ts_spec/MaxEntClassifier_spec.ts +0 -130
  7. package/ts_spec/SentimentAnalyzer_spec.ts +0 -232
  8. package/ts_spec/WordPunctTokenizer_spec.ts +0 -57
  9. package/ts_spec/aggressive_tokenizer_de_spec.ts +0 -36
  10. package/ts_spec/aggressive_tokenizer_es_spec.ts +0 -39
  11. package/ts_spec/aggressive_tokenizer_fr_spec.ts +0 -111
  12. package/ts_spec/aggressive_tokenizer_hi_spec.ts +0 -70
  13. package/ts_spec/aggressive_tokenizer_nl_spec.ts +0 -39
  14. package/ts_spec/aggressive_tokenizer_pt_spec.ts +0 -48
  15. package/ts_spec/aggressive_tokenizer_spec.ts +0 -57
  16. package/ts_spec/aggressive_tokenizer_sv_spec.ts +0 -52
  17. package/ts_spec/aggressive_tokenizer_vi_spec.ts +0 -48
  18. package/ts_spec/bayes_classifier_spec.ts +0 -165
  19. package/ts_spec/brill_pos_tagger_spec.ts +0 -79
  20. package/ts_spec/brill_pos_trainer_spec.ts +0 -107
  21. package/ts_spec/carry_stemmer_fr_spec.ts +0 -97
  22. package/ts_spec/classifier_spec.ts +0 -104
  23. package/ts_spec/count_inflector_fr_spec.ts +0 -47
  24. package/ts_spec/count_inflector_spec.ts +0 -76
  25. package/ts_spec/damerau_levenshtein_spec.ts +0 -53
  26. package/ts_spec/dice_coefficient_spec.ts +0 -37
  27. package/ts_spec/double_metaphone_spec.ts +0 -739
  28. package/ts_spec/hamming_distance_spec.ts +0 -66
  29. package/ts_spec/jaro-winkler_spec.ts +0 -79
  30. package/ts_spec/lancaster_stemmer_spec.ts +0 -63
  31. package/ts_spec/levenshtein_spec.ts +0 -107
  32. package/ts_spec/logistic_regression_classifier_spec.ts +0 -93
  33. package/ts_spec/longest_path_tree_spec.ts +0 -111
  34. package/ts_spec/metaphone_spec.ts +0 -258
  35. package/ts_spec/ngram_spec.ts +0 -145
  36. package/ts_spec/ngram_zh_spec.ts +0 -124
  37. package/ts_spec/normalizer_ja_spec.ts +0 -165
  38. package/ts_spec/normalizer_no_spec.ts +0 -112
  39. package/ts_spec/normalizer_spec.ts +0 -141
  40. package/ts_spec/noun_inflector_fr_spec.ts +0 -472
  41. package/ts_spec/noun_inflector_ja_spec.ts +0 -116
  42. package/ts_spec/noun_inflector_spec.ts +0 -369
  43. package/ts_spec/orthography_tokenizer_spec.ts +0 -47
  44. package/ts_spec/porter_stemmer_de_spec.ts +0 -53
  45. package/ts_spec/porter_stemmer_es_spec.ts +0 -63
  46. package/ts_spec/porter_stemmer_fr_spec.ts +0 -103
  47. package/ts_spec/porter_stemmer_it_spec.ts +0 -56
  48. package/ts_spec/porter_stemmer_nl_spec.ts +0 -73
  49. package/ts_spec/porter_stemmer_no_spec.ts +0 -148
  50. package/ts_spec/porter_stemmer_pt_spec.ts +0 -50
  51. package/ts_spec/porter_stemmer_ru_spec.ts +0 -67
  52. package/ts_spec/porter_stemmer_spec.ts +0 -202
  53. package/ts_spec/porter_stemmer_sv_spec.ts +0 -41
  54. package/ts_spec/present_verb_inflector_spec.ts +0 -127
  55. package/ts_spec/remove_diacritics_spec.ts +0 -30
  56. package/ts_spec/sentence_analyzer_spec.ts +0 -293
  57. package/ts_spec/sentence_tokenizer_parser_spec.ts +0 -200
  58. package/ts_spec/sentence_tokenizer_spec.ts +0 -189
  59. package/ts_spec/shortest_path_tree_spec.ts +0 -141
  60. package/ts_spec/soundex_spec.ts +0 -162
  61. package/ts_spec/spellcheck_spec.ts +0 -112
  62. package/ts_spec/stemmer_id_spec.ts +0 -374
  63. package/ts_spec/stemmer_ja_spec.ts +0 -69
  64. package/ts_spec/stemmer_token_spec.ts +0 -163
  65. package/ts_spec/tfidf_spec.ts +0 -275
  66. package/ts_spec/tokenizer_case_spec.ts +0 -242
  67. package/ts_spec/tokenizer_ja_spec.ts +0 -63
  68. package/ts_spec/transliterator_ja_spec.ts +0 -105
  69. package/ts_spec/treebank_word_tokenizer_spec.ts +0 -38
  70. package/ts_spec/trie_spec.ts +0 -197
@@ -1,374 +0,0 @@
1
- /*
2
- Copyright (c) 2018
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this softwé and associated documentation files (the "Softwé"), to deal
6
- in the Softwé without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Softwé, and to permit persons to whom the Softwé is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Softwé.
13
-
14
- THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- // Adapted from original Mocha test script in tests folder
26
-
27
- import { StemmerId as Stemmer } from 'lib/natural'
28
-
29
- declare type Tests = Array<[string, string]>
30
- const data: Tests = []
31
-
32
- // THE TESTCASES
33
- // don't stem short words
34
- data.push(['mei', 'mei'])
35
- data.push(['bui', 'bui'])
36
-
37
- // lookup up the dictionary, to prevent overstemming
38
- // don't stem nilai to nila
39
- data.push(['nilai', 'nilai'])
40
-
41
- // lah|kah|tah|pun
42
- data.push(['hancurlah', 'hancur'])
43
- // data.push(['benarkah', 'benar']);
44
- // data.push(['apatah', 'apa']);
45
- // data.push(['siapapun', 'siapa']);
46
-
47
- // ku|mu|nya
48
- data.push(['jubahku', 'jubah'])
49
- data.push(['bajumu', 'baju'])
50
- data.push(['celananya', 'celana'])
51
-
52
- // i|kan|an
53
- data.push(['hantui', 'hantu'])
54
- // data.push(['belikan', 'beli']);
55
- data.push(['jualan', 'jual'])
56
-
57
- // combination of suffixes
58
- data.push(['bukumukah', 'buku']) // gagal karena -ku dianggap suffix dan dihilangkan
59
- data.push(['miliknyalah', 'milik'])
60
- data.push(['kulitkupun', 'kulit']) // gagal karena -ku dianggap suffix dan dihilangkan
61
- data.push(['berikanku', 'beri'])
62
- data.push(['sakitimu', 'sakit'])
63
- data.push(['beriannya', 'beri'])
64
- data.push(['kasihilah', 'kasih'])
65
-
66
- // plain prefix
67
- data.push(['dibuang', 'buang'])
68
- data.push(['kesakitan', 'sakit'])
69
- data.push(['sesuap', 'suap'])
70
-
71
- // data.push(['teriakanmu', 'teriak']); // wtf? kok jadi ria?
72
- // teriakanmu -> te-ria-kan-mu
73
-
74
- // template formulas for derivation prefix rules (disambiguation) //
75
-
76
- // rule 1a : berV -> ber-V
77
- data.push(['beradu', 'adu'])
78
-
79
- // rule 1b : berV -> be-rV
80
- data.push(['berambut', 'rambut'])
81
-
82
- // rule 2 : berCAP -> ber-CAP
83
- data.push(['bersuara', 'suara'])
84
-
85
- // rule 3 : berCAerV -> ber-CAerV where C != 'r'
86
- data.push(['berdaerah', 'daerah'])
87
-
88
- // rule 4 : belajar -> bel-ajar
89
- data.push(['belajar', 'ajar'])
90
-
91
- // rule 5 : beC1erC2 -> be-C1erC2 where C1 != {'r'|'l'}
92
- // data.push(['bekerja', 'kerja']);
93
- data.push(['beternak', 'ternak'])
94
-
95
- // rule 6a : terV -> ter-V
96
- data.push(['terasing', 'asing'])
97
-
98
- // rule 6b : terV -> te-rV
99
- data.push(['teraup', 'raup'])
100
-
101
- // rule 7 : terCerV -> ter-CerV where C != 'r'
102
- data.push(['tergerak', 'gerak'])
103
-
104
- // rule 8 : terCP -> ter-CP where C != 'r' and P != 'er'
105
- data.push(['terpuruk', 'puruk'])
106
-
107
- // rule 9 : teC1erC2 -> te-C1erC2 where C1 != 'r'
108
- data.push(['teterbang', 'terbang'])
109
-
110
- // rule 10 : me{l|r|w|y}V -> me-{l|r|w|y}V
111
- data.push(['melipat', 'lipat'])
112
- data.push(['meringkas', 'ringkas'])
113
- data.push(['mewarnai', 'warna'])
114
- // data.push(['meyakinkan', 'yakin']);
115
-
116
- // rule 11 : mem{b|f|v} -> mem-{b|f|v}
117
- data.push(['membangun', 'bangun'])
118
- data.push(['memfitnah', 'fitnah'])
119
- data.push(['memvonis', 'vonis'])
120
-
121
- // rule 12 : mempe{r|l} -> mem-pe
122
- data.push(['memperbarui', 'baru'])
123
- data.push(['mempelajari', 'ajar'])
124
-
125
- // rule 13a : mem{rV|V} -> mem{rV|V}
126
- data.push(['meminum', 'minum'])
127
-
128
- // rule 13b : mem{rV|V} -> me-p{rV|V}
129
- data.push(['memukul', 'pukul'])
130
-
131
- // rule 14 : men{c|d|j|z} -> men-{c|d|j|z}
132
- data.push(['mencinta', 'cinta'])
133
- data.push(['mendua', 'dua'])
134
- data.push(['menjauh', 'jauh'])
135
- data.push(['menziarah', 'ziarah'])
136
-
137
- // rule 15a : men{V} -> me-n{V}
138
- data.push(['menuklir', 'nuklir'])
139
-
140
- // rule 15b : men{V} -> me-t{V}
141
- data.push(['menangkap', 'tangkap'])
142
-
143
- // rule 16 : meng{g|h|q} -> meng-{g|h|q}
144
- data.push(['menggila', 'gila'])
145
- data.push(['menghajar', 'hajar'])
146
- data.push(['mengqasar', 'qasar'])
147
-
148
- // rule 17a : mengV -> meng-V
149
- data.push(['mengudara', 'udara'])
150
-
151
- // rule 17b : mengV -> meng-kV
152
- data.push(['mengupas', 'kupas'])
153
-
154
- // rule 18 : menyV -> meny-sV
155
- data.push(['menyuarakan', 'suara'])
156
-
157
- // rule 19 : mempV -> mem-pV where V != 'e'
158
- data.push(['mempopulerkan', 'populer'])
159
-
160
- // rule 20 : pe{w|y}V -> pe-{w|y}V
161
- data.push(['pewarna', 'warna'])
162
- data.push(['peyoga', 'yoga'])
163
-
164
- // rule 21a : perV -> per-V
165
- data.push(['peradilan', 'adil'])
166
-
167
- // rule 21b : perV -> pe-rV
168
- data.push(['perumahan', 'rumah'])
169
-
170
- // rule 22 is missing in the document?
171
-
172
- // rule 23 : perCAP -> per-CAP where C != 'r' and P != 'er'
173
- data.push(['permuka', 'muka'])
174
-
175
- // rule 24 : perCAerV -> per-CAerV where C != 'r'
176
- data.push(['perdaerah', 'daerah'])
177
-
178
- // rule 25 : pem{b|f|v} -> pem-{b|f|v}
179
- data.push(['pembangun', 'bangun'])
180
- data.push(['pemfitnah', 'fitnah'])
181
- data.push(['pemvonis', 'vonis'])
182
-
183
- // rule 26a : pem{rV|V} -> pe-m{rV|V}
184
- data.push(['peminum', 'minum'])
185
-
186
- // rule 26b : pem{rV|V} -> pe-p{rV|V}
187
- data.push(['pemukul', 'pukul'])
188
-
189
- // rule 27 : men{c|d|j|z} -> men-{c|d|j|z}
190
- data.push(['pencinta', 'cinta'])
191
- data.push(['pendahulu', 'dahulu'])
192
- data.push(['penjarah', 'jarah'])
193
- data.push(['penziarah', 'ziarah'])
194
-
195
- // rule 28a : pen{V} -> pe-n{V}
196
- data.push(['penasihat', 'nasihat'])
197
-
198
- // rule 28b : pen{V} -> pe-t{V}
199
- data.push(['penangkap', 'tangkap'])
200
-
201
- // rule 29 : peng{g|h|q} -> peng-{g|h|q}
202
- data.push(['penggila', 'gila'])
203
- data.push(['penghajar', 'hajar'])
204
- data.push(['pengqasar', 'qasar'])
205
-
206
- // rule 30a : pengV -> peng-V
207
- data.push(['pengudara', 'udara'])
208
-
209
- // rule 30b : pengV -> peng-kV
210
- data.push(['pengupas', 'kupas'])
211
-
212
- // rule 31 : penyV -> peny-sV
213
- data.push(['penyuara', 'suara'])
214
-
215
- // rule 32 : pelV -> pe-lV except pelajar -> ajar
216
- data.push(['pelajar', 'ajar'])
217
- data.push(['pelabuhan', 'labuh'])
218
-
219
- // rule 33 : peCerV -> per-erV where C != {r|w|y|l|m|n}
220
- // TODO : find the examples
221
-
222
- // rule 34 : peCP -> pe-CP where C != {r|w|y|l|m|n} and P != 'er'
223
- data.push(['petarung', 'tarung'])
224
-
225
- // CS additional rules
226
-
227
- // rule 35 : terC1erC2 -> ter-C1erC2 where C1 != 'r'
228
- data.push(['terpercaya', 'percaya'])
229
-
230
- // rule 36 : peC1erC2 -> pe-C1erC2 where C1 != {r|w|y|l|m|n}
231
- data.push(['pekerja', 'kerja'])
232
- data.push(['peserta', 'serta'])
233
-
234
- // CS modify rule 12
235
- data.push(['mempengaruhi', 'pengaruh'])
236
-
237
- // CS modify rule 16
238
- data.push(['mengkritik', 'kritik'])
239
-
240
- // CS adjusting rule precedence
241
- data.push(['bersekolah', 'sekolah']) // gagal sekolah -> seko why?
242
- data.push(['bertahan', 'tahan'])
243
- data.push(['mencapai', 'capai']) // gagal mencapai -> capa
244
- // data.push(['dimulai', 'mulai']);
245
- data.push(['petani', 'tani']) // gagal petani -> petan
246
- data.push(['terabai', 'abai']) // gagal terabai -> aba
247
-
248
- // ECS
249
- data.push(['mensyaratkan', 'syarat'])
250
- data.push(['mensyukuri', 'syukur'])
251
- data.push(['mengebom', 'bom'])
252
- data.push(['mempromosikan', 'promosi'])
253
- data.push(['memproteksi', 'proteksi'])
254
- data.push(['memprediksi', 'prediksi'])
255
- data.push(['pengkajian', 'kaji'])
256
- data.push(['pengebom', 'bom'])
257
-
258
- // ECS loop pengembalian akhiran
259
- data.push(['bersembunyi', 'sembunyi'])
260
- data.push(['bersembunyilah', 'sembunyi'])
261
- data.push(['pelanggan', 'langgan'])
262
- data.push(['pelaku', 'laku'])
263
- data.push(['pelangganmukah', 'langgan'])
264
- data.push(['pelakunyalah', 'laku'])
265
-
266
- data.push(['perbaikan', 'baik'])
267
- data.push(['kebaikannya', 'baik'])
268
- data.push(['bisikan', 'bisik'])
269
- // data.push(['menerangi', 'terang']);
270
- // data.push(['berimanlah', 'iman']);
271
-
272
- // data.push(['memuaskan', 'puas']);
273
- data.push(['berpelanggan', 'langgan'])
274
- data.push(['bermakanan', 'makan'])
275
-
276
- // CC (Modified ECS)
277
- data.push(['menyala', 'nyala'])
278
- data.push(['menyanyikan', 'nyanyi'])
279
- data.push(['menyatakannya', 'nyata'])
280
-
281
- data.push(['penyanyi', 'nyanyi'])
282
- data.push(['penyawaan', 'nyawa'])
283
-
284
- // CC infix
285
- // data.push(['rerata', 'rata']);
286
- // data.push(['lelembut', 'lembut']);
287
- data.push(['lemigas', 'ligas'])
288
- data.push(['kinerja', 'kerja'])
289
-
290
- // plurals
291
- data.push(['buku-buku', 'buku'])
292
- data.push(['berbalas-balasan', 'balas'])
293
- data.push(['bolak-balik', 'bolak-balik'])
294
-
295
- // combination of prefix + suffix
296
- data.push(['bertebaran', 'tebar'])
297
- data.push(['terasingkan', 'asing'])
298
- data.push(['membangunkan', 'bangun'])
299
- data.push(['mencintai', 'cinta'])
300
- data.push(['menduakan', 'dua'])
301
- data.push(['menjauhi', 'jauh'])
302
- data.push(['menggilai', 'gila'])
303
- data.push(['pembangunan', 'bangun'])
304
-
305
- // return the word if not found in the dictionary
306
- data.push(['marwan', 'marwan'])
307
- data.push(['subarkah', 'subarkah'])
308
-
309
- // recursively remove prefix
310
- data.push(['memberdayakan', 'daya'])
311
- data.push(['persemakmuran', 'makmur'])
312
- data.push(['keberuntunganmu', 'untung'])
313
- data.push(['kesepersepuluhnya', 'sepuluh'])
314
-
315
- // test stem sentence
316
- // data.push(['siapakah memberdayakan pembangunan', 'siapa daya bangun']);
317
-
318
- // issues
319
- data.push(['Perekonomian', 'ekonomi'])
320
- data.push(['menahan', 'tahan'])
321
-
322
- // test stem multiple sentences
323
- // var multipleSentence1 = 'Cinta telah bertebaran.Keduanya saling mencintai.';
324
- // var multipleSentence2 = "(Cinta telah bertebaran)\n\n\n\nKeduanya saling mencintai.";
325
- // data.push([multipleSentence1, 'cinta telah tebar dua saling cinta']);
326
- // data.push([multipleSentence2, 'cinta telah tebar dua saling cinta']);
327
-
328
- // failed on other method / algorithm but we should succeed
329
- data.push(['peranan', 'peran'])
330
- // data.push(['memberikan', 'beri']);
331
- data.push(['medannya', 'medan'])
332
-
333
- // TODO:
334
- // data.push(['sebagai', 'bagai']);
335
- // data.push(['bagian', 'bagian']);
336
- data.push(['berbadan', 'badan'])
337
- data.push(['abdullah', 'abdullah'])
338
-
339
- // adopted foreign suffixes
340
- // data.push(['budayawan', 'budaya']);
341
- // data.push(['karyawati', 'karya']);
342
- // data.push(['idealis', 'ideal']);
343
- // data.push(['idealisme', 'ideal']);
344
- data.push(['finalisasi', 'final'])
345
-
346
- // sastrawi additional rules
347
- data.push(['penstabilan', 'stabil'])
348
- data.push(['pentranskripsi', 'transkripsi'])
349
-
350
- data.push(['mentaati', 'taat'])
351
- data.push(['meniru-nirukan', 'tiru'])
352
- data.push(['menyepak-nyepak', 'sepak'])
353
-
354
- data.push(['melewati', 'lewat'])
355
- data.push(['menganga', 'nganga'])
356
-
357
- data.push(['kupukul', 'pukul'])
358
- data.push(['kauhajar', 'hajar'])
359
-
360
- data.push(['kuasa-Mu', 'kuasa'])
361
- data.push(['malaikat-malaikat-Nya', 'malaikat'])
362
- data.push(['nikmat-Ku', 'nikmat'])
363
- data.push(['allah-lah', 'allah'])
364
-
365
- describe('Indonesian stemmer', function () {
366
- data.forEach(function (testPair) {
367
- it('should correctly tokenize and stem ' + testPair[0] + ' to ' + testPair[1], function () {
368
- const output = Stemmer.tokenizeAndStem(testPair[0])
369
- const outputConcat = output.join(' ')
370
- // console.log(output);
371
- expect(outputConcat).toEqual(testPair[1])
372
- })
373
- })
374
- })
@@ -1,69 +0,0 @@
1
- /*
2
- Copyright (c) 2012, Guillaume Marty
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { StemmerJa as stemmer } from 'lib/natural'
26
-
27
- const test = ['コピー', 'コーヒー', 'タクシー', 'パーティー', 'パーティ', 'ヘルプ・センター']
28
- const testResult = ['コピー', 'コーヒ', 'タクシ', 'パーティ', 'パーティ', 'ヘルプ・センタ']
29
- const text = '明後日パーティーに行く予定がある。図書館で資料をコピーしました。'
30
-
31
- describe('StemmerJa', function () {
32
- it('should stem words', function () {
33
- for (let i = 0; i < test.length; i++) {
34
- expect(stemmer.stem(test[i])).toBe(testResult[i])
35
- }
36
- })
37
-
38
- it('should not tokenize halfwidth katakana', function () {
39
- expect(stemmer.stem('タクシー')).toEqual('タクシー')
40
- })
41
-
42
- it('should tokenize, stem and exclude stop words (default behavior)', function () {
43
- const tokens = [
44
- '明後日', 'パーティ', '行く', '予定', // パーティー should be stemmed
45
- '図書館', '資料', 'コピー', 'まし']
46
- expect(stemmer.tokenizeAndStem(text)).toEqual(tokens)
47
- expect(stemmer.tokenizeAndStem(text, false)).toEqual(tokens)
48
- })
49
-
50
- it('should tokenize, stem and keep stop words', function () {
51
- expect(stemmer.tokenizeAndStem(text, true)).toEqual([
52
- '明後日', 'パーティ', 'に', '行く', '予定', 'が', 'ある',
53
- '図書館', 'で', '資料', 'を', 'コピー', 'し', 'まし', 'た'])
54
- })
55
-
56
- /*
57
- it('should attach new methods to String', function() {
58
- stemmer.attach();
59
- expect('コーヒー'.stem()).toEqual('コーヒ');
60
- expect('コピー'.stem()).toEqual('コピー');
61
- expect('図書館で資料をコピーしました。'.tokenizeAndStem()).toEqual([
62
- '図書館', '資料', 'コピー', 'まし']);
63
- expect('図書館で資料をコピーしました。'.tokenizeAndStem(false)).toEqual([
64
- '図書館', '資料', 'コピー', 'まし']);
65
- expect('図書館で資料をコピーしました。'.tokenizeAndStem(true)).toEqual([
66
- '図書館', 'で', '資料', 'を', 'コピー', 'し', 'まし', 'た']);
67
- });
68
- */
69
- })
@@ -1,163 +0,0 @@
1
- /*
2
- Copyright (c) 2014, Luís Rodrigues
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { Token } from 'lib/natural'
26
-
27
- describe('stemmer token', function () {
28
- it('should receive a string', function () {
29
- const string = 'test'
30
- const token = new Token(string)
31
-
32
- expect(token.string).toBe(string)
33
- })
34
-
35
- it('should hold the original token string', function () {
36
- const string = 'test'
37
- const token = new Token(string)
38
-
39
- expect(token.original).toBe(string)
40
- })
41
-
42
- it('should replace all instances of a string', function () {
43
- const string = 'tester'
44
- const token = new Token(string)
45
-
46
- token.replaceAll('e', 'a')
47
- expect(token.string).toBe('tastar')
48
-
49
- token.replaceAll('ar', 'er')
50
- expect(token.string).toBe('taster')
51
-
52
- token.replaceAll('r', '')
53
- expect(token.string).toBe('taste')
54
- })
55
-
56
- it('should allow vowels to be set', function () {
57
- const vowels = 'aeiou'
58
- const vowelsArray = vowels.split('')
59
- const token = new Token('')
60
-
61
- token.usingVowels(vowels)
62
- expect(token.vowels).toBe(vowels)
63
-
64
- token.usingVowels(vowelsArray)
65
- expect(token.vowels).toBe(vowelsArray)
66
- })
67
-
68
- it('should check for vowels', function () {
69
- const vowels = 'aeiou'
70
- const token = new Token('test')
71
-
72
- token.usingVowels(vowels)
73
-
74
- expect(token.hasVowelAtIndex(0)).toBe(false)
75
- expect(token.hasVowelAtIndex(1)).toBe(true)
76
- expect(token.hasVowelAtIndex(99)).toBe(false)
77
-
78
- token.usingVowels(vowels.split(''))
79
-
80
- expect(token.hasVowelAtIndex(0)).toBe(false)
81
- expect(token.hasVowelAtIndex(1)).toBe(true)
82
- expect(token.hasVowelAtIndex(99)).toBe(false)
83
- })
84
-
85
- it('should find the next vowel', function () {
86
- const token = new Token('tester').usingVowels('aeiou')
87
-
88
- expect(token.nextVowelIndex(0)).toBe(1)
89
- expect(token.nextVowelIndex(1)).toBe(1)
90
- expect(token.nextVowelIndex(2)).toBe(4)
91
- expect(token.nextVowelIndex(5)).toBe(6)
92
- expect(token.nextVowelIndex(99)).toBe(6)
93
- expect(token.nextVowelIndex(-1)).toBe(6)
94
- })
95
-
96
- it('should find the next consonant', function () {
97
- const token = new Token('testee').usingVowels('aeiou')
98
-
99
- expect(token.nextConsonantIndex(0)).toBe(0)
100
- expect(token.nextConsonantIndex(1)).toBe(2)
101
- expect(token.nextConsonantIndex(5)).toBe(6)
102
- expect(token.nextConsonantIndex(99)).toBe(6)
103
- expect(token.nextConsonantIndex(-1)).toBe(6)
104
- })
105
-
106
- it('should mark regions', function () {
107
- const token = new Token('tester')
108
-
109
- token.markRegion('test', 1)
110
- expect(token.regions.test).toBe(1)
111
- })
112
-
113
- it('should mark regions with a callback', function () {
114
- const token = new Token('tester')
115
- const context = { value: 99 }
116
-
117
- token.markRegion('test', 1, function (arg) { return arg })
118
- expect(token.regions.test).toBe(1)
119
-
120
- token.markRegion('test', [1], function (arg) { return arg })
121
- expect(token.regions.test).toBe(1)
122
-
123
- token.markRegion('test', [1, 1], function (a1, a2) { return a1 + a2 })
124
- expect(token.regions.test).toBe(2)
125
-
126
- token.markRegion('test', null, function (this: Token) {
127
- return this.string.length
128
- })
129
- expect(token.regions.test).toBe(6)
130
-
131
- token.markRegion('test', null, function (this: any) { return this.value }, context)
132
- expect(token.regions.test).toBe(99)
133
- })
134
-
135
- it('should check for suffixes', function () {
136
- const token = new Token('tester')
137
-
138
- expect(token.hasSuffix('er')).toBeTruthy()
139
- expect(token.hasSuffix('st')).toBeFalsy()
140
- })
141
-
142
- it('should check for suffixes within a region', function () {
143
- const token = new Token('tester').markRegion('region', 2)
144
-
145
- expect(token.hasSuffixInRegion('st', 'region')).toBe(false)
146
- expect(token.hasSuffixInRegion('ster', 'region')).toBe(true)
147
- expect(token.hasSuffixInRegion('ester', 'region')).toBe(false)
148
- })
149
-
150
- it('should replace the suffix within a region', function () {
151
- const t1 = new Token('tester').markRegion('region', 4)
152
- const t2 = new Token('tester').markRegion('region', 0)
153
-
154
- t1.replaceSuffixInRegion('ter', '<s>', 'region')
155
- expect(t1.string).toBe('tester')
156
-
157
- t1.replaceSuffixInRegion('er', '<s>', 'region')
158
- expect(t1.string).toBe('test<s>')
159
-
160
- t2.replaceSuffixInRegion('protester', '<s>', 'region')
161
- expect(t2.string).toBe('tester')
162
- })
163
- })