natural 7.0.2 → 7.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/.github/workflows/npm-publish.yml +2 -0
  2. package/.nycrc +9 -0
  3. package/lib/natural/analyzers/sentence_analyzer.js +115 -113
  4. package/lib/natural/spellcheck/spellcheck.js +63 -61
  5. package/lib/natural/util/bag.js +27 -25
  6. package/package.json +3 -4
  7. package/tsconfig.json +1 -1
  8. package/ts_spec/MaxEntClassifier_spec.ts +0 -130
  9. package/ts_spec/SentimentAnalyzer_spec.ts +0 -232
  10. package/ts_spec/WordPunctTokenizer_spec.ts +0 -57
  11. package/ts_spec/aggressive_tokenizer_de_spec.ts +0 -36
  12. package/ts_spec/aggressive_tokenizer_es_spec.ts +0 -39
  13. package/ts_spec/aggressive_tokenizer_fr_spec.ts +0 -111
  14. package/ts_spec/aggressive_tokenizer_hi_spec.ts +0 -70
  15. package/ts_spec/aggressive_tokenizer_nl_spec.ts +0 -39
  16. package/ts_spec/aggressive_tokenizer_pt_spec.ts +0 -48
  17. package/ts_spec/aggressive_tokenizer_spec.ts +0 -57
  18. package/ts_spec/aggressive_tokenizer_sv_spec.ts +0 -52
  19. package/ts_spec/aggressive_tokenizer_vi_spec.ts +0 -48
  20. package/ts_spec/bayes_classifier_spec.ts +0 -165
  21. package/ts_spec/brill_pos_tagger_spec.ts +0 -79
  22. package/ts_spec/brill_pos_trainer_spec.ts +0 -107
  23. package/ts_spec/carry_stemmer_fr_spec.ts +0 -97
  24. package/ts_spec/classifier_spec.ts +0 -104
  25. package/ts_spec/count_inflector_fr_spec.ts +0 -47
  26. package/ts_spec/count_inflector_spec.ts +0 -76
  27. package/ts_spec/damerau_levenshtein_spec.ts +0 -53
  28. package/ts_spec/dice_coefficient_spec.ts +0 -37
  29. package/ts_spec/double_metaphone_spec.ts +0 -739
  30. package/ts_spec/hamming_distance_spec.ts +0 -66
  31. package/ts_spec/jaro-winkler_spec.ts +0 -79
  32. package/ts_spec/lancaster_stemmer_spec.ts +0 -63
  33. package/ts_spec/levenshtein_spec.ts +0 -107
  34. package/ts_spec/logistic_regression_classifier_spec.ts +0 -93
  35. package/ts_spec/longest_path_tree_spec.ts +0 -111
  36. package/ts_spec/metaphone_spec.ts +0 -258
  37. package/ts_spec/ngram_spec.ts +0 -145
  38. package/ts_spec/ngram_zh_spec.ts +0 -124
  39. package/ts_spec/normalizer_ja_spec.ts +0 -165
  40. package/ts_spec/normalizer_no_spec.ts +0 -112
  41. package/ts_spec/normalizer_spec.ts +0 -141
  42. package/ts_spec/noun_inflector_fr_spec.ts +0 -472
  43. package/ts_spec/noun_inflector_ja_spec.ts +0 -116
  44. package/ts_spec/noun_inflector_spec.ts +0 -369
  45. package/ts_spec/orthography_tokenizer_spec.ts +0 -47
  46. package/ts_spec/porter_stemmer_de_spec.ts +0 -53
  47. package/ts_spec/porter_stemmer_es_spec.ts +0 -63
  48. package/ts_spec/porter_stemmer_fr_spec.ts +0 -103
  49. package/ts_spec/porter_stemmer_it_spec.ts +0 -56
  50. package/ts_spec/porter_stemmer_nl_spec.ts +0 -73
  51. package/ts_spec/porter_stemmer_no_spec.ts +0 -148
  52. package/ts_spec/porter_stemmer_pt_spec.ts +0 -50
  53. package/ts_spec/porter_stemmer_ru_spec.ts +0 -67
  54. package/ts_spec/porter_stemmer_spec.ts +0 -202
  55. package/ts_spec/porter_stemmer_sv_spec.ts +0 -41
  56. package/ts_spec/present_verb_inflector_spec.ts +0 -127
  57. package/ts_spec/remove_diacritics_spec.ts +0 -30
  58. package/ts_spec/sentence_analyzer_spec.ts +0 -293
  59. package/ts_spec/sentence_tokenizer_parser_spec.ts +0 -200
  60. package/ts_spec/sentence_tokenizer_spec.ts +0 -189
  61. package/ts_spec/shortest_path_tree_spec.ts +0 -141
  62. package/ts_spec/soundex_spec.ts +0 -162
  63. package/ts_spec/spellcheck_spec.ts +0 -112
  64. package/ts_spec/stemmer_id_spec.ts +0 -374
  65. package/ts_spec/stemmer_ja_spec.ts +0 -69
  66. package/ts_spec/stemmer_token_spec.ts +0 -163
  67. package/ts_spec/tfidf_spec.ts +0 -275
  68. package/ts_spec/tokenizer_case_spec.ts +0 -242
  69. package/ts_spec/tokenizer_ja_spec.ts +0 -63
  70. package/ts_spec/transliterator_ja_spec.ts +0 -105
  71. package/ts_spec/treebank_word_tokenizer_spec.ts +0 -38
  72. package/ts_spec/trie_spec.ts +0 -197
@@ -1,369 +0,0 @@
1
- /*
2
- Copyright (c) 2011, Chris Umbel
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { NounInflector } from 'lib/natural'
26
- const inflector = new NounInflector()
27
-
28
- describe('noun inflector', function () {
29
- describe('singularization', function () {
30
- it('should drop an S by default', function () {
31
- expect(inflector.singularize('rrrs')).toBe('rrr')
32
- expect(inflector.singularize('hackers')).toBe('hacker')
33
- expect(inflector.singularize('movies')).toBe('movie')
34
-
35
- // MAN cases that don't pluralize to MEN
36
- expect(inflector.singularize('talismans')).toBe('talisman')
37
- expect(inflector.singularize('humans')).toBe('human')
38
- expect(inflector.singularize('prehumans')).toBe('prehuman')
39
- })
40
-
41
- it('should handle ambiguous form', function () {
42
- expect(inflector.singularize('deer')).toBe('deer')
43
- expect(inflector.singularize('fish')).toBe('fish')
44
- expect(inflector.singularize('series')).toBe('series')
45
- expect(inflector.singularize('sheep')).toBe('sheep')
46
- expect(inflector.singularize('trout')).toBe('trout')
47
- })
48
-
49
- it('should convert plurals ending SES to S', function () {
50
- expect(inflector.singularize('statuses')).toBe('status')
51
- expect(inflector.singularize('buses')).toBe('bus')
52
- })
53
-
54
- it('should match irregulars', function () {
55
- expect(inflector.singularize('people')).toBe('person')
56
- expect(inflector.singularize('children')).toBe('child')
57
- expect(inflector.singularize('oxen')).toBe('ox')
58
- expect(inflector.singularize('clothes')).toBe('cloth')
59
- expect(inflector.singularize('heroes')).toBe('hero')
60
- expect(inflector.singularize('torsi')).toBe('torso')
61
- })
62
-
63
- it('should handle IX cases', function () {
64
- expect(inflector.singularize('matrices')).toBe('matrix')
65
- expect(inflector.singularize('indices')).toBe('index')
66
- expect(inflector.singularize('cortices')).toBe('cortex')
67
-
68
- // our pluralizer won''t cause this form of appendix (appendicies)
69
- // but we should handle it
70
- expect(inflector.singularize('appendices')).toBe('appendix')
71
- })
72
-
73
- it('should regulars to ES', function () {
74
- expect(inflector.singularize('churches')).toBe('church')
75
- expect(inflector.singularize('appendixes')).toBe('appendix')
76
- expect(inflector.singularize('messes')).toBe('mess')
77
- expect(inflector.singularize('quizes')).toBe('quiz')
78
- expect(inflector.singularize('shoes')).toBe('shoe')
79
- expect(inflector.singularize('funguses')).toBe('fungus')
80
- })
81
-
82
- it('should handle SIS cases', function () {
83
- expect(inflector.singularize('synopses')).toBe('synopsis')
84
- expect(inflector.singularize('parentheses')).toBe('parenthesis')
85
- })
86
-
87
- it('should handle special OES cases', function () {
88
- expect(inflector.singularize('tomatoes')).toBe('tomato')
89
- })
90
-
91
- it('should handle I cases', function () {
92
- expect(inflector.singularize('octopi')).toBe('octopus')
93
- expect(inflector.singularize('stimuli')).toBe('stimulus')
94
- expect(inflector.singularize('radii')).toBe('radius')
95
- expect(inflector.singularize('nuclei')).toBe('nucleus')
96
- expect(inflector.singularize('fungi')).toBe('fungus')
97
- expect(inflector.singularize('cacti')).toBe('cactus')
98
- expect(inflector.singularize('alumni')).toBe('alumnus')
99
- expect(inflector.singularize('calculi')).toBe('calculus')
100
- expect(inflector.singularize('hippopotami')).toBe('hippopotamus')
101
- expect(inflector.singularize('macrofungi')).toBe('macrofungus')
102
- expect(inflector.singularize('phoeti')).toBe('phoetus')
103
- expect(inflector.singularize('syllabi')).toBe('syllabus')
104
- expect(inflector.singularize('trophi')).toBe('trophus')
105
- })
106
-
107
- it('should handle IVES cases', function () {
108
- expect(inflector.singularize('lives')).toBe('life')
109
- expect(inflector.singularize('knives')).toBe('knife')
110
- })
111
-
112
- it('should handle Y cases', function () {
113
- expect(inflector.singularize('parties')).toBe('party')
114
- expect(inflector.singularize('flies')).toBe('fly')
115
- expect(inflector.singularize('victories')).toBe('victory')
116
- expect(inflector.singularize('monstrosities')).toBe('monstrosity')
117
- })
118
-
119
- it('should handle SS cases', function () {
120
- expect(inflector.singularize('dresses')).toBe('dress')
121
- expect(inflector.singularize('dress')).toBe('dress')
122
- expect(inflector.singularize('messes')).toBe('mess')
123
- })
124
-
125
- it('should handle MAN->MAN cases', function () {
126
- expect(inflector.singularize('men')).toBe('man')
127
- expect(inflector.singularize('women')).toBe('woman')
128
- expect(inflector.singularize('workmen')).toBe('workman')
129
- expect(inflector.singularize('riflemen')).toBe('rifleman')
130
- })
131
-
132
- it('should handle irregular cases', function () {
133
- expect(inflector.singularize('feet')).toBe('foot')
134
- expect(inflector.singularize('geese')).toBe('goose')
135
- expect(inflector.singularize('teeth')).toBe('tooth')
136
- expect(inflector.singularize('ephemerides')).toBe('ephemeris')
137
- })
138
-
139
- it('should handle AE cases', function () {
140
- expect(inflector.singularize('antennae')).toBe('antenna')
141
- expect(inflector.singularize('formulae')).toBe('formula')
142
- expect(inflector.singularize('nebulae')).toBe('nebula')
143
- expect(inflector.singularize('vertebrae')).toBe('vertebra')
144
- expect(inflector.singularize('vitae')).toBe('vita')
145
- })
146
-
147
- it('should allow AE cases to be S', function () {
148
- expect(inflector.singularize('antennas')).toBe('antenna')
149
- expect(inflector.singularize('formulas')).toBe('formula')
150
- })
151
- })
152
-
153
- describe('pluralization', function () {
154
- it('should append an S by default', function () {
155
- expect(inflector.pluralize('rrr')).toBe('rrrs')
156
- expect(inflector.pluralize('hacker')).toBe('hackers')
157
- expect(inflector.pluralize('movie')).toBe('movies')
158
- })
159
-
160
- it('should handle ambiguous form', function () {
161
- expect(inflector.pluralize('deer')).toBe('deer')
162
- expect(inflector.pluralize('fish')).toBe('fish')
163
- expect(inflector.pluralize('series')).toBe('series')
164
- expect(inflector.pluralize('sheep')).toBe('sheep')
165
- expect(inflector.pluralize('trout')).toBe('trout')
166
- })
167
-
168
- it('should convert singulars ending s to ses', function () {
169
- expect(inflector.pluralize('status')).toBe('statuses')
170
- expect(inflector.pluralize('bus')).toBe('buses')
171
- })
172
-
173
- it('should match irregulars', function () {
174
- expect(inflector.pluralize('person')).toBe('people')
175
- expect(inflector.pluralize('child')).toBe('children')
176
- expect(inflector.pluralize('ox')).toBe('oxen')
177
- })
178
-
179
- it('should maintain case of irregulars', function () {
180
- expect(inflector.pluralize('OX')).toBe('OXEN')
181
- expect(inflector.pluralize('Person')).toBe('People')
182
- expect(inflector.pluralize('child')).toBe('children')
183
- expect(inflector.pluralize('cloth')).toBe('clothes')
184
- })
185
-
186
- it('should handle IX cases', function () {
187
- expect(inflector.pluralize('matrix')).toBe('matrices')
188
- expect(inflector.pluralize('index')).toBe('indices')
189
- expect(inflector.pluralize('cortex')).toBe('cortices')
190
- })
191
-
192
- it('should regulars to ES', function () {
193
- expect(inflector.pluralize('church')).toBe('churches')
194
- expect(inflector.pluralize('appendix')).toBe('appendixes')
195
- expect(inflector.pluralize('mess')).toBe('messes')
196
- expect(inflector.pluralize('quiz')).toBe('quizes')
197
- expect(inflector.pluralize('shoe')).toBe('shoes')
198
- })
199
-
200
- it('should handle SIS cases', function () {
201
- expect(inflector.pluralize('synopsis')).toBe('synopses')
202
- expect(inflector.pluralize('parenthesis')).toBe('parentheses')
203
- })
204
-
205
- it('should handle special OES cases', function () {
206
- expect(inflector.pluralize('tomato')).toBe('tomatoes')
207
- expect(inflector.pluralize('buffalo')).toBe('buffaloes')
208
- expect(inflector.pluralize('tornado')).toBe('tornadoes')
209
- })
210
-
211
- it('should handle I cases', function () {
212
- expect(inflector.pluralize('radius')).toBe('radii')
213
- expect(inflector.pluralize('octopus')).toBe('octopi')
214
- expect(inflector.pluralize('stimulus')).toBe('stimuli')
215
- expect(inflector.pluralize('nucleus')).toBe('nuclei')
216
- expect(inflector.pluralize('fungus')).toBe('fungi')
217
- expect(inflector.pluralize('cactus')).toBe('cacti')
218
- })
219
-
220
- it('should handle IVES cases', function () {
221
- expect(inflector.pluralize('knife')).toBe('knives')
222
- expect(inflector.pluralize('life')).toBe('lives')
223
- })
224
-
225
- it('should handle Y cases', function () {
226
- expect(inflector.pluralize('party')).toBe('parties')
227
- expect(inflector.pluralize('fly')).toBe('flies')
228
- expect(inflector.pluralize('victory')).toBe('victories')
229
- expect(inflector.pluralize('monstrosity')).toBe('monstrosities')
230
- })
231
-
232
- it('should handle [aeiou]Y cases', function () {
233
- expect(inflector.pluralize('day')).toBe('days')
234
- expect(inflector.pluralize('toy')).toBe('toys')
235
- expect(inflector.pluralize('journey')).toBe('journeys')
236
- })
237
-
238
- it('should handle SS cases', function () {
239
- expect(inflector.pluralize('dress')).toBe('dresses')
240
- expect(inflector.pluralize('dresses')).toBe('dresses')
241
- expect(inflector.pluralize('mess')).toBe('messes')
242
- })
243
-
244
- it('should handle MAN->MEN cases', function () {
245
- expect(inflector.pluralize('man')).toBe('men')
246
- expect(inflector.pluralize('woman')).toBe('women')
247
- expect(inflector.pluralize('workman')).toBe('workmen')
248
- expect(inflector.pluralize('rifleman')).toBe('riflemen')
249
- })
250
-
251
- it('should handle irregular cases', function () {
252
- expect(inflector.pluralize('foot')).toBe('feet')
253
- expect(inflector.pluralize('goose')).toBe('geese')
254
- expect(inflector.pluralize('tooth')).toBe('teeth')
255
- expect(inflector.pluralize('ephemeris')).toBe('ephemerides')
256
-
257
- // MAN cases that don't pluralize to MEN
258
- expect(inflector.pluralize('talisman')).toBe('talismans')
259
- expect(inflector.pluralize('human')).toBe('humans')
260
- expect(inflector.pluralize('prehuman')).toBe('prehumans')
261
- })
262
-
263
- it('should handle AE cases', function () {
264
- expect(inflector.pluralize('antenna')).toBe('antennae')
265
- expect(inflector.pluralize('formula')).toBe('formulae')
266
- expect(inflector.pluralize('nebula')).toBe('nebulae')
267
- expect(inflector.pluralize('vertebra')).toBe('vertebrae')
268
- expect(inflector.pluralize('vita')).toBe('vitae')
269
- })
270
- })
271
-
272
- it('should pluralize and singularize string from patch', function () {
273
- expect(inflector.pluralize('synopsis')).toBe('synopses')
274
- expect(inflector.singularize('synopses')).toBe('synopsis')
275
- expect(inflector.pluralize('mess')).toBe('messes')
276
- expect(inflector.singularize('messes')).toBe('mess')
277
- })
278
-
279
- describe('custom inflections', function () {
280
- it('should pluralize and singularize custom forms', function () {
281
- const myInflector = new NounInflector()
282
- myInflector.addPlural(/(code|ware)/i, '$1z')
283
- myInflector.addSingular(/(code|ware)z/i, '$1')
284
- expect(myInflector.pluralize('code')).toBe('codez')
285
- expect(myInflector.pluralize('ware')).toBe('warez')
286
- expect(myInflector.singularize('codez')).toBe('code')
287
- expect(myInflector.singularize('warez')).toBe('ware')
288
- })
289
-
290
- it('should not break regular forms', function () {
291
- const myInflector = new NounInflector()
292
- myInflector.addPlural(/(code|ware)/i, '$1z')
293
- myInflector.addSingular(/(code|ware)z/i, '$1')
294
- expect(myInflector.pluralize('bus')).toBe('buses')
295
- expect(myInflector.singularize('buses')).toBe('bus')
296
- })
297
- })
298
-
299
- describe('should handle words ending in f, ff, ffe', function () {
300
- it('should handle words ending in ff and ffe', function () {
301
- const myInflector = new NounInflector()
302
- expect(myInflector.pluralize('sherriff')).toBe('sherriffs')
303
- expect(myInflector.pluralize('giraffe')).toBe('giraffes')
304
- expect(myInflector.singularize('sherriffs')).toBe('sherriff')
305
- expect(myInflector.singularize('giraffes')).toBe('giraffe')
306
- })
307
-
308
- it('should handle words ending in f', function () {
309
- const myInflector = new NounInflector()
310
- expect(myInflector.pluralize('roof')).toBe('roofs')
311
- expect(myInflector.pluralize('chief')).toBe('chiefs')
312
- expect(myInflector.pluralize('oaf')).toBe('oafs')
313
- expect(myInflector.singularize('roofs')).toBe('roof')
314
- expect(myInflector.singularize('chiefs')).toBe('chief')
315
- expect(myInflector.singularize('oafs')).toBe('oaf')
316
- })
317
-
318
- it('should handle words ending in f or fe', function () {
319
- const myInflector = new NounInflector()
320
- expect(myInflector.pluralize('leaf')).toBe('leaves')
321
- expect(myInflector.pluralize('wolf')).toBe('wolves')
322
- expect(myInflector.pluralize('calf')).toBe('calves')
323
- expect(myInflector.pluralize('half')).toBe('halves')
324
- expect(myInflector.pluralize('knife')).toBe('knives')
325
- expect(myInflector.pluralize('loaf')).toBe('loaves')
326
- expect(myInflector.pluralize('life')).toBe('lives')
327
- expect(myInflector.pluralize('wife')).toBe('wives')
328
- expect(myInflector.pluralize('shelf')).toBe('shelves')
329
- expect(myInflector.pluralize('thief')).toBe('thieves')
330
- expect(myInflector.pluralize('yourself')).toBe('yourselves')
331
-
332
- expect(myInflector.singularize('leaves')).toBe('leaf')
333
- expect(myInflector.singularize('wolves')).toBe('wolf')
334
- expect(myInflector.singularize('calves')).toBe('calf')
335
- expect(myInflector.singularize('halves')).toBe('half')
336
- expect(myInflector.singularize('knives')).toBe('knife')
337
- expect(myInflector.singularize('loaves')).toBe('loaf')
338
- expect(myInflector.singularize('lives')).toBe('life')
339
- expect(myInflector.singularize('wives')).toBe('wife')
340
- expect(myInflector.singularize('shelves')).toBe('shelf')
341
- expect(myInflector.singularize('thieves')).toBe('thief')
342
- expect(myInflector.singularize('yourselves')).toBe('yourself')
343
- })
344
-
345
- it('should handle words ending in rf', function () {
346
- const myInflector = new NounInflector()
347
- expect(myInflector.pluralize('scarf')).toBe('scarfs')
348
- expect(myInflector.pluralize('dwarf')).toBe('dwarfs')
349
- expect(myInflector.pluralize('handkerchief')).toBe('handkerchiefs')
350
- expect(myInflector.pluralize('wharf')).toBe('wharfs')
351
- expect(myInflector.singularize('scarfs')).toBe('scarf')
352
- expect(myInflector.singularize('dwarfs')).toBe('dwarf')
353
- expect(myInflector.singularize('wharfs')).toBe('wharf')
354
- expect(myInflector.singularize('handkerchiefs')).toBe('handkerchief')
355
- expect(myInflector.singularize('scarves')).toBe('scarf')
356
- expect(myInflector.singularize('dwarves')).toBe('dwarf')
357
- expect(myInflector.singularize('wharves')).toBe('wharf')
358
- expect(myInflector.singularize('handkerchieves')).toBe('handkerchief')
359
- })
360
-
361
- it('should handle words ending in rf', function () {
362
- const myInflector = new NounInflector()
363
- expect(myInflector.singularize('expenses')).toBe('expense')
364
- expect(myInflector.singularize('defenses')).toBe('defense')
365
- expect(myInflector.pluralize('expense')).toBe('expenses')
366
- expect(myInflector.pluralize('defense')).toBe('defenses')
367
- })
368
- })
369
- })
@@ -1,47 +0,0 @@
1
- /*
2
- Copyright (c) 2018, Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { OrthographyTokenizer } from 'lib/natural'
26
-
27
- const sentencesInFinnish = [
28
- 'Mikä sinun nimesi on?',
29
- 'Hyvää kiitos, entä sinulle?',
30
- 'Tämä herrasmies maksaa kaiken'
31
- ]
32
-
33
- const results = [
34
- ['Mikä', 'sinun', 'nimesi', 'on'],
35
- ['Hyvää', 'kiitos', 'entä', 'sinulle'],
36
- ['Tämä', 'herrasmies', 'maksaa', 'kaiken']
37
- ]
38
-
39
- describe('The orthography tokenizer tokenizes sentences in Finnish', function () {
40
- const tokenizer = new OrthographyTokenizer({ language: 'fi' })
41
- sentencesInFinnish.forEach(function (sentence, index) {
42
- it('It should correctly tokenize the following sentence: ' + sentence, function () {
43
- // console.log(tokenizer.tokenize(sentencePlusResult[0]));
44
- expect(tokenizer.tokenize(sentence)).toEqual(results[index])
45
- })
46
- })
47
- })
@@ -1,53 +0,0 @@
1
- /*
2
- Copyright (c) 2023, Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { PorterStemmerDe as stemmer } from 'lib/natural'
26
- import rawJson from '../spec/test_data/snowball_de.json'
27
- const snowBallDict = rawJson as Record<string, string>
28
- const DEBUG = false
29
-
30
- describe('porter_stemmer_de', function () {
31
- it('should perform stemming on a lot of words', function () {
32
- const ok = []
33
- const ko = []
34
-
35
- Object.keys(snowBallDict).forEach(word => {
36
- const stemmed = stemmer.stem(word)
37
- const expectedStem = snowBallDict[word]
38
- DEBUG && console.log(word + ' -> ' + stemmed)
39
-
40
- if (stemmed === expectedStem) {
41
- ok.push(word)
42
- } else {
43
- ko.push({
44
- word,
45
- expected: expectedStem,
46
- actual: stemmed
47
- })
48
- }
49
- })
50
-
51
- expect(ko.length).toBe(0)
52
- })
53
- })
@@ -1,63 +0,0 @@
1
- /*
2
- Copyright (c) 2019, Luís Rodrigues (adapted for Spanish stemmer), Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { PorterStemmerEs as PorterStemmer } from 'lib/natural'
26
- import rawJson from '../spec/test_data/snowball_es.json'
27
- const snowBallDict = rawJson as Record<string, string>
28
- const DEBUG = false
29
-
30
- describe('porter_stemmer_es', function () {
31
- it('should perform stemming on a lot of words', function () {
32
- const errors = []
33
-
34
- Object.keys(snowBallDict).forEach(word => {
35
- const stemmed = PorterStemmer.stem(word)
36
- const expectedStem = snowBallDict[word]
37
- if (stemmed !== snowBallDict[word]) {
38
- DEBUG && console.log('Error:', word, 'Expected:', expectedStem, 'Got:', stemmed)
39
- errors.push({
40
- word,
41
- expected: expectedStem,
42
- actual: stemmed
43
- })
44
- }
45
- })
46
-
47
- expect(errors.length).toBe(0)
48
- })
49
-
50
- it('should tokenize a piece of text', function () {
51
- expect(PorterStemmer.tokenizeAndStem('Pues, en efecto, es solo el plano y lo que muestra aquello que interesa a Zemeckis.', true)).toEqual(
52
- ['pues', 'en', 'efect', 'es', 'sol', 'el', 'plan', 'y', 'lo', 'que', 'muestr', 'aquell', 'que', 'interes', 'a', 'zemeckis'])
53
- })
54
-
55
- /*
56
- it('should work with the attached notation', function() {
57
- PorterStemmer.attach();
58
- expect("efecto".stem()).toEqual("efect");
59
- expect("Pues, en efecto, es solo el plano y lo que muestra aquello que interesa a Zemeckis.".tokenizeAndStem(true)).toEqual(
60
- [ 'pues', 'en', 'efect', 'es', 'sol', 'el', 'plan', 'y', 'lo', 'que', 'muestr', 'aquell', 'que', 'interes', 'a', 'zemeckis' ]);
61
- });
62
- */
63
- })
@@ -1,103 +0,0 @@
1
- /*
2
- Copyright (c) 2019, Ismaël Héry, Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { PorterStemmerFr as stemmer } from 'lib/natural'
26
- import rawJson from '../spec/test_data/snowball_fr.json'
27
- const snowBallDict = rawJson as Record<string, string>
28
-
29
- describe('porter_stemmer', function () {
30
- it('should prelude', function () {
31
- expect(stemmer.prelude('JOUER')).toBe('joUer')
32
- expect(stemmer.prelude('ennuie')).toBe('ennuIe')
33
- expect(stemmer.prelude('yeux')).toBe('Yeux')
34
- expect(stemmer.prelude('quand')).toBe('qUand')
35
- })
36
-
37
- it('should compute regions', function () {
38
- expect(stemmer.regions('fameusement').r1).toBe(3)
39
- expect(stemmer.regions('fameusement').r2).toBe(6)
40
-
41
- expect(stemmer.regions('taii').r1).toBe(4)
42
- expect(stemmer.regions('taii').r2).toBe(4)
43
-
44
- expect(stemmer.regions('parade').rv).toBe(3)
45
- expect(stemmer.regions('colet').rv).toBe(3)
46
- expect(stemmer.regions('tapis').rv).toBe(3)
47
- expect(stemmer.regions('aimer').rv).toBe(3)
48
- expect(stemmer.regions('adorer').rv).toBe(3)
49
- expect(stemmer.regions('voler').rv).toBe(2)
50
- expect(stemmer.regions('tue').rv).toBe(2)
51
- })
52
-
53
- it('should compute longest suffix ends in Arr', function () {
54
- expect(stemmer.endsinArr('voudriez', ['ez', 'iez', 'z'])).toBe('iez')
55
- })
56
-
57
- it('should stem some word', function () {
58
- expect(stemmer.stem('volera')).toBe('vol')
59
- expect(stemmer.stem('volerait')).toBe('vol')
60
- expect(stemmer.stem('subitement')).toBe('subit')
61
- expect(stemmer.stem('tempérament')).toBe('temper')
62
- expect(stemmer.stem('voudriez')).toBe('voudr')
63
- expect(stemmer.stem('vengeait')).toBe('veng')
64
- expect(stemmer.stem('saisissement')).toBe('sais')
65
- expect(stemmer.stem('transatlantique')).toBe('transatlant')
66
- expect(stemmer.stem('premièrement')).toBe('premi')
67
- expect(stemmer.stem('instruments')).toBe('instrument')
68
- expect(stemmer.stem('trouverions')).toBe('trouv')
69
- expect(stemmer.stem('voyiez')).toBe('voi')
70
- expect(stemmer.stem('publicité')).toBe('publiqu')
71
- expect(stemmer.stem('pitoyable')).toBe('pitoi')
72
- })
73
-
74
- it('should perform stemming on a lot of words', function () {
75
- const ok = []
76
- const ko = []
77
-
78
- Object.keys(snowBallDict).forEach(word => {
79
- const stemmed = stemmer.stem(word)
80
- const expectedStem = snowBallDict[word]
81
-
82
- const regs = stemmer.regions(word)
83
- const txtRegions = {
84
- r1: word.substring(regs.r1),
85
- r2: word.substring(regs.r2),
86
- rv: word.substring(regs.rv)
87
- }
88
-
89
- if (stemmed === expectedStem) {
90
- ok.push(word)
91
- } else {
92
- ko.push({
93
- word,
94
- expected: expectedStem,
95
- actual: stemmed,
96
- regions: txtRegions
97
- })
98
- }
99
- })
100
-
101
- expect(ko.length).toBe(0)
102
- })
103
- })