natural 7.0.3 → 7.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.github/workflows/npm-publish.yml +2 -0
  2. package/.nycrc +9 -0
  3. package/lib/natural/classifiers/logistic_regression_classifier.js +1 -1
  4. package/package.json +3 -4
  5. package/tsconfig.json +1 -1
  6. package/ts_spec/MaxEntClassifier_spec.ts +0 -130
  7. package/ts_spec/SentimentAnalyzer_spec.ts +0 -232
  8. package/ts_spec/WordPunctTokenizer_spec.ts +0 -57
  9. package/ts_spec/aggressive_tokenizer_de_spec.ts +0 -36
  10. package/ts_spec/aggressive_tokenizer_es_spec.ts +0 -39
  11. package/ts_spec/aggressive_tokenizer_fr_spec.ts +0 -111
  12. package/ts_spec/aggressive_tokenizer_hi_spec.ts +0 -70
  13. package/ts_spec/aggressive_tokenizer_nl_spec.ts +0 -39
  14. package/ts_spec/aggressive_tokenizer_pt_spec.ts +0 -48
  15. package/ts_spec/aggressive_tokenizer_spec.ts +0 -57
  16. package/ts_spec/aggressive_tokenizer_sv_spec.ts +0 -52
  17. package/ts_spec/aggressive_tokenizer_vi_spec.ts +0 -48
  18. package/ts_spec/bayes_classifier_spec.ts +0 -165
  19. package/ts_spec/brill_pos_tagger_spec.ts +0 -79
  20. package/ts_spec/brill_pos_trainer_spec.ts +0 -107
  21. package/ts_spec/carry_stemmer_fr_spec.ts +0 -97
  22. package/ts_spec/classifier_spec.ts +0 -104
  23. package/ts_spec/count_inflector_fr_spec.ts +0 -47
  24. package/ts_spec/count_inflector_spec.ts +0 -76
  25. package/ts_spec/damerau_levenshtein_spec.ts +0 -53
  26. package/ts_spec/dice_coefficient_spec.ts +0 -37
  27. package/ts_spec/double_metaphone_spec.ts +0 -739
  28. package/ts_spec/hamming_distance_spec.ts +0 -66
  29. package/ts_spec/jaro-winkler_spec.ts +0 -79
  30. package/ts_spec/lancaster_stemmer_spec.ts +0 -63
  31. package/ts_spec/levenshtein_spec.ts +0 -107
  32. package/ts_spec/logistic_regression_classifier_spec.ts +0 -93
  33. package/ts_spec/longest_path_tree_spec.ts +0 -111
  34. package/ts_spec/metaphone_spec.ts +0 -258
  35. package/ts_spec/ngram_spec.ts +0 -145
  36. package/ts_spec/ngram_zh_spec.ts +0 -124
  37. package/ts_spec/normalizer_ja_spec.ts +0 -165
  38. package/ts_spec/normalizer_no_spec.ts +0 -112
  39. package/ts_spec/normalizer_spec.ts +0 -141
  40. package/ts_spec/noun_inflector_fr_spec.ts +0 -472
  41. package/ts_spec/noun_inflector_ja_spec.ts +0 -116
  42. package/ts_spec/noun_inflector_spec.ts +0 -369
  43. package/ts_spec/orthography_tokenizer_spec.ts +0 -47
  44. package/ts_spec/porter_stemmer_de_spec.ts +0 -53
  45. package/ts_spec/porter_stemmer_es_spec.ts +0 -63
  46. package/ts_spec/porter_stemmer_fr_spec.ts +0 -103
  47. package/ts_spec/porter_stemmer_it_spec.ts +0 -56
  48. package/ts_spec/porter_stemmer_nl_spec.ts +0 -73
  49. package/ts_spec/porter_stemmer_no_spec.ts +0 -148
  50. package/ts_spec/porter_stemmer_pt_spec.ts +0 -50
  51. package/ts_spec/porter_stemmer_ru_spec.ts +0 -67
  52. package/ts_spec/porter_stemmer_spec.ts +0 -202
  53. package/ts_spec/porter_stemmer_sv_spec.ts +0 -41
  54. package/ts_spec/present_verb_inflector_spec.ts +0 -127
  55. package/ts_spec/remove_diacritics_spec.ts +0 -30
  56. package/ts_spec/sentence_analyzer_spec.ts +0 -293
  57. package/ts_spec/sentence_tokenizer_parser_spec.ts +0 -200
  58. package/ts_spec/sentence_tokenizer_spec.ts +0 -189
  59. package/ts_spec/shortest_path_tree_spec.ts +0 -141
  60. package/ts_spec/soundex_spec.ts +0 -162
  61. package/ts_spec/spellcheck_spec.ts +0 -112
  62. package/ts_spec/stemmer_id_spec.ts +0 -374
  63. package/ts_spec/stemmer_ja_spec.ts +0 -69
  64. package/ts_spec/stemmer_token_spec.ts +0 -163
  65. package/ts_spec/tfidf_spec.ts +0 -275
  66. package/ts_spec/tokenizer_case_spec.ts +0 -242
  67. package/ts_spec/tokenizer_ja_spec.ts +0 -63
  68. package/ts_spec/transliterator_ja_spec.ts +0 -105
  69. package/ts_spec/treebank_word_tokenizer_spec.ts +0 -38
  70. package/ts_spec/trie_spec.ts +0 -197
@@ -1,369 +0,0 @@
1
- /*
2
- Copyright (c) 2011, Chris Umbel
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { NounInflector } from 'lib/natural'
26
- const inflector = new NounInflector()
27
-
28
- describe('noun inflector', function () {
29
- describe('singularization', function () {
30
- it('should drop an S by default', function () {
31
- expect(inflector.singularize('rrrs')).toBe('rrr')
32
- expect(inflector.singularize('hackers')).toBe('hacker')
33
- expect(inflector.singularize('movies')).toBe('movie')
34
-
35
- // MAN cases that don't pluralize to MEN
36
- expect(inflector.singularize('talismans')).toBe('talisman')
37
- expect(inflector.singularize('humans')).toBe('human')
38
- expect(inflector.singularize('prehumans')).toBe('prehuman')
39
- })
40
-
41
- it('should handle ambiguous form', function () {
42
- expect(inflector.singularize('deer')).toBe('deer')
43
- expect(inflector.singularize('fish')).toBe('fish')
44
- expect(inflector.singularize('series')).toBe('series')
45
- expect(inflector.singularize('sheep')).toBe('sheep')
46
- expect(inflector.singularize('trout')).toBe('trout')
47
- })
48
-
49
- it('should convert plurals ending SES to S', function () {
50
- expect(inflector.singularize('statuses')).toBe('status')
51
- expect(inflector.singularize('buses')).toBe('bus')
52
- })
53
-
54
- it('should match irregulars', function () {
55
- expect(inflector.singularize('people')).toBe('person')
56
- expect(inflector.singularize('children')).toBe('child')
57
- expect(inflector.singularize('oxen')).toBe('ox')
58
- expect(inflector.singularize('clothes')).toBe('cloth')
59
- expect(inflector.singularize('heroes')).toBe('hero')
60
- expect(inflector.singularize('torsi')).toBe('torso')
61
- })
62
-
63
- it('should handle IX cases', function () {
64
- expect(inflector.singularize('matrices')).toBe('matrix')
65
- expect(inflector.singularize('indices')).toBe('index')
66
- expect(inflector.singularize('cortices')).toBe('cortex')
67
-
68
- // our pluralizer won''t cause this form of appendix (appendicies)
69
- // but we should handle it
70
- expect(inflector.singularize('appendices')).toBe('appendix')
71
- })
72
-
73
- it('should regulars to ES', function () {
74
- expect(inflector.singularize('churches')).toBe('church')
75
- expect(inflector.singularize('appendixes')).toBe('appendix')
76
- expect(inflector.singularize('messes')).toBe('mess')
77
- expect(inflector.singularize('quizes')).toBe('quiz')
78
- expect(inflector.singularize('shoes')).toBe('shoe')
79
- expect(inflector.singularize('funguses')).toBe('fungus')
80
- })
81
-
82
- it('should handle SIS cases', function () {
83
- expect(inflector.singularize('synopses')).toBe('synopsis')
84
- expect(inflector.singularize('parentheses')).toBe('parenthesis')
85
- })
86
-
87
- it('should handle special OES cases', function () {
88
- expect(inflector.singularize('tomatoes')).toBe('tomato')
89
- })
90
-
91
- it('should handle I cases', function () {
92
- expect(inflector.singularize('octopi')).toBe('octopus')
93
- expect(inflector.singularize('stimuli')).toBe('stimulus')
94
- expect(inflector.singularize('radii')).toBe('radius')
95
- expect(inflector.singularize('nuclei')).toBe('nucleus')
96
- expect(inflector.singularize('fungi')).toBe('fungus')
97
- expect(inflector.singularize('cacti')).toBe('cactus')
98
- expect(inflector.singularize('alumni')).toBe('alumnus')
99
- expect(inflector.singularize('calculi')).toBe('calculus')
100
- expect(inflector.singularize('hippopotami')).toBe('hippopotamus')
101
- expect(inflector.singularize('macrofungi')).toBe('macrofungus')
102
- expect(inflector.singularize('phoeti')).toBe('phoetus')
103
- expect(inflector.singularize('syllabi')).toBe('syllabus')
104
- expect(inflector.singularize('trophi')).toBe('trophus')
105
- })
106
-
107
- it('should handle IVES cases', function () {
108
- expect(inflector.singularize('lives')).toBe('life')
109
- expect(inflector.singularize('knives')).toBe('knife')
110
- })
111
-
112
- it('should handle Y cases', function () {
113
- expect(inflector.singularize('parties')).toBe('party')
114
- expect(inflector.singularize('flies')).toBe('fly')
115
- expect(inflector.singularize('victories')).toBe('victory')
116
- expect(inflector.singularize('monstrosities')).toBe('monstrosity')
117
- })
118
-
119
- it('should handle SS cases', function () {
120
- expect(inflector.singularize('dresses')).toBe('dress')
121
- expect(inflector.singularize('dress')).toBe('dress')
122
- expect(inflector.singularize('messes')).toBe('mess')
123
- })
124
-
125
- it('should handle MAN->MAN cases', function () {
126
- expect(inflector.singularize('men')).toBe('man')
127
- expect(inflector.singularize('women')).toBe('woman')
128
- expect(inflector.singularize('workmen')).toBe('workman')
129
- expect(inflector.singularize('riflemen')).toBe('rifleman')
130
- })
131
-
132
- it('should handle irregular cases', function () {
133
- expect(inflector.singularize('feet')).toBe('foot')
134
- expect(inflector.singularize('geese')).toBe('goose')
135
- expect(inflector.singularize('teeth')).toBe('tooth')
136
- expect(inflector.singularize('ephemerides')).toBe('ephemeris')
137
- })
138
-
139
- it('should handle AE cases', function () {
140
- expect(inflector.singularize('antennae')).toBe('antenna')
141
- expect(inflector.singularize('formulae')).toBe('formula')
142
- expect(inflector.singularize('nebulae')).toBe('nebula')
143
- expect(inflector.singularize('vertebrae')).toBe('vertebra')
144
- expect(inflector.singularize('vitae')).toBe('vita')
145
- })
146
-
147
- it('should allow AE cases to be S', function () {
148
- expect(inflector.singularize('antennas')).toBe('antenna')
149
- expect(inflector.singularize('formulas')).toBe('formula')
150
- })
151
- })
152
-
153
- describe('pluralization', function () {
154
- it('should append an S by default', function () {
155
- expect(inflector.pluralize('rrr')).toBe('rrrs')
156
- expect(inflector.pluralize('hacker')).toBe('hackers')
157
- expect(inflector.pluralize('movie')).toBe('movies')
158
- })
159
-
160
- it('should handle ambiguous form', function () {
161
- expect(inflector.pluralize('deer')).toBe('deer')
162
- expect(inflector.pluralize('fish')).toBe('fish')
163
- expect(inflector.pluralize('series')).toBe('series')
164
- expect(inflector.pluralize('sheep')).toBe('sheep')
165
- expect(inflector.pluralize('trout')).toBe('trout')
166
- })
167
-
168
- it('should convert singulars ending s to ses', function () {
169
- expect(inflector.pluralize('status')).toBe('statuses')
170
- expect(inflector.pluralize('bus')).toBe('buses')
171
- })
172
-
173
- it('should match irregulars', function () {
174
- expect(inflector.pluralize('person')).toBe('people')
175
- expect(inflector.pluralize('child')).toBe('children')
176
- expect(inflector.pluralize('ox')).toBe('oxen')
177
- })
178
-
179
- it('should maintain case of irregulars', function () {
180
- expect(inflector.pluralize('OX')).toBe('OXEN')
181
- expect(inflector.pluralize('Person')).toBe('People')
182
- expect(inflector.pluralize('child')).toBe('children')
183
- expect(inflector.pluralize('cloth')).toBe('clothes')
184
- })
185
-
186
- it('should handle IX cases', function () {
187
- expect(inflector.pluralize('matrix')).toBe('matrices')
188
- expect(inflector.pluralize('index')).toBe('indices')
189
- expect(inflector.pluralize('cortex')).toBe('cortices')
190
- })
191
-
192
- it('should regulars to ES', function () {
193
- expect(inflector.pluralize('church')).toBe('churches')
194
- expect(inflector.pluralize('appendix')).toBe('appendixes')
195
- expect(inflector.pluralize('mess')).toBe('messes')
196
- expect(inflector.pluralize('quiz')).toBe('quizes')
197
- expect(inflector.pluralize('shoe')).toBe('shoes')
198
- })
199
-
200
- it('should handle SIS cases', function () {
201
- expect(inflector.pluralize('synopsis')).toBe('synopses')
202
- expect(inflector.pluralize('parenthesis')).toBe('parentheses')
203
- })
204
-
205
- it('should handle special OES cases', function () {
206
- expect(inflector.pluralize('tomato')).toBe('tomatoes')
207
- expect(inflector.pluralize('buffalo')).toBe('buffaloes')
208
- expect(inflector.pluralize('tornado')).toBe('tornadoes')
209
- })
210
-
211
- it('should handle I cases', function () {
212
- expect(inflector.pluralize('radius')).toBe('radii')
213
- expect(inflector.pluralize('octopus')).toBe('octopi')
214
- expect(inflector.pluralize('stimulus')).toBe('stimuli')
215
- expect(inflector.pluralize('nucleus')).toBe('nuclei')
216
- expect(inflector.pluralize('fungus')).toBe('fungi')
217
- expect(inflector.pluralize('cactus')).toBe('cacti')
218
- })
219
-
220
- it('should handle IVES cases', function () {
221
- expect(inflector.pluralize('knife')).toBe('knives')
222
- expect(inflector.pluralize('life')).toBe('lives')
223
- })
224
-
225
- it('should handle Y cases', function () {
226
- expect(inflector.pluralize('party')).toBe('parties')
227
- expect(inflector.pluralize('fly')).toBe('flies')
228
- expect(inflector.pluralize('victory')).toBe('victories')
229
- expect(inflector.pluralize('monstrosity')).toBe('monstrosities')
230
- })
231
-
232
- it('should handle [aeiou]Y cases', function () {
233
- expect(inflector.pluralize('day')).toBe('days')
234
- expect(inflector.pluralize('toy')).toBe('toys')
235
- expect(inflector.pluralize('journey')).toBe('journeys')
236
- })
237
-
238
- it('should handle SS cases', function () {
239
- expect(inflector.pluralize('dress')).toBe('dresses')
240
- expect(inflector.pluralize('dresses')).toBe('dresses')
241
- expect(inflector.pluralize('mess')).toBe('messes')
242
- })
243
-
244
- it('should handle MAN->MEN cases', function () {
245
- expect(inflector.pluralize('man')).toBe('men')
246
- expect(inflector.pluralize('woman')).toBe('women')
247
- expect(inflector.pluralize('workman')).toBe('workmen')
248
- expect(inflector.pluralize('rifleman')).toBe('riflemen')
249
- })
250
-
251
- it('should handle irregular cases', function () {
252
- expect(inflector.pluralize('foot')).toBe('feet')
253
- expect(inflector.pluralize('goose')).toBe('geese')
254
- expect(inflector.pluralize('tooth')).toBe('teeth')
255
- expect(inflector.pluralize('ephemeris')).toBe('ephemerides')
256
-
257
- // MAN cases that don't pluralize to MEN
258
- expect(inflector.pluralize('talisman')).toBe('talismans')
259
- expect(inflector.pluralize('human')).toBe('humans')
260
- expect(inflector.pluralize('prehuman')).toBe('prehumans')
261
- })
262
-
263
- it('should handle AE cases', function () {
264
- expect(inflector.pluralize('antenna')).toBe('antennae')
265
- expect(inflector.pluralize('formula')).toBe('formulae')
266
- expect(inflector.pluralize('nebula')).toBe('nebulae')
267
- expect(inflector.pluralize('vertebra')).toBe('vertebrae')
268
- expect(inflector.pluralize('vita')).toBe('vitae')
269
- })
270
- })
271
-
272
- it('should pluralize and singularize string from patch', function () {
273
- expect(inflector.pluralize('synopsis')).toBe('synopses')
274
- expect(inflector.singularize('synopses')).toBe('synopsis')
275
- expect(inflector.pluralize('mess')).toBe('messes')
276
- expect(inflector.singularize('messes')).toBe('mess')
277
- })
278
-
279
- describe('custom inflections', function () {
280
- it('should pluralize and singularize custom forms', function () {
281
- const myInflector = new NounInflector()
282
- myInflector.addPlural(/(code|ware)/i, '$1z')
283
- myInflector.addSingular(/(code|ware)z/i, '$1')
284
- expect(myInflector.pluralize('code')).toBe('codez')
285
- expect(myInflector.pluralize('ware')).toBe('warez')
286
- expect(myInflector.singularize('codez')).toBe('code')
287
- expect(myInflector.singularize('warez')).toBe('ware')
288
- })
289
-
290
- it('should not break regular forms', function () {
291
- const myInflector = new NounInflector()
292
- myInflector.addPlural(/(code|ware)/i, '$1z')
293
- myInflector.addSingular(/(code|ware)z/i, '$1')
294
- expect(myInflector.pluralize('bus')).toBe('buses')
295
- expect(myInflector.singularize('buses')).toBe('bus')
296
- })
297
- })
298
-
299
- describe('should handle words ending in f, ff, ffe', function () {
300
- it('should handle words ending in ff and ffe', function () {
301
- const myInflector = new NounInflector()
302
- expect(myInflector.pluralize('sherriff')).toBe('sherriffs')
303
- expect(myInflector.pluralize('giraffe')).toBe('giraffes')
304
- expect(myInflector.singularize('sherriffs')).toBe('sherriff')
305
- expect(myInflector.singularize('giraffes')).toBe('giraffe')
306
- })
307
-
308
- it('should handle words ending in f', function () {
309
- const myInflector = new NounInflector()
310
- expect(myInflector.pluralize('roof')).toBe('roofs')
311
- expect(myInflector.pluralize('chief')).toBe('chiefs')
312
- expect(myInflector.pluralize('oaf')).toBe('oafs')
313
- expect(myInflector.singularize('roofs')).toBe('roof')
314
- expect(myInflector.singularize('chiefs')).toBe('chief')
315
- expect(myInflector.singularize('oafs')).toBe('oaf')
316
- })
317
-
318
- it('should handle words ending in f or fe', function () {
319
- const myInflector = new NounInflector()
320
- expect(myInflector.pluralize('leaf')).toBe('leaves')
321
- expect(myInflector.pluralize('wolf')).toBe('wolves')
322
- expect(myInflector.pluralize('calf')).toBe('calves')
323
- expect(myInflector.pluralize('half')).toBe('halves')
324
- expect(myInflector.pluralize('knife')).toBe('knives')
325
- expect(myInflector.pluralize('loaf')).toBe('loaves')
326
- expect(myInflector.pluralize('life')).toBe('lives')
327
- expect(myInflector.pluralize('wife')).toBe('wives')
328
- expect(myInflector.pluralize('shelf')).toBe('shelves')
329
- expect(myInflector.pluralize('thief')).toBe('thieves')
330
- expect(myInflector.pluralize('yourself')).toBe('yourselves')
331
-
332
- expect(myInflector.singularize('leaves')).toBe('leaf')
333
- expect(myInflector.singularize('wolves')).toBe('wolf')
334
- expect(myInflector.singularize('calves')).toBe('calf')
335
- expect(myInflector.singularize('halves')).toBe('half')
336
- expect(myInflector.singularize('knives')).toBe('knife')
337
- expect(myInflector.singularize('loaves')).toBe('loaf')
338
- expect(myInflector.singularize('lives')).toBe('life')
339
- expect(myInflector.singularize('wives')).toBe('wife')
340
- expect(myInflector.singularize('shelves')).toBe('shelf')
341
- expect(myInflector.singularize('thieves')).toBe('thief')
342
- expect(myInflector.singularize('yourselves')).toBe('yourself')
343
- })
344
-
345
- it('should handle words ending in rf', function () {
346
- const myInflector = new NounInflector()
347
- expect(myInflector.pluralize('scarf')).toBe('scarfs')
348
- expect(myInflector.pluralize('dwarf')).toBe('dwarfs')
349
- expect(myInflector.pluralize('handkerchief')).toBe('handkerchiefs')
350
- expect(myInflector.pluralize('wharf')).toBe('wharfs')
351
- expect(myInflector.singularize('scarfs')).toBe('scarf')
352
- expect(myInflector.singularize('dwarfs')).toBe('dwarf')
353
- expect(myInflector.singularize('wharfs')).toBe('wharf')
354
- expect(myInflector.singularize('handkerchiefs')).toBe('handkerchief')
355
- expect(myInflector.singularize('scarves')).toBe('scarf')
356
- expect(myInflector.singularize('dwarves')).toBe('dwarf')
357
- expect(myInflector.singularize('wharves')).toBe('wharf')
358
- expect(myInflector.singularize('handkerchieves')).toBe('handkerchief')
359
- })
360
-
361
- it('should handle words ending in rf', function () {
362
- const myInflector = new NounInflector()
363
- expect(myInflector.singularize('expenses')).toBe('expense')
364
- expect(myInflector.singularize('defenses')).toBe('defense')
365
- expect(myInflector.pluralize('expense')).toBe('expenses')
366
- expect(myInflector.pluralize('defense')).toBe('defenses')
367
- })
368
- })
369
- })
@@ -1,47 +0,0 @@
1
- /*
2
- Copyright (c) 2018, Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { OrthographyTokenizer } from 'lib/natural'
26
-
27
- const sentencesInFinnish = [
28
- 'Mikä sinun nimesi on?',
29
- 'Hyvää kiitos, entä sinulle?',
30
- 'Tämä herrasmies maksaa kaiken'
31
- ]
32
-
33
- const results = [
34
- ['Mikä', 'sinun', 'nimesi', 'on'],
35
- ['Hyvää', 'kiitos', 'entä', 'sinulle'],
36
- ['Tämä', 'herrasmies', 'maksaa', 'kaiken']
37
- ]
38
-
39
- describe('The orthography tokenizer tokenizes sentences in Finnish', function () {
40
- const tokenizer = new OrthographyTokenizer({ language: 'fi' })
41
- sentencesInFinnish.forEach(function (sentence, index) {
42
- it('It should correctly tokenize the following sentence: ' + sentence, function () {
43
- // console.log(tokenizer.tokenize(sentencePlusResult[0]));
44
- expect(tokenizer.tokenize(sentence)).toEqual(results[index])
45
- })
46
- })
47
- })
@@ -1,53 +0,0 @@
1
- /*
2
- Copyright (c) 2023, Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { PorterStemmerDe as stemmer } from 'lib/natural'
26
- import rawJson from '../spec/test_data/snowball_de.json'
27
- const snowBallDict = rawJson as Record<string, string>
28
- const DEBUG = false
29
-
30
- describe('porter_stemmer_de', function () {
31
- it('should perform stemming on a lot of words', function () {
32
- const ok = []
33
- const ko = []
34
-
35
- Object.keys(snowBallDict).forEach(word => {
36
- const stemmed = stemmer.stem(word)
37
- const expectedStem = snowBallDict[word]
38
- DEBUG && console.log(word + ' -> ' + stemmed)
39
-
40
- if (stemmed === expectedStem) {
41
- ok.push(word)
42
- } else {
43
- ko.push({
44
- word,
45
- expected: expectedStem,
46
- actual: stemmed
47
- })
48
- }
49
- })
50
-
51
- expect(ko.length).toBe(0)
52
- })
53
- })
@@ -1,63 +0,0 @@
1
- /*
2
- Copyright (c) 2019, Luís Rodrigues (adapted for Spanish stemmer), Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { PorterStemmerEs as PorterStemmer } from 'lib/natural'
26
- import rawJson from '../spec/test_data/snowball_es.json'
27
- const snowBallDict = rawJson as Record<string, string>
28
- const DEBUG = false
29
-
30
- describe('porter_stemmer_es', function () {
31
- it('should perform stemming on a lot of words', function () {
32
- const errors = []
33
-
34
- Object.keys(snowBallDict).forEach(word => {
35
- const stemmed = PorterStemmer.stem(word)
36
- const expectedStem = snowBallDict[word]
37
- if (stemmed !== snowBallDict[word]) {
38
- DEBUG && console.log('Error:', word, 'Expected:', expectedStem, 'Got:', stemmed)
39
- errors.push({
40
- word,
41
- expected: expectedStem,
42
- actual: stemmed
43
- })
44
- }
45
- })
46
-
47
- expect(errors.length).toBe(0)
48
- })
49
-
50
- it('should tokenize a piece of text', function () {
51
- expect(PorterStemmer.tokenizeAndStem('Pues, en efecto, es solo el plano y lo que muestra aquello que interesa a Zemeckis.', true)).toEqual(
52
- ['pues', 'en', 'efect', 'es', 'sol', 'el', 'plan', 'y', 'lo', 'que', 'muestr', 'aquell', 'que', 'interes', 'a', 'zemeckis'])
53
- })
54
-
55
- /*
56
- it('should work with the attached notation', function() {
57
- PorterStemmer.attach();
58
- expect("efecto".stem()).toEqual("efect");
59
- expect("Pues, en efecto, es solo el plano y lo que muestra aquello que interesa a Zemeckis.".tokenizeAndStem(true)).toEqual(
60
- [ 'pues', 'en', 'efect', 'es', 'sol', 'el', 'plan', 'y', 'lo', 'que', 'muestr', 'aquell', 'que', 'interes', 'a', 'zemeckis' ]);
61
- });
62
- */
63
- })
@@ -1,103 +0,0 @@
1
- /*
2
- Copyright (c) 2019, Ismaël Héry, Hugo W.L. ter Doest
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
21
- */
22
-
23
- 'use strict'
24
-
25
- import { PorterStemmerFr as stemmer } from 'lib/natural'
26
- import rawJson from '../spec/test_data/snowball_fr.json'
27
- const snowBallDict = rawJson as Record<string, string>
28
-
29
- describe('porter_stemmer', function () {
30
- it('should prelude', function () {
31
- expect(stemmer.prelude('JOUER')).toBe('joUer')
32
- expect(stemmer.prelude('ennuie')).toBe('ennuIe')
33
- expect(stemmer.prelude('yeux')).toBe('Yeux')
34
- expect(stemmer.prelude('quand')).toBe('qUand')
35
- })
36
-
37
- it('should compute regions', function () {
38
- expect(stemmer.regions('fameusement').r1).toBe(3)
39
- expect(stemmer.regions('fameusement').r2).toBe(6)
40
-
41
- expect(stemmer.regions('taii').r1).toBe(4)
42
- expect(stemmer.regions('taii').r2).toBe(4)
43
-
44
- expect(stemmer.regions('parade').rv).toBe(3)
45
- expect(stemmer.regions('colet').rv).toBe(3)
46
- expect(stemmer.regions('tapis').rv).toBe(3)
47
- expect(stemmer.regions('aimer').rv).toBe(3)
48
- expect(stemmer.regions('adorer').rv).toBe(3)
49
- expect(stemmer.regions('voler').rv).toBe(2)
50
- expect(stemmer.regions('tue').rv).toBe(2)
51
- })
52
-
53
- it('should compute longest suffix ends in Arr', function () {
54
- expect(stemmer.endsinArr('voudriez', ['ez', 'iez', 'z'])).toBe('iez')
55
- })
56
-
57
- it('should stem some word', function () {
58
- expect(stemmer.stem('volera')).toBe('vol')
59
- expect(stemmer.stem('volerait')).toBe('vol')
60
- expect(stemmer.stem('subitement')).toBe('subit')
61
- expect(stemmer.stem('tempérament')).toBe('temper')
62
- expect(stemmer.stem('voudriez')).toBe('voudr')
63
- expect(stemmer.stem('vengeait')).toBe('veng')
64
- expect(stemmer.stem('saisissement')).toBe('sais')
65
- expect(stemmer.stem('transatlantique')).toBe('transatlant')
66
- expect(stemmer.stem('premièrement')).toBe('premi')
67
- expect(stemmer.stem('instruments')).toBe('instrument')
68
- expect(stemmer.stem('trouverions')).toBe('trouv')
69
- expect(stemmer.stem('voyiez')).toBe('voi')
70
- expect(stemmer.stem('publicité')).toBe('publiqu')
71
- expect(stemmer.stem('pitoyable')).toBe('pitoi')
72
- })
73
-
74
- it('should perform stemming on a lot of words', function () {
75
- const ok = []
76
- const ko = []
77
-
78
- Object.keys(snowBallDict).forEach(word => {
79
- const stemmed = stemmer.stem(word)
80
- const expectedStem = snowBallDict[word]
81
-
82
- const regs = stemmer.regions(word)
83
- const txtRegions = {
84
- r1: word.substring(regs.r1),
85
- r2: word.substring(regs.r2),
86
- rv: word.substring(regs.rv)
87
- }
88
-
89
- if (stemmed === expectedStem) {
90
- ok.push(word)
91
- } else {
92
- ko.push({
93
- word,
94
- expected: expectedStem,
95
- actual: stemmed,
96
- regions: txtRegions
97
- })
98
- }
99
- })
100
-
101
- expect(ko.length).toBe(0)
102
- })
103
- })