natural 0.5.6 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/.dockerignore +2 -0
  2. package/.idea/misc.xml +6 -0
  3. package/.idea/modules.xml +8 -0
  4. package/.idea/natural.iml +12 -0
  5. package/.idea/vcs.xml +6 -0
  6. package/.idea/workspace.xml +511 -0
  7. package/.travis.yml +8 -1
  8. package/README.md +331 -40
  9. package/bayes_classifier.json +1 -1
  10. package/classifier.json +418 -0
  11. package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tagger.js +8 -9
  12. package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tester.js +5 -5
  13. package/lib/natural/brill_pos_tagger/lib/Brill_POS_Trainer.js +10 -6
  14. package/lib/natural/brill_pos_tagger/lib/Corpus.js +21 -16
  15. package/lib/natural/brill_pos_tagger/lib/Lexicon.js +22 -13
  16. package/lib/natural/brill_pos_tagger/lib/Predicate.js +2 -2
  17. package/lib/natural/brill_pos_tagger/lib/RuleTemplate.js +2 -2
  18. package/lib/natural/brill_pos_tagger/lib/RuleTemplates.js +132 -146
  19. package/lib/natural/brill_pos_tagger/lib/Sentence.js +41 -0
  20. package/lib/natural/brill_pos_tagger/lib/TransformationRule.js +10 -11
  21. package/lib/natural/classifiers/classifier.js +5 -3
  22. package/lib/natural/classifiers/maxent/Classifier.js +125 -0
  23. package/lib/natural/classifiers/maxent/Context.js +33 -0
  24. package/lib/natural/classifiers/maxent/Distribution.js +145 -0
  25. package/lib/natural/classifiers/maxent/Element.js +34 -0
  26. package/lib/natural/classifiers/maxent/Feature.js +88 -0
  27. package/lib/natural/classifiers/maxent/FeatureSet.js +67 -0
  28. package/lib/natural/classifiers/maxent/GISScaler.js +148 -0
  29. package/lib/natural/classifiers/maxent/POS/ME_Corpus.js +56 -0
  30. package/lib/natural/classifiers/maxent/POS/ME_Sentence.js +73 -0
  31. package/lib/natural/classifiers/maxent/POS/POS_Element.js +172 -0
  32. package/lib/natural/classifiers/maxent/Sample.js +141 -0
  33. package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js +49 -0
  34. package/lib/natural/distance/hamming_distance.js +52 -0
  35. package/lib/natural/distance/jaro-winkler_distance.js +16 -6
  36. package/lib/natural/distance/levenshtein_distance.js +2 -2
  37. package/lib/natural/index.js +21 -2
  38. package/lib/natural/inflectors/noun_inflector.js +5 -3
  39. package/lib/natural/ngrams/ngrams.js +83 -25
  40. package/lib/natural/sentiment/Basque/senticon_eu.json +21557 -0
  41. package/lib/natural/sentiment/Catalan/senticon_ca.json +36352 -0
  42. package/lib/natural/sentiment/Dutch/negations_du.json +3 -0
  43. package/lib/natural/sentiment/Dutch/pattern-sentiment-nl.json +39650 -0
  44. package/lib/natural/sentiment/English/negations_en.json +3 -0
  45. package/lib/natural/sentiment/English/pattern-sentiment-en.json +16024 -0
  46. package/lib/natural/sentiment/English/senticon_en.json +124197 -0
  47. package/lib/natural/sentiment/French/pattern-sentiment-fr.json +40906 -0
  48. package/lib/natural/sentiment/Galician/senticon_gl.json +24427 -0
  49. package/lib/natural/sentiment/Italian/pattern-sentiment-it.json +24522 -0
  50. package/lib/natural/sentiment/SentimentAnalyzer.js +116 -0
  51. package/lib/natural/sentiment/Spanish/afinnShortSortedSpanish.json +1655 -0
  52. package/lib/natural/sentiment/Spanish/negations_es.json +3 -0
  53. package/lib/natural/sentiment/Spanish/senticon_es.json +56722 -0
  54. package/lib/natural/sentiment/tools/README.md +31 -0
  55. package/lib/natural/sentiment/tools/XmlParser4PatternData.js +31 -0
  56. package/lib/natural/sentiment/tools/sentimentXmlParser.js +30 -0
  57. package/lib/natural/stemmers/indonesian/data/kata-dasar.json +29935 -0
  58. package/lib/natural/stemmers/indonesian/prefix_rules.js +10 -9
  59. package/lib/natural/stemmers/indonesian/stemmer_id.js +16 -15
  60. package/lib/natural/stemmers/indonesian/suffix_rules.js +2 -2
  61. package/lib/natural/stemmers/porter_stemmer_es.js +265 -199
  62. package/lib/natural/stemmers/porter_stemmer_fr.js +1 -1
  63. package/lib/natural/stemmers/porter_stemmer_nl.js +362 -0
  64. package/lib/natural/stemmers/stemmer_es.js +18 -12
  65. package/lib/natural/stemmers/stemmer_nl.js +64 -0
  66. package/lib/natural/tfidf/tfidf.js +27 -4
  67. package/lib/natural/tokenizers/aggressive_tokenizer_sv.js +1 -1
  68. package/lib/natural/tokenizers/aggressive_tokenizer_vi.js +37 -0
  69. package/lib/natural/tokenizers/orthography_matchers.js +33 -0
  70. package/lib/natural/tokenizers/regexp_tokenizer.js +24 -4
  71. package/lib/natural/tokenizers/sentence_tokenizer.js +1 -1
  72. package/lib/natural/wordnet/wordnet_file.js +7 -1
  73. package/lr_classifier.json +1 -1
  74. package/package.json +35 -23
  75. package/.npmignore +0 -4
  76. package/.vscode/temp.sql +0 -0
  77. package/Makefile +0 -38
  78. package/data/kata-dasar.txt +0 -29933
  79. package/package-lock.json +0 -79
  80. package/tests/stemmer_id_test.js +0 -361
@@ -29,17 +29,18 @@ var Removal = require("./removal");
29
29
  var dictionary = [];
30
30
  function loadDictionary(){
31
31
  var fs = require('fs');
32
- var dirname = __dirname + "/../../../../data/kata-dasar.txt";
33
- var fin = fs.readFileSync(dirname).toString().split("\n");
34
- for(var i in fin){
35
- var word = fin[i];
36
- word = word.trim();
37
- dictionary.push(word);
38
- }
32
+ //var dirname = __dirname + "/../../../../data/kata-dasar.txt";
33
+ //var fin = fs.readFileSync(dirname).toString().split("\n");
34
+ var fin = require('./data/kata-dasar.json');
35
+ fin.forEach(function (word) {
36
+ if (word) {
37
+ dictionary.push(word.trim());
38
+ }
39
+ });
39
40
  }
40
41
  loadDictionary();
41
42
 
42
- PrefixRules = function() {
43
+ function PrefixRules() {
43
44
  var PrefixRules = this;
44
45
 
45
46
  this.removal = undefined;
@@ -1024,4 +1025,4 @@ rules.push(pr.DisambiguatorPrefixRule40);
1024
1025
  rules.push(pr.DisambiguatorPrefixRule41);
1025
1026
  rules.push(pr.DisambiguatorPrefixRule42);
1026
1027
 
1027
- PrefixRules.rules = rules;
1028
+ PrefixRules.rules = rules;
@@ -129,20 +129,21 @@ function find(word) {
129
129
 
130
130
  function loadDictionary(){
131
131
  var fs = require('fs');
132
- var dirname = __dirname + "/../../../../data/kata-dasar.txt";
133
- var fin = fs.readFileSync(dirname).toString().split("\n");
134
- for(var i in fin){
135
- var word = fin[i];
136
- word = word.trim();
137
- dictionary.push(word);
138
- }
132
+ //var dirname = __dirname + "/../../../../data/kata-dasar.txt";
133
+ //var fin = fs.readFileSync(dirname).toString().split("\n");
134
+ var fin = require('./data/kata-dasar.json');
135
+ fin.forEach(function (word) {
136
+ if (word) {
137
+ dictionary.push(word.trim());
138
+ }
139
+ });
139
140
  }
140
141
 
141
142
  // Stemming from step 2-5
142
143
  function stemmingProcess(){
143
144
  if(find(current_word))
144
145
  return
145
-
146
+
146
147
  // Confix Stripping
147
148
  // Try to remove prefixes first before suffixes if the specification is met
148
149
  if(precedenceAdjustmentSpecification(original_word)){
@@ -168,7 +169,7 @@ function stemmingProcess(){
168
169
  removeSuffixes();
169
170
  if(find(current_word))
170
171
  return
171
-
172
+
172
173
  // Step 4, 5
173
174
  removePrefixes();
174
175
  if(find(current_word))
@@ -182,7 +183,7 @@ function stemmingProcess(){
182
183
  function removeSuffixes(){
183
184
  for(var i in suffix_rules){
184
185
  resultObj = suffix_rules[i](current_word);
185
-
186
+
186
187
  // Add result to variable
187
188
  if(resultObj.removal!=undefined){
188
189
  removals.push(resultObj.removal);
@@ -209,7 +210,7 @@ function checkPrefixRules(){
209
210
  var j = 0;
210
211
  for(j=0; j<prefix_rules.length; j++){
211
212
  resultObj = prefix_rules[j](current_word);
212
-
213
+
213
214
  // Add result to variable
214
215
  if(resultObj.removal!=undefined){
215
216
  removals.push(resultObj.removal);
@@ -233,7 +234,7 @@ function loopRestorePrefixes(){
233
234
 
234
235
  for(var i in reversed_removals){
235
236
  current_removal = reversed_removals[i];
236
-
237
+
237
238
  if(!isSuffixRemovals(current_removal)){
238
239
  continue
239
240
  }
@@ -245,7 +246,7 @@ function loopRestorePrefixes(){
245
246
  removePrefixes();
246
247
  if(find(current_word))
247
248
  return
248
- current_word = current_removal.getResult() + "kan";
249
+ current_word = current_removal.getResult() + "kan";
249
250
  }
250
251
  else{
251
252
  current_word = current_removal.getOriginalWord();
@@ -292,7 +293,7 @@ function precedenceAdjustmentSpecification(word){
292
293
  /^pe(.*)i$/,
293
294
  /^ter(.*)i$/,
294
295
  ];
295
-
296
+
296
297
  for(var i in regex_rules){
297
298
  if(word.match(regex_rules[i])){
298
299
  return true;
@@ -304,4 +305,4 @@ function precedenceAdjustmentSpecification(word){
304
305
  //exports for tests
305
306
  stemmer.isPlural = isPlural;
306
307
  stemmer.dictionary = dictionary;
307
- stemmer.a = suffix_rules[0];
308
+ stemmer.a = suffix_rules[0];
@@ -25,7 +25,7 @@ THE SOFTWARE.
25
25
 
26
26
  var Removal = require("./removal");
27
27
 
28
- SuffixRules = function() {
28
+ function SuffixRules() {
29
29
  var SuffixRules = this;
30
30
 
31
31
  this.removal = undefined;
@@ -72,4 +72,4 @@ rules.push(sr.RemoveInflectionalParticle);
72
72
  rules.push(sr.RemoveInflectionalPossessivePronoun);
73
73
  rules.push(sr.RemoveDerivationalSuffix);
74
74
 
75
- SuffixRules.rules = rules;
75
+ SuffixRules.rules = rules;
@@ -1,221 +1,287 @@
1
1
  /*
2
- Copyright (c) 2012, David Przybilla, Chris Umbel
3
-
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
-
11
- The above copyright notice and this permission notice shall be included in
12
- all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
- THE SOFTWARE.
2
+ Copyright (c) 2018, Domingo Martín Mancera
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ of this software and associated documentation files (the "Software"), to deal
6
+ in the Software without restriction, including without limitation the rights
7
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ copies of the Software, and to permit persons to whom the Software is
9
+ furnished to do so, subject to the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be included in
12
+ all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
+ THE SOFTWARE.
21
21
  */
22
22
 
23
+ 'use strict';
24
+
23
25
  var Stemmer = require('./stemmer_es');
24
26
 
25
- var PorterStemmer = new Stemmer();
26
- module.exports = PorterStemmer;
27
-
28
-
29
- function isVowel(letter){
30
- return (letter == 'a' || letter == 'e' || letter == 'i' || letter == 'o' || letter == 'u' || letter == 'á' || letter == 'é' ||
31
- letter == 'í' || letter == 'ó' || letter == 'ú');
32
- };
33
-
34
- function getNextVowelPos(token,start){
35
- length=token.length
36
- for (var i = start; i < length; i++)
37
- if (isVowel(token[i])) return i;
38
- return length;
39
- };
40
-
41
- function getNextConsonantPos(token,start){
42
- length=token.length
43
- for (var i = start; i < length; i++)
44
- if (!isVowel(token[i])) return i;
45
- return length;
46
- };
47
-
48
-
49
- function endsin(token, suffix) {
50
- if (token.length < suffix.length) return false;
51
- return (token.slice(-suffix.length) == suffix);
52
- };
53
-
54
- function endsinArr(token, suffixes) {
55
- for(var i=0;i<suffixes.length;i++){
56
- if (endsin(token, suffixes[i])) return suffixes[i];
57
- }
58
- return '';
59
- };
60
-
61
- function removeAccent(token) {
62
- var str=token.replace(/á/gi,'a');
63
- str=str.replace(/é/gi,'e');
64
- str=str.replace(/í/gi,'i');
65
- str=str.replace(/ó/gi,'o');
66
- str=str.replace(/ú/gi,'u');
67
- return str;
68
- };
69
-
70
- // perform full stemming algorithm on a single word
71
- PorterStemmer.stem = function(token) {
72
- token = token.toLowerCase();
73
-
74
- if (token.length<3){
75
- return token;
76
- }
77
-
78
- var r1,r2,rv,len= token.length;
79
- //looking for regions after vowels
80
-
81
- for(var i=0; i< token.length-1 && r1==len;i++){
82
- if(isVowel(token[i]) && !isVowel(token[i+1]) ){
83
- r1=i+2;
84
- }
85
-
86
- }
87
-
88
- for(var i=r1; i< token.length-1 && r2==len;i++){
89
- if(isVowel(token[i]) && !isVowel(token[i+1])){
90
- r2=i+2;
27
+ // Inherit from the utility class in stemmer_es
28
+ class PorterStemmer extends Stemmer
29
+ {
30
+ constructor() {
31
+ super();
32
+ }
33
+
34
+ isVowel(c)
35
+ {
36
+ var regex = /[aeiouáéíóú]/gi;
37
+
38
+ return regex.test(c);
39
+ }
40
+
41
+ nextVowelPosition(word, start = 0)
42
+ {
43
+ var length = word.length;
44
+
45
+ for(var position = start; position < length; position++){
46
+ if(this.isVowel(word[position])){
47
+ return position;
48
+ }
49
+ }
50
+
51
+ return length;
52
+ }
53
+
54
+ nextConsonantPosition(word, start = 0)
55
+ {
56
+ var length = word.length;
57
+
58
+ for(var position = start; position < length; position++){
59
+ if(!this.isVowel(word[position])){
60
+ return position;
61
+ }
62
+ }
63
+
64
+ return length;
65
+ }
66
+
67
+ endsIn(word, suffix)
68
+ {
69
+ if(word.length < suffix.length){
70
+ return false;
71
+ }
72
+
73
+ return (word.slice(-suffix.length) === suffix);
74
+ }
75
+
76
+ endsInArr(word, suffixes)
77
+ {
78
+ var matches = [];
79
+ for(var i in suffixes) {
80
+ if(this.endsIn(word, suffixes[i])){
81
+ matches.push(suffixes[i]);
82
+ }
83
+ }
84
+ var longest = matches.sort(function (a, b) {
85
+ return b.length - a.length;
86
+ })[0];
87
+
88
+ if(longest){
89
+ return longest
90
+ } else {
91
+ return '';
92
+ }
93
+ }
94
+
95
+ removeAccent(word)
96
+ {
97
+ var accentedVowels = ['á', 'é', 'í', 'ó', 'ú'];
98
+ var vowels = ['a', 'e', 'i', 'o', 'u'];
99
+
100
+ for(var i in accentedVowels){
101
+ word = word.replace(accentedVowels[i], vowels[i]);
102
+ }
103
+
104
+ return word;
105
+ }
106
+
107
+ stem(word)
108
+ {
109
+ var length = word.length;
110
+
111
+ word.toLowerCase();
112
+
113
+ if(length < 2){
114
+ return this.removeAccent(word);
115
+ }
116
+
117
+ var r1, r2, rv;
118
+ r1 = length;
119
+ r2 = length;
120
+ rv = length;
121
+
122
+ // R1 is the region after the first non-vowel following a vowel, or is the null region
123
+ // at the end of the word if there is no such non-vowel.
124
+ for(var i = 0; i < (length - 1) && r1 == length; i++){
125
+ if(this.isVowel(word[i]) && !this.isVowel(word[i + 1])){
126
+ r1 = i + 2;
127
+ }
128
+ }
129
+
130
+ // R2 is the region after the first non-vowel following a vowel in R1,
131
+ // or is the null region at the end of the word if there is no such non-vowel.
132
+ for(var i = r1; i < (length - 1) && r2 == length; i++){
133
+ if(this.isVowel(word[i]) && !this.isVowel(word[i + 1])){
134
+ r2 = i + 2;
135
+ }
136
+ }
137
+
138
+ if(length > 3){
139
+ if(!this.isVowel(word[1])){
140
+ rv = this.nextVowelPosition(word, 2) + 1;
141
+ } else if(this.isVowel(word[0]) && this.isVowel(word[1])){
142
+ rv = this.nextConsonantPosition(word, 2) + 1;
143
+ } else {
144
+ rv = 3;
145
+ }
146
+ }
147
+
148
+ var r1Text = word.slice(r1);
149
+ var r2Text = word.slice(r2);
150
+ var rvText = word.slice(rv);
151
+ var originalWord = word;
152
+
153
+ // Step 0: Attached pronoun
154
+ var pronounSuffix = ['me', 'se', 'sela', 'selo', 'selas', 'selos', 'la', 'le', 'lo', 'las', 'les', 'los', 'nos'];
155
+ var pronounSuffixPre1 = ['iéndo', 'ándo', 'ár', 'ér', 'ír'];
156
+ var pronounSuffixPre2 = ['iendo', 'ando', 'ar', 'er', 'ir'];
157
+
158
+ var suffix = this.endsInArr(word, pronounSuffix);
159
+
160
+ if(suffix != ''){
161
+ var preSuffix = this.endsInArr(rvText.slice(0, -suffix.length), pronounSuffixPre1);
162
+
163
+ if(preSuffix != ''){
164
+ word = this.removeAccent(word.slice(0, -suffix.length));
165
+ } else {
166
+ preSuffix = this.endsInArr(rvText.slice(0, -suffix.length), pronounSuffixPre2);
167
+
168
+ if(preSuffix != '' || (this.endsIn(word.slice(0, -suffix.length), 'uyendo'))){
169
+
170
+ word = word.slice(0, -suffix.length);
171
+ }
172
+ }
173
+ }
174
+
175
+ if(word != originalWord){
176
+ r1Text = word.slice(r1);
177
+ r2Text = word.slice(r2);
178
+ rvText = word.slice(rv);
179
+ }
180
+
181
+ var wordAfter0 = word;
182
+
183
+ if(( suf = this.endsInArr(r2Text, ['anza', 'anzas', 'ico', 'ica', 'icos', 'icas', 'ismo', 'ismos',
184
+ 'able', 'ables', 'ible', 'ibles', 'ista', 'istas', 'oso', 'osa',
185
+ 'osos', 'osas', 'amiento', 'amientos', 'imiento', 'imientos'])) != '')
186
+ {
187
+ word = word.slice(0, -suf.length);
91
188
  }
92
- }
93
-
94
- if (len > 3) {
95
- if(isVowel(token[1])) {
96
- // If the second letter is a consonant, RV is the region after the next following vowel
97
- rv = getNextVowelPos(token, 2) +1;
98
- } else if (isVowel(token[0]) && isVowel(token[1])) {
99
- // or if the first two letters are vowels, RV is the region after the next consonant
100
- rv = getNextConsonantPos(token, 2) + 1;
101
- } else {
102
- //otherwise (consonant-vowel case) RV is the region after the third letter. But RV is the end of the word if these positions cannot be found.
103
- rv = 3;
104
- }
189
+ else if((suf = this.endsInArr(r2Text, ['icadora', 'icador', 'icación', 'icadoras', 'icadores', 'icaciones',
190
+ 'icante', 'icantes', 'icancia', 'icancias', 'adora', 'ador', 'ación',
191
+ 'adoras', 'adores', 'aciones', 'ante', 'antes', 'ancia', 'ancias'])) != '')
192
+ {
193
+ word = word.slice(0, -suf.length);
105
194
  }
106
-
107
- var r1_txt = token.substring(r1-1);
108
- var r2_txt = token.substring(r2-1);
109
- var rv_txt = token.substring(rv-1);
110
-
111
-
112
- var token_orig = token;
113
-
114
- // Step 0: Attached pronoun
115
- var pronoun_suf = new Array('me', 'se', 'sela', 'selo', 'selas', 'selos', 'la', 'le', 'lo', 'las', 'les', 'los', 'nos');
116
- var pronoun_suf_pre1 = new Array('éndo', 'ándo', 'ár', 'ér', 'ír');
117
- var pronoun_suf_pre2 = new Array('ando', 'iendo', 'ar', 'er', 'ir');
118
- var suf = endsinArr(token, pronoun_suf);
119
-
120
-
121
- if (suf!='') {
122
-
123
- var pre_suff = endsinArr(rv_txt.slice(0,-suf.length),pronoun_suf_pre1);
124
-
125
- if (pre_suff != '') {
126
-
127
- token = removeAccent(token.slice(0,-suf.length));
128
- } else {
129
- var pre_suff = endsinArr(rv_txt.slice(0,-suf.length),pronoun_suf_pre2);
130
-
131
- if (pre_suff != '' ||
132
- (endsin(token, 'yendo' ) &&
133
- (token.slice(-suf.length-6,1) == 'u'))) {
134
- token = token.slice(0,-suf.length);
135
- }
195
+ else if((suf = this.endsInArr(r2Text, ['logía', 'logías'])) != ''){
196
+ word = word.slice(0, -suf.length) + 'log';
136
197
  }
137
- }
138
-
139
- if (token != token_orig) {
140
- r1_txt = token.substring(r1-1);
141
- r2_txt = token.substring(r2-1);
142
- rv_txt = token.substring(rv-1);
198
+ else if((suf = this.endsInArr(r2Text, ['ución', 'uciones'])) != ''){
199
+ word = word.slice(0, -suf.length) + 'u';
143
200
  }
144
- var token_after0 = token;
145
-
146
- if ((suf = endsinArr(r2_txt, new Array('anza', 'anzas', 'ico', 'ica', 'icos', 'icas', 'ismo', 'ismos', 'able', 'ables', 'ible', 'ibles', 'ista', 'istas', 'oso', 'osa', 'osos', 'osas', 'amiento', 'amientos', 'imiento', 'imientos'))) != '') {
147
- token = token.slice(0, -suf.length);
148
- } else if ((suf = endsinArr(r2_txt, new Array('icadora', 'icador', 'icación', 'icadoras', 'icadores', 'icaciones', 'icante', 'icantes', 'icancia', 'icancias', 'adora', 'ador', 'ación', 'adoras', 'adores', 'aciones', 'ante', 'antes', 'ancia', 'ancias'))) != '') {
149
- token = token.slice(0, -suf.length);
150
- } else if ((suf = endsinArr(r2_txt, new Array('logía', 'logías'))) != '') {
151
- token = token.slice(0, -suf.length)+ 'log';
152
- } else if ((suf =endsinArr(r2_txt, new Array('ución', 'uciones'))) != '') {
153
- token = token.slice(0, -suf.length) + 'u';
154
- } else if ((suf = endsinArr(r2_txt, new Array('encia', 'encias'))) != '') {
155
- token = token.slice(0, -suf.length)+ 'ente';
156
- } else if ((suf = endsinArr(r2_txt, new Array('ativamente', 'ivamente', 'osamente', 'icamente', 'adamente'))) != '') {
157
- token = token.slice(0, -suf.length);
158
- } else if ((suf = endsinArr(r1_txt, new Array('amente'))) != '') {
159
- token = token.slice(0, -suf.length);
160
- } else if ((suf = endsinArr(r2_txt, new Array('antemente', 'ablemente', 'iblemente', 'mente'))) != '') {
161
- token = token.slice(0, -suf.length);
162
- } else if ((suf = endsinArr(r2_txt, new Array('abilidad', 'abilidades', 'icidad', 'icidades', 'ividad', 'ividades', 'idad', 'idades'))) != '') {
163
- token = token.slice(0, -suf.length);
164
- } else if ((suf = endsinArr(r2_txt, new Array('ativa', 'ativo', 'ativas', 'ativos', 'iva', 'ivo', 'ivas', 'ivos'))) != '') {
165
- token = token.slice(0, -suf.length);
201
+ else if((suf = this.endsInArr(r2Text, ['encia', 'encias'])) != ''){
202
+ word = word.slice(0, -suf.length) + 'ente';
166
203
  }
167
-
168
- if (token != token_after0) {
169
- r1_txt = token.substring(r1-1);
170
- r2_txt = token.substring(r2-1);
171
- rv_txt = token.substring(rv-1);
204
+ else if((suf = this.endsInArr(r2Text, ['ativamente', 'ivamente', 'osamente', 'icamente', 'adamente'])) != ''){
205
+ word = word.slice(0, -suf.length);
206
+ }
207
+ else if((suf = this.endsInArr(r1Text, ['amente'])) != ''){
208
+ word = word.slice(0, -suf.length);
209
+ }
210
+ else if((suf = this.endsInArr(r2Text, ['antemente', 'ablemente', 'iblemente', 'mente'])) != ''){
211
+ word = word.slice(0, -suf.length);
212
+ }
213
+ else if((suf = this.endsInArr(r2Text, ['abilidad', 'abilidades', 'icidad', 'icidades', 'ividad', 'ividades', 'idad', 'idades'])) != ''){
214
+ word = word.slice(0, -suf.length);
215
+ }
216
+ else if((suf = this.endsInArr(r2Text, ['ativa', 'ativo', 'ativas', 'ativos', 'iva', 'ivo', 'ivas', 'ivos'])) != ''){
217
+ word = word.slice(0, -suf.length);
172
218
  }
173
- var token_after1 = token;
174
219
 
175
- if (token_after0 == token_after1) {
176
- // Do step 2a if no ending was removed by step 1.
177
- if ((suf = endsinArr(rv_txt, new Array('ya', 'ye', 'yan', 'yen', 'yeron', 'yendo', 'yo', 'yó', 'yas', 'yes', 'yais', 'yamos'))) != '' && (token.substring(suf.length-1,1) == 'u')) {
178
- token = token.slice(0, -suf.length);
179
- }
220
+ if(word != wordAfter0){
221
+ r1Text = word.slice(r1);
222
+ r2Text = word.slice(r2);
223
+ rvText = word.slice(rv);
224
+ }
225
+ var wordAfter1 = word;
180
226
 
181
- if (token != token_after1) {
182
- r1_txt = token.substring(r1-1);
183
- r2_txt = token.substring(r2-1);
184
- rv_txt = token.substring(rv-1);
185
- }
186
- var token_after2a = token;
227
+ if(wordAfter0 === wordAfter1){
187
228
 
188
- // Do Step 2b if step 2a was done, but failed to remove a suffix.
189
- if (token_after2a == token_after1) {
229
+ // Do step 2a if no ending was removed by step 1.
230
+ var suf = this.endsInArr(rvText, ['ya', 'ye', 'yan', 'yen', 'yeron', 'yendo', 'yo', 'yó', 'yas', 'yes', 'yais', 'yamos']);
190
231
 
191
- if ((suf = endsinArr(rv_txt,new Array('en', 'es', 'éis', 'emos'))) != '') {
192
- token = token.slice(0,-suf.length);
193
- if (endsin(token, 'gu')) {
194
- token = token.slice(0,-1);
195
- }
196
- } else if ((suf = endsinArr(rv_txt, new Array('arían', 'arías', 'arán', 'arás', 'aríais', 'aría', 'aréis', 'aríamos', 'aremos', 'ará', 'aré', 'erían', 'erías', 'erán', 'erás', 'eríais', 'ería', 'eréis', 'eríamos', 'eremos', 'erá', 'eré', 'irían', 'irías', 'irán', 'irás', 'iríais', 'iría', 'iréis', 'iríamos', 'iremos', 'irá', 'iré', 'aba', 'ada', 'ida', 'ía', 'ara', 'iera', 'ad', 'ed', 'id', 'ase', 'iese', 'aste', 'iste', 'an', 'aban', 'ían', 'aran', 'ieran', 'asen', 'iesen', 'aron', 'ieron', 'ado', 'ido', 'ando', 'iendo', 'ió', 'ar', 'er', 'ir', 'as', 'abas', 'adas', 'idas', 'ías', 'aras', 'ieras', 'ases', 'ieses', 'ís', 'áis', 'abais', 'íais', 'arais', 'ierais', ' aseis', 'ieseis', 'asteis', 'isteis', 'ados', 'idos', 'amos', 'ábamos', 'íamos', 'imos', 'áramos', 'iéramos', 'iésemos', 'ásemos'))) != '') {
197
-
198
- token = token.slice(0, -suf.length);
232
+ if(suf != '' && (word.slice(-suf.length - 1, -suf.length) == 'u')){
233
+ word = word.slice(0, -suf.length);
234
+ }
199
235
 
236
+ if(word != wordAfter1){
237
+ r1Text = word.slice(r1);
238
+ r2Text = word.slice(r2);
239
+ rvText = word.slice(rv);
240
+ }
241
+
242
+ var wordAfter2a = word;
243
+ // Do Step 2b if step 2a was done, but failed to remove a suffix.
244
+ if (wordAfter2a == wordAfter1) {
245
+ if((suf = this.endsInArr(rvText, ['arían', 'arías', 'arán', 'arás', 'aríais', 'aría', 'aréis',
246
+ 'aríamos', 'aremos', 'ará', 'aré', 'erían', 'erías', 'erán',
247
+ 'erás', 'eríais', 'ería', 'eréis', 'eríamos', 'eremos', 'erá',
248
+ 'eré', 'irían', 'irías', 'irán', 'irás', 'iríais', 'iría', 'iréis',
249
+ 'iríamos', 'iremos', 'irá', 'iré', 'aba', 'ada', 'ida', 'ía', 'ara',
250
+ 'iera', 'ad', 'ed', 'id', 'ase', 'iese', 'aste', 'iste', 'an',
251
+ 'aban', 'ían', 'aran', 'ieran', 'asen', 'iesen', 'aron', 'ieron',
252
+ 'ado', 'ido', 'ando', 'iendo', 'ió', 'ar', 'er', 'ir', 'as', 'abas',
253
+ 'adas', 'idas', 'ías', 'aras', 'ieras', 'ases', 'ieses', 'ís', 'áis',
254
+ 'abais', 'íais', 'arais', 'ierais', ' aseis', 'ieseis', 'asteis',
255
+ 'isteis', 'ados', 'idos', 'amos', 'ábamos', 'íamos', 'imos', 'áramos',
256
+ 'iéramos', 'iésemos', 'ásemos'])) != '')
257
+ {
258
+ word = word.slice(0, -suf.length);
259
+ }else if((suf = this.endsInArr(rvText, ['en', 'es', 'éis', 'emos'])) != '') {
260
+ word = word.slice(0, -suf.length);
261
+ if(this.endsIn(word, 'gu')){
262
+ word = word.slice(0, -1);
263
+ }
200
264
  }
265
+ }
266
+ }
267
+
268
+ r1Text = word.slice(r1);
269
+ r2Text = word.slice(r2);
270
+ rvText = word.slice(rv);
271
+
272
+ if ((suf = this.endsInArr(rvText, ['os', 'a', 'o', 'á', 'í', 'ó'])) != '') {
273
+ word = word.slice(0, -suf.length);
274
+ } else if ((this.endsInArr(rvText , ['e','é'])) != '') {
275
+ word = word.slice(0, -1);
276
+ rvText = word.slice(rv);
277
+ if (this.endsIn(rvText, 'u') && this.endsIn(word, 'gu')) {
278
+ word = word.slice(0, -1);
201
279
  }
202
280
  }
203
281
 
204
- // Always do step 3.
205
- r1_txt = token.substring(r1-1);
206
- r2_txt = token.substring(r2-1);
207
- rv_txt = token.substring(rv-1);
208
-
209
- if ((suf = endsinArr(rv_txt, new Array('os', 'a', 'o', 'á', 'í', 'ó'))) != '') {
210
- token = token.slice(0, -suf.length);
211
- } else if ((suf = endsinArr(rv_txt ,new Array('e','é'))) != '') {
212
- token = token.slice(0,-1);
213
- rv_txt = token.substring(rv-1);
214
- if (endsin(rv_txt,'u') && endsin(token,'gu')) {
215
- token = token.slice(0,-1);
216
- }
217
- }
282
+ return this.removeAccent(word);
283
+ }
218
284
 
219
- return removeAccent(token);
285
+ }
220
286
 
221
- };
287
+ module.exports = new PorterStemmer();
@@ -374,4 +374,4 @@ function isVowel(letter) {
374
374
  function endsin(token, suffix) {
375
375
  if (token.length < suffix.length) return false;
376
376
  return (token.slice(-suffix.length) == suffix);
377
- };
377
+ };