natural 0.5.6 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +2 -0
- package/.idea/misc.xml +6 -0
- package/.idea/modules.xml +8 -0
- package/.idea/natural.iml +12 -0
- package/.idea/vcs.xml +6 -0
- package/.idea/workspace.xml +511 -0
- package/.travis.yml +8 -1
- package/README.md +331 -40
- package/bayes_classifier.json +1 -1
- package/classifier.json +418 -0
- package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tagger.js +8 -9
- package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tester.js +5 -5
- package/lib/natural/brill_pos_tagger/lib/Brill_POS_Trainer.js +10 -6
- package/lib/natural/brill_pos_tagger/lib/Corpus.js +21 -16
- package/lib/natural/brill_pos_tagger/lib/Lexicon.js +22 -13
- package/lib/natural/brill_pos_tagger/lib/Predicate.js +2 -2
- package/lib/natural/brill_pos_tagger/lib/RuleTemplate.js +2 -2
- package/lib/natural/brill_pos_tagger/lib/RuleTemplates.js +132 -146
- package/lib/natural/brill_pos_tagger/lib/Sentence.js +41 -0
- package/lib/natural/brill_pos_tagger/lib/TransformationRule.js +10 -11
- package/lib/natural/classifiers/classifier.js +5 -3
- package/lib/natural/classifiers/maxent/Classifier.js +125 -0
- package/lib/natural/classifiers/maxent/Context.js +33 -0
- package/lib/natural/classifiers/maxent/Distribution.js +145 -0
- package/lib/natural/classifiers/maxent/Element.js +34 -0
- package/lib/natural/classifiers/maxent/Feature.js +88 -0
- package/lib/natural/classifiers/maxent/FeatureSet.js +67 -0
- package/lib/natural/classifiers/maxent/GISScaler.js +148 -0
- package/lib/natural/classifiers/maxent/POS/ME_Corpus.js +56 -0
- package/lib/natural/classifiers/maxent/POS/ME_Sentence.js +73 -0
- package/lib/natural/classifiers/maxent/POS/POS_Element.js +172 -0
- package/lib/natural/classifiers/maxent/Sample.js +141 -0
- package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js +49 -0
- package/lib/natural/distance/hamming_distance.js +52 -0
- package/lib/natural/distance/jaro-winkler_distance.js +16 -6
- package/lib/natural/distance/levenshtein_distance.js +2 -2
- package/lib/natural/index.js +21 -2
- package/lib/natural/inflectors/noun_inflector.js +5 -3
- package/lib/natural/ngrams/ngrams.js +83 -25
- package/lib/natural/sentiment/Basque/senticon_eu.json +21557 -0
- package/lib/natural/sentiment/Catalan/senticon_ca.json +36352 -0
- package/lib/natural/sentiment/Dutch/negations_du.json +3 -0
- package/lib/natural/sentiment/Dutch/pattern-sentiment-nl.json +39650 -0
- package/lib/natural/sentiment/English/negations_en.json +3 -0
- package/lib/natural/sentiment/English/pattern-sentiment-en.json +16024 -0
- package/lib/natural/sentiment/English/senticon_en.json +124197 -0
- package/lib/natural/sentiment/French/pattern-sentiment-fr.json +40906 -0
- package/lib/natural/sentiment/Galician/senticon_gl.json +24427 -0
- package/lib/natural/sentiment/Italian/pattern-sentiment-it.json +24522 -0
- package/lib/natural/sentiment/SentimentAnalyzer.js +116 -0
- package/lib/natural/sentiment/Spanish/afinnShortSortedSpanish.json +1655 -0
- package/lib/natural/sentiment/Spanish/negations_es.json +3 -0
- package/lib/natural/sentiment/Spanish/senticon_es.json +56722 -0
- package/lib/natural/sentiment/tools/README.md +31 -0
- package/lib/natural/sentiment/tools/XmlParser4PatternData.js +31 -0
- package/lib/natural/sentiment/tools/sentimentXmlParser.js +30 -0
- package/lib/natural/stemmers/indonesian/data/kata-dasar.json +29935 -0
- package/lib/natural/stemmers/indonesian/prefix_rules.js +10 -9
- package/lib/natural/stemmers/indonesian/stemmer_id.js +16 -15
- package/lib/natural/stemmers/indonesian/suffix_rules.js +2 -2
- package/lib/natural/stemmers/porter_stemmer_es.js +265 -199
- package/lib/natural/stemmers/porter_stemmer_fr.js +1 -1
- package/lib/natural/stemmers/porter_stemmer_nl.js +362 -0
- package/lib/natural/stemmers/stemmer_es.js +18 -12
- package/lib/natural/stemmers/stemmer_nl.js +64 -0
- package/lib/natural/tfidf/tfidf.js +27 -4
- package/lib/natural/tokenizers/aggressive_tokenizer_sv.js +1 -1
- package/lib/natural/tokenizers/aggressive_tokenizer_vi.js +37 -0
- package/lib/natural/tokenizers/orthography_matchers.js +33 -0
- package/lib/natural/tokenizers/regexp_tokenizer.js +24 -4
- package/lib/natural/tokenizers/sentence_tokenizer.js +1 -1
- package/lib/natural/wordnet/wordnet_file.js +7 -1
- package/lr_classifier.json +1 -1
- package/package.json +35 -23
- package/.npmignore +0 -4
- package/.vscode/temp.sql +0 -0
- package/Makefile +0 -38
- package/data/kata-dasar.txt +0 -29933
- package/package-lock.json +0 -79
- package/tests/stemmer_id_test.js +0 -361
|
@@ -29,17 +29,18 @@ var Removal = require("./removal");
|
|
|
29
29
|
var dictionary = [];
|
|
30
30
|
function loadDictionary(){
|
|
31
31
|
var fs = require('fs');
|
|
32
|
-
var dirname = __dirname + "/../../../../data/kata-dasar.txt";
|
|
33
|
-
var fin = fs.readFileSync(dirname).toString().split("\n");
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
dictionary.push(word);
|
|
38
|
-
|
|
32
|
+
//var dirname = __dirname + "/../../../../data/kata-dasar.txt";
|
|
33
|
+
//var fin = fs.readFileSync(dirname).toString().split("\n");
|
|
34
|
+
var fin = require('./data/kata-dasar.json');
|
|
35
|
+
fin.forEach(function (word) {
|
|
36
|
+
if (word) {
|
|
37
|
+
dictionary.push(word.trim());
|
|
38
|
+
}
|
|
39
|
+
});
|
|
39
40
|
}
|
|
40
41
|
loadDictionary();
|
|
41
42
|
|
|
42
|
-
PrefixRules
|
|
43
|
+
function PrefixRules() {
|
|
43
44
|
var PrefixRules = this;
|
|
44
45
|
|
|
45
46
|
this.removal = undefined;
|
|
@@ -1024,4 +1025,4 @@ rules.push(pr.DisambiguatorPrefixRule40);
|
|
|
1024
1025
|
rules.push(pr.DisambiguatorPrefixRule41);
|
|
1025
1026
|
rules.push(pr.DisambiguatorPrefixRule42);
|
|
1026
1027
|
|
|
1027
|
-
PrefixRules.rules = rules;
|
|
1028
|
+
PrefixRules.rules = rules;
|
|
@@ -129,20 +129,21 @@ function find(word) {
|
|
|
129
129
|
|
|
130
130
|
function loadDictionary(){
|
|
131
131
|
var fs = require('fs');
|
|
132
|
-
var dirname = __dirname + "/../../../../data/kata-dasar.txt";
|
|
133
|
-
var fin = fs.readFileSync(dirname).toString().split("\n");
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
132
|
+
//var dirname = __dirname + "/../../../../data/kata-dasar.txt";
|
|
133
|
+
//var fin = fs.readFileSync(dirname).toString().split("\n");
|
|
134
|
+
var fin = require('./data/kata-dasar.json');
|
|
135
|
+
fin.forEach(function (word) {
|
|
136
|
+
if (word) {
|
|
137
|
+
dictionary.push(word.trim());
|
|
138
|
+
}
|
|
139
|
+
});
|
|
139
140
|
}
|
|
140
141
|
|
|
141
142
|
// Stemming from step 2-5
|
|
142
143
|
function stemmingProcess(){
|
|
143
144
|
if(find(current_word))
|
|
144
145
|
return
|
|
145
|
-
|
|
146
|
+
|
|
146
147
|
// Confix Stripping
|
|
147
148
|
// Try to remove prefixes first before suffixes if the specification is met
|
|
148
149
|
if(precedenceAdjustmentSpecification(original_word)){
|
|
@@ -168,7 +169,7 @@ function stemmingProcess(){
|
|
|
168
169
|
removeSuffixes();
|
|
169
170
|
if(find(current_word))
|
|
170
171
|
return
|
|
171
|
-
|
|
172
|
+
|
|
172
173
|
// Step 4, 5
|
|
173
174
|
removePrefixes();
|
|
174
175
|
if(find(current_word))
|
|
@@ -182,7 +183,7 @@ function stemmingProcess(){
|
|
|
182
183
|
function removeSuffixes(){
|
|
183
184
|
for(var i in suffix_rules){
|
|
184
185
|
resultObj = suffix_rules[i](current_word);
|
|
185
|
-
|
|
186
|
+
|
|
186
187
|
// Add result to variable
|
|
187
188
|
if(resultObj.removal!=undefined){
|
|
188
189
|
removals.push(resultObj.removal);
|
|
@@ -209,7 +210,7 @@ function checkPrefixRules(){
|
|
|
209
210
|
var j = 0;
|
|
210
211
|
for(j=0; j<prefix_rules.length; j++){
|
|
211
212
|
resultObj = prefix_rules[j](current_word);
|
|
212
|
-
|
|
213
|
+
|
|
213
214
|
// Add result to variable
|
|
214
215
|
if(resultObj.removal!=undefined){
|
|
215
216
|
removals.push(resultObj.removal);
|
|
@@ -233,7 +234,7 @@ function loopRestorePrefixes(){
|
|
|
233
234
|
|
|
234
235
|
for(var i in reversed_removals){
|
|
235
236
|
current_removal = reversed_removals[i];
|
|
236
|
-
|
|
237
|
+
|
|
237
238
|
if(!isSuffixRemovals(current_removal)){
|
|
238
239
|
continue
|
|
239
240
|
}
|
|
@@ -245,7 +246,7 @@ function loopRestorePrefixes(){
|
|
|
245
246
|
removePrefixes();
|
|
246
247
|
if(find(current_word))
|
|
247
248
|
return
|
|
248
|
-
current_word = current_removal.getResult() + "kan";
|
|
249
|
+
current_word = current_removal.getResult() + "kan";
|
|
249
250
|
}
|
|
250
251
|
else{
|
|
251
252
|
current_word = current_removal.getOriginalWord();
|
|
@@ -292,7 +293,7 @@ function precedenceAdjustmentSpecification(word){
|
|
|
292
293
|
/^pe(.*)i$/,
|
|
293
294
|
/^ter(.*)i$/,
|
|
294
295
|
];
|
|
295
|
-
|
|
296
|
+
|
|
296
297
|
for(var i in regex_rules){
|
|
297
298
|
if(word.match(regex_rules[i])){
|
|
298
299
|
return true;
|
|
@@ -304,4 +305,4 @@ function precedenceAdjustmentSpecification(word){
|
|
|
304
305
|
//exports for tests
|
|
305
306
|
stemmer.isPlural = isPlural;
|
|
306
307
|
stemmer.dictionary = dictionary;
|
|
307
|
-
stemmer.a = suffix_rules[0];
|
|
308
|
+
stemmer.a = suffix_rules[0];
|
|
@@ -25,7 +25,7 @@ THE SOFTWARE.
|
|
|
25
25
|
|
|
26
26
|
var Removal = require("./removal");
|
|
27
27
|
|
|
28
|
-
SuffixRules
|
|
28
|
+
function SuffixRules() {
|
|
29
29
|
var SuffixRules = this;
|
|
30
30
|
|
|
31
31
|
this.removal = undefined;
|
|
@@ -72,4 +72,4 @@ rules.push(sr.RemoveInflectionalParticle);
|
|
|
72
72
|
rules.push(sr.RemoveInflectionalPossessivePronoun);
|
|
73
73
|
rules.push(sr.RemoveDerivationalSuffix);
|
|
74
74
|
|
|
75
|
-
SuffixRules.rules = rules;
|
|
75
|
+
SuffixRules.rules = rules;
|
|
@@ -1,221 +1,287 @@
|
|
|
1
1
|
/*
|
|
2
|
-
Copyright (c)
|
|
3
|
-
|
|
4
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
5
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
6
|
-
in the Software without restriction, including without limitation the rights
|
|
7
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
9
|
-
furnished to do so, subject to the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be included in
|
|
12
|
-
all copies or substantial portions of the Software.
|
|
13
|
-
|
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
19
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
20
|
-
THE SOFTWARE.
|
|
2
|
+
Copyright (c) 2018, Domingo Martín Mancera
|
|
3
|
+
|
|
4
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
5
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
6
|
+
in the Software without restriction, including without limitation the rights
|
|
7
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
9
|
+
furnished to do so, subject to the following conditions:
|
|
10
|
+
|
|
11
|
+
The above copyright notice and this permission notice shall be included in
|
|
12
|
+
all copies or substantial portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
19
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
20
|
+
THE SOFTWARE.
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
+
'use strict';
|
|
24
|
+
|
|
23
25
|
var Stemmer = require('./stemmer_es');
|
|
24
26
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
27
|
+
// Inherit from the utility class in stemmer_es
|
|
28
|
+
class PorterStemmer extends Stemmer
|
|
29
|
+
{
|
|
30
|
+
constructor() {
|
|
31
|
+
super();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
isVowel(c)
|
|
35
|
+
{
|
|
36
|
+
var regex = /[aeiouáéíóú]/gi;
|
|
37
|
+
|
|
38
|
+
return regex.test(c);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
nextVowelPosition(word, start = 0)
|
|
42
|
+
{
|
|
43
|
+
var length = word.length;
|
|
44
|
+
|
|
45
|
+
for(var position = start; position < length; position++){
|
|
46
|
+
if(this.isVowel(word[position])){
|
|
47
|
+
return position;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return length;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
nextConsonantPosition(word, start = 0)
|
|
55
|
+
{
|
|
56
|
+
var length = word.length;
|
|
57
|
+
|
|
58
|
+
for(var position = start; position < length; position++){
|
|
59
|
+
if(!this.isVowel(word[position])){
|
|
60
|
+
return position;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return length;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
endsIn(word, suffix)
|
|
68
|
+
{
|
|
69
|
+
if(word.length < suffix.length){
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return (word.slice(-suffix.length) === suffix);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
endsInArr(word, suffixes)
|
|
77
|
+
{
|
|
78
|
+
var matches = [];
|
|
79
|
+
for(var i in suffixes) {
|
|
80
|
+
if(this.endsIn(word, suffixes[i])){
|
|
81
|
+
matches.push(suffixes[i]);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
var longest = matches.sort(function (a, b) {
|
|
85
|
+
return b.length - a.length;
|
|
86
|
+
})[0];
|
|
87
|
+
|
|
88
|
+
if(longest){
|
|
89
|
+
return longest
|
|
90
|
+
} else {
|
|
91
|
+
return '';
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
removeAccent(word)
|
|
96
|
+
{
|
|
97
|
+
var accentedVowels = ['á', 'é', 'í', 'ó', 'ú'];
|
|
98
|
+
var vowels = ['a', 'e', 'i', 'o', 'u'];
|
|
99
|
+
|
|
100
|
+
for(var i in accentedVowels){
|
|
101
|
+
word = word.replace(accentedVowels[i], vowels[i]);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return word;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
stem(word)
|
|
108
|
+
{
|
|
109
|
+
var length = word.length;
|
|
110
|
+
|
|
111
|
+
word.toLowerCase();
|
|
112
|
+
|
|
113
|
+
if(length < 2){
|
|
114
|
+
return this.removeAccent(word);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
var r1, r2, rv;
|
|
118
|
+
r1 = length;
|
|
119
|
+
r2 = length;
|
|
120
|
+
rv = length;
|
|
121
|
+
|
|
122
|
+
// R1 is the region after the first non-vowel following a vowel, or is the null region
|
|
123
|
+
// at the end of the word if there is no such non-vowel.
|
|
124
|
+
for(var i = 0; i < (length - 1) && r1 == length; i++){
|
|
125
|
+
if(this.isVowel(word[i]) && !this.isVowel(word[i + 1])){
|
|
126
|
+
r1 = i + 2;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// R2 is the region after the first non-vowel following a vowel in R1,
|
|
131
|
+
// or is the null region at the end of the word if there is no such non-vowel.
|
|
132
|
+
for(var i = r1; i < (length - 1) && r2 == length; i++){
|
|
133
|
+
if(this.isVowel(word[i]) && !this.isVowel(word[i + 1])){
|
|
134
|
+
r2 = i + 2;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if(length > 3){
|
|
139
|
+
if(!this.isVowel(word[1])){
|
|
140
|
+
rv = this.nextVowelPosition(word, 2) + 1;
|
|
141
|
+
} else if(this.isVowel(word[0]) && this.isVowel(word[1])){
|
|
142
|
+
rv = this.nextConsonantPosition(word, 2) + 1;
|
|
143
|
+
} else {
|
|
144
|
+
rv = 3;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
var r1Text = word.slice(r1);
|
|
149
|
+
var r2Text = word.slice(r2);
|
|
150
|
+
var rvText = word.slice(rv);
|
|
151
|
+
var originalWord = word;
|
|
152
|
+
|
|
153
|
+
// Step 0: Attached pronoun
|
|
154
|
+
var pronounSuffix = ['me', 'se', 'sela', 'selo', 'selas', 'selos', 'la', 'le', 'lo', 'las', 'les', 'los', 'nos'];
|
|
155
|
+
var pronounSuffixPre1 = ['iéndo', 'ándo', 'ár', 'ér', 'ír'];
|
|
156
|
+
var pronounSuffixPre2 = ['iendo', 'ando', 'ar', 'er', 'ir'];
|
|
157
|
+
|
|
158
|
+
var suffix = this.endsInArr(word, pronounSuffix);
|
|
159
|
+
|
|
160
|
+
if(suffix != ''){
|
|
161
|
+
var preSuffix = this.endsInArr(rvText.slice(0, -suffix.length), pronounSuffixPre1);
|
|
162
|
+
|
|
163
|
+
if(preSuffix != ''){
|
|
164
|
+
word = this.removeAccent(word.slice(0, -suffix.length));
|
|
165
|
+
} else {
|
|
166
|
+
preSuffix = this.endsInArr(rvText.slice(0, -suffix.length), pronounSuffixPre2);
|
|
167
|
+
|
|
168
|
+
if(preSuffix != '' || (this.endsIn(word.slice(0, -suffix.length), 'uyendo'))){
|
|
169
|
+
|
|
170
|
+
word = word.slice(0, -suffix.length);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if(word != originalWord){
|
|
176
|
+
r1Text = word.slice(r1);
|
|
177
|
+
r2Text = word.slice(r2);
|
|
178
|
+
rvText = word.slice(rv);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
var wordAfter0 = word;
|
|
182
|
+
|
|
183
|
+
if(( suf = this.endsInArr(r2Text, ['anza', 'anzas', 'ico', 'ica', 'icos', 'icas', 'ismo', 'ismos',
|
|
184
|
+
'able', 'ables', 'ible', 'ibles', 'ista', 'istas', 'oso', 'osa',
|
|
185
|
+
'osos', 'osas', 'amiento', 'amientos', 'imiento', 'imientos'])) != '')
|
|
186
|
+
{
|
|
187
|
+
word = word.slice(0, -suf.length);
|
|
91
188
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
rv = getNextVowelPos(token, 2) +1;
|
|
98
|
-
} else if (isVowel(token[0]) && isVowel(token[1])) {
|
|
99
|
-
// or if the first two letters are vowels, RV is the region after the next consonant
|
|
100
|
-
rv = getNextConsonantPos(token, 2) + 1;
|
|
101
|
-
} else {
|
|
102
|
-
//otherwise (consonant-vowel case) RV is the region after the third letter. But RV is the end of the word if these positions cannot be found.
|
|
103
|
-
rv = 3;
|
|
104
|
-
}
|
|
189
|
+
else if((suf = this.endsInArr(r2Text, ['icadora', 'icador', 'icación', 'icadoras', 'icadores', 'icaciones',
|
|
190
|
+
'icante', 'icantes', 'icancia', 'icancias', 'adora', 'ador', 'ación',
|
|
191
|
+
'adoras', 'adores', 'aciones', 'ante', 'antes', 'ancia', 'ancias'])) != '')
|
|
192
|
+
{
|
|
193
|
+
word = word.slice(0, -suf.length);
|
|
105
194
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
var r2_txt = token.substring(r2-1);
|
|
109
|
-
var rv_txt = token.substring(rv-1);
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
var token_orig = token;
|
|
113
|
-
|
|
114
|
-
// Step 0: Attached pronoun
|
|
115
|
-
var pronoun_suf = new Array('me', 'se', 'sela', 'selo', 'selas', 'selos', 'la', 'le', 'lo', 'las', 'les', 'los', 'nos');
|
|
116
|
-
var pronoun_suf_pre1 = new Array('éndo', 'ándo', 'ár', 'ér', 'ír');
|
|
117
|
-
var pronoun_suf_pre2 = new Array('ando', 'iendo', 'ar', 'er', 'ir');
|
|
118
|
-
var suf = endsinArr(token, pronoun_suf);
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if (suf!='') {
|
|
122
|
-
|
|
123
|
-
var pre_suff = endsinArr(rv_txt.slice(0,-suf.length),pronoun_suf_pre1);
|
|
124
|
-
|
|
125
|
-
if (pre_suff != '') {
|
|
126
|
-
|
|
127
|
-
token = removeAccent(token.slice(0,-suf.length));
|
|
128
|
-
} else {
|
|
129
|
-
var pre_suff = endsinArr(rv_txt.slice(0,-suf.length),pronoun_suf_pre2);
|
|
130
|
-
|
|
131
|
-
if (pre_suff != '' ||
|
|
132
|
-
(endsin(token, 'yendo' ) &&
|
|
133
|
-
(token.slice(-suf.length-6,1) == 'u'))) {
|
|
134
|
-
token = token.slice(0,-suf.length);
|
|
135
|
-
}
|
|
195
|
+
else if((suf = this.endsInArr(r2Text, ['logía', 'logías'])) != ''){
|
|
196
|
+
word = word.slice(0, -suf.length) + 'log';
|
|
136
197
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
if (token != token_orig) {
|
|
140
|
-
r1_txt = token.substring(r1-1);
|
|
141
|
-
r2_txt = token.substring(r2-1);
|
|
142
|
-
rv_txt = token.substring(rv-1);
|
|
198
|
+
else if((suf = this.endsInArr(r2Text, ['ución', 'uciones'])) != ''){
|
|
199
|
+
word = word.slice(0, -suf.length) + 'u';
|
|
143
200
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
if ((suf = endsinArr(r2_txt, new Array('anza', 'anzas', 'ico', 'ica', 'icos', 'icas', 'ismo', 'ismos', 'able', 'ables', 'ible', 'ibles', 'ista', 'istas', 'oso', 'osa', 'osos', 'osas', 'amiento', 'amientos', 'imiento', 'imientos'))) != '') {
|
|
147
|
-
token = token.slice(0, -suf.length);
|
|
148
|
-
} else if ((suf = endsinArr(r2_txt, new Array('icadora', 'icador', 'icación', 'icadoras', 'icadores', 'icaciones', 'icante', 'icantes', 'icancia', 'icancias', 'adora', 'ador', 'ación', 'adoras', 'adores', 'aciones', 'ante', 'antes', 'ancia', 'ancias'))) != '') {
|
|
149
|
-
token = token.slice(0, -suf.length);
|
|
150
|
-
} else if ((suf = endsinArr(r2_txt, new Array('logía', 'logías'))) != '') {
|
|
151
|
-
token = token.slice(0, -suf.length)+ 'log';
|
|
152
|
-
} else if ((suf =endsinArr(r2_txt, new Array('ución', 'uciones'))) != '') {
|
|
153
|
-
token = token.slice(0, -suf.length) + 'u';
|
|
154
|
-
} else if ((suf = endsinArr(r2_txt, new Array('encia', 'encias'))) != '') {
|
|
155
|
-
token = token.slice(0, -suf.length)+ 'ente';
|
|
156
|
-
} else if ((suf = endsinArr(r2_txt, new Array('ativamente', 'ivamente', 'osamente', 'icamente', 'adamente'))) != '') {
|
|
157
|
-
token = token.slice(0, -suf.length);
|
|
158
|
-
} else if ((suf = endsinArr(r1_txt, new Array('amente'))) != '') {
|
|
159
|
-
token = token.slice(0, -suf.length);
|
|
160
|
-
} else if ((suf = endsinArr(r2_txt, new Array('antemente', 'ablemente', 'iblemente', 'mente'))) != '') {
|
|
161
|
-
token = token.slice(0, -suf.length);
|
|
162
|
-
} else if ((suf = endsinArr(r2_txt, new Array('abilidad', 'abilidades', 'icidad', 'icidades', 'ividad', 'ividades', 'idad', 'idades'))) != '') {
|
|
163
|
-
token = token.slice(0, -suf.length);
|
|
164
|
-
} else if ((suf = endsinArr(r2_txt, new Array('ativa', 'ativo', 'ativas', 'ativos', 'iva', 'ivo', 'ivas', 'ivos'))) != '') {
|
|
165
|
-
token = token.slice(0, -suf.length);
|
|
201
|
+
else if((suf = this.endsInArr(r2Text, ['encia', 'encias'])) != ''){
|
|
202
|
+
word = word.slice(0, -suf.length) + 'ente';
|
|
166
203
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
204
|
+
else if((suf = this.endsInArr(r2Text, ['ativamente', 'ivamente', 'osamente', 'icamente', 'adamente'])) != ''){
|
|
205
|
+
word = word.slice(0, -suf.length);
|
|
206
|
+
}
|
|
207
|
+
else if((suf = this.endsInArr(r1Text, ['amente'])) != ''){
|
|
208
|
+
word = word.slice(0, -suf.length);
|
|
209
|
+
}
|
|
210
|
+
else if((suf = this.endsInArr(r2Text, ['antemente', 'ablemente', 'iblemente', 'mente'])) != ''){
|
|
211
|
+
word = word.slice(0, -suf.length);
|
|
212
|
+
}
|
|
213
|
+
else if((suf = this.endsInArr(r2Text, ['abilidad', 'abilidades', 'icidad', 'icidades', 'ividad', 'ividades', 'idad', 'idades'])) != ''){
|
|
214
|
+
word = word.slice(0, -suf.length);
|
|
215
|
+
}
|
|
216
|
+
else if((suf = this.endsInArr(r2Text, ['ativa', 'ativo', 'ativas', 'ativos', 'iva', 'ivo', 'ivas', 'ivos'])) != ''){
|
|
217
|
+
word = word.slice(0, -suf.length);
|
|
172
218
|
}
|
|
173
|
-
var token_after1 = token;
|
|
174
219
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
220
|
+
if(word != wordAfter0){
|
|
221
|
+
r1Text = word.slice(r1);
|
|
222
|
+
r2Text = word.slice(r2);
|
|
223
|
+
rvText = word.slice(rv);
|
|
224
|
+
}
|
|
225
|
+
var wordAfter1 = word;
|
|
180
226
|
|
|
181
|
-
|
|
182
|
-
r1_txt = token.substring(r1-1);
|
|
183
|
-
r2_txt = token.substring(r2-1);
|
|
184
|
-
rv_txt = token.substring(rv-1);
|
|
185
|
-
}
|
|
186
|
-
var token_after2a = token;
|
|
227
|
+
if(wordAfter0 === wordAfter1){
|
|
187
228
|
|
|
188
|
-
|
|
189
|
-
|
|
229
|
+
// Do step 2a if no ending was removed by step 1.
|
|
230
|
+
var suf = this.endsInArr(rvText, ['ya', 'ye', 'yan', 'yen', 'yeron', 'yendo', 'yo', 'yó', 'yas', 'yes', 'yais', 'yamos']);
|
|
190
231
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
token = token.slice(0,-1);
|
|
195
|
-
}
|
|
196
|
-
} else if ((suf = endsinArr(rv_txt, new Array('arían', 'arías', 'arán', 'arás', 'aríais', 'aría', 'aréis', 'aríamos', 'aremos', 'ará', 'aré', 'erían', 'erías', 'erán', 'erás', 'eríais', 'ería', 'eréis', 'eríamos', 'eremos', 'erá', 'eré', 'irían', 'irías', 'irán', 'irás', 'iríais', 'iría', 'iréis', 'iríamos', 'iremos', 'irá', 'iré', 'aba', 'ada', 'ida', 'ía', 'ara', 'iera', 'ad', 'ed', 'id', 'ase', 'iese', 'aste', 'iste', 'an', 'aban', 'ían', 'aran', 'ieran', 'asen', 'iesen', 'aron', 'ieron', 'ado', 'ido', 'ando', 'iendo', 'ió', 'ar', 'er', 'ir', 'as', 'abas', 'adas', 'idas', 'ías', 'aras', 'ieras', 'ases', 'ieses', 'ís', 'áis', 'abais', 'íais', 'arais', 'ierais', ' aseis', 'ieseis', 'asteis', 'isteis', 'ados', 'idos', 'amos', 'ábamos', 'íamos', 'imos', 'áramos', 'iéramos', 'iésemos', 'ásemos'))) != '') {
|
|
197
|
-
|
|
198
|
-
token = token.slice(0, -suf.length);
|
|
232
|
+
if(suf != '' && (word.slice(-suf.length - 1, -suf.length) == 'u')){
|
|
233
|
+
word = word.slice(0, -suf.length);
|
|
234
|
+
}
|
|
199
235
|
|
|
236
|
+
if(word != wordAfter1){
|
|
237
|
+
r1Text = word.slice(r1);
|
|
238
|
+
r2Text = word.slice(r2);
|
|
239
|
+
rvText = word.slice(rv);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
var wordAfter2a = word;
|
|
243
|
+
// Do Step 2b if step 2a was done, but failed to remove a suffix.
|
|
244
|
+
if (wordAfter2a == wordAfter1) {
|
|
245
|
+
if((suf = this.endsInArr(rvText, ['arían', 'arías', 'arán', 'arás', 'aríais', 'aría', 'aréis',
|
|
246
|
+
'aríamos', 'aremos', 'ará', 'aré', 'erían', 'erías', 'erán',
|
|
247
|
+
'erás', 'eríais', 'ería', 'eréis', 'eríamos', 'eremos', 'erá',
|
|
248
|
+
'eré', 'irían', 'irías', 'irán', 'irás', 'iríais', 'iría', 'iréis',
|
|
249
|
+
'iríamos', 'iremos', 'irá', 'iré', 'aba', 'ada', 'ida', 'ía', 'ara',
|
|
250
|
+
'iera', 'ad', 'ed', 'id', 'ase', 'iese', 'aste', 'iste', 'an',
|
|
251
|
+
'aban', 'ían', 'aran', 'ieran', 'asen', 'iesen', 'aron', 'ieron',
|
|
252
|
+
'ado', 'ido', 'ando', 'iendo', 'ió', 'ar', 'er', 'ir', 'as', 'abas',
|
|
253
|
+
'adas', 'idas', 'ías', 'aras', 'ieras', 'ases', 'ieses', 'ís', 'áis',
|
|
254
|
+
'abais', 'íais', 'arais', 'ierais', ' aseis', 'ieseis', 'asteis',
|
|
255
|
+
'isteis', 'ados', 'idos', 'amos', 'ábamos', 'íamos', 'imos', 'áramos',
|
|
256
|
+
'iéramos', 'iésemos', 'ásemos'])) != '')
|
|
257
|
+
{
|
|
258
|
+
word = word.slice(0, -suf.length);
|
|
259
|
+
}else if((suf = this.endsInArr(rvText, ['en', 'es', 'éis', 'emos'])) != '') {
|
|
260
|
+
word = word.slice(0, -suf.length);
|
|
261
|
+
if(this.endsIn(word, 'gu')){
|
|
262
|
+
word = word.slice(0, -1);
|
|
263
|
+
}
|
|
200
264
|
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
r1Text = word.slice(r1);
|
|
269
|
+
r2Text = word.slice(r2);
|
|
270
|
+
rvText = word.slice(rv);
|
|
271
|
+
|
|
272
|
+
if ((suf = this.endsInArr(rvText, ['os', 'a', 'o', 'á', 'í', 'ó'])) != '') {
|
|
273
|
+
word = word.slice(0, -suf.length);
|
|
274
|
+
} else if ((this.endsInArr(rvText , ['e','é'])) != '') {
|
|
275
|
+
word = word.slice(0, -1);
|
|
276
|
+
rvText = word.slice(rv);
|
|
277
|
+
if (this.endsIn(rvText, 'u') && this.endsIn(word, 'gu')) {
|
|
278
|
+
word = word.slice(0, -1);
|
|
201
279
|
}
|
|
202
280
|
}
|
|
203
281
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
r2_txt = token.substring(r2-1);
|
|
207
|
-
rv_txt = token.substring(rv-1);
|
|
208
|
-
|
|
209
|
-
if ((suf = endsinArr(rv_txt, new Array('os', 'a', 'o', 'á', 'í', 'ó'))) != '') {
|
|
210
|
-
token = token.slice(0, -suf.length);
|
|
211
|
-
} else if ((suf = endsinArr(rv_txt ,new Array('e','é'))) != '') {
|
|
212
|
-
token = token.slice(0,-1);
|
|
213
|
-
rv_txt = token.substring(rv-1);
|
|
214
|
-
if (endsin(rv_txt,'u') && endsin(token,'gu')) {
|
|
215
|
-
token = token.slice(0,-1);
|
|
216
|
-
}
|
|
217
|
-
}
|
|
282
|
+
return this.removeAccent(word);
|
|
283
|
+
}
|
|
218
284
|
|
|
219
|
-
|
|
285
|
+
}
|
|
220
286
|
|
|
221
|
-
|
|
287
|
+
module.exports = new PorterStemmer();
|