natural 0.1.28 → 0.4.0

package/lib/natural/stemmers/porter_stemmer.js

@@ -25,19 +25,19 @@ var Stemmer = require('./stemmer');
 // denote groups of consecutive consonants with a C and consecutive vowels
 // with a V.
 function categorizeGroups(token) {
-    return token.replace(/[^aeiou]+/g, 'C').replace(/[aeiouy]+/g, 'V');
+    return token.replace(/[^aeiouy]+y/g, 'CV').replace(/[aeiou]+/g, 'V').replace(/[^V]+/g, 'C');
 }
 
 // denote single consonants with a C and single vowels with a V
 function categorizeChars(token) {
-    return token.replace(/[^aeiou]/g, 'C').replace(/[aeiouy]/g, 'V');
+    return token.replace(/[^aeiouy]y/g, 'CV').replace(/[aeiou]/g, 'V').replace(/[^V]/g, 'C');
 }
 
 // calculate the "measure" M of a word. M is the count of VC sequences dropping
 // an initial C if it exists and a trailing V if it exists.
 function measure(token) {
     if(!token)
-	return -1;
+    	return -1;
 
     return categorizeGroups(token).replace(/^C/, '').replace(/V$/, '').length / 2;
 }
@@ -67,14 +67,12 @@ function attemptReplace(token, pattern, replacement, callback) {
 // attempt to replace a list of patterns/replacements on a token for a minimum
 // measure M.
 function attemptReplacePatterns(token, replacements, measureThreshold) {
-    var replacement = null;
+    var replacement = token;
 
-    for(var i = 0; i < replacements.length; i++) {
-	if(measureThreshold == null || measure(attemptReplace(token, replacements[i][0], '')) > measureThreshold)
-	    replacement = attemptReplace(token, replacements[i][0], replacements[i][1]);
-
-	if(replacement)
-	    break;
+    for(var i = 0; i < replacements.length; i++) {   
+    	if(measureThreshold == null || measure(attemptReplace(token, replacements[i][0], replacements[i][1])) > measureThreshold) {
+    	    replacement = attemptReplace(replacement, replacements[i][0], replacements[i][2]) || replacement;
+        }
     }
     
     return replacement;
@@ -83,50 +81,73 @@ function attemptReplacePatterns(token, replacements, measureThreshold) {
 // replace a list of patterns/replacements on a word. if no match is made return
 // the original token.
 function replacePatterns(token, replacements, measureThreshold) {
-    var result = attemptReplacePatterns(token, replacements, measureThreshold);
-    token = result == null ? token : result;
-    
-    return token;
+    return attemptReplacePatterns(token, replacements, measureThreshold) || token;
+}
+
+// TODO: this should replace all of the messy replacement stuff above
+function replaceRegex(token, regex, includeParts, minimumMeasure) {
+    var parts;
+    var result = '';
+
+    if(regex.test(token)) {
+        parts = regex.exec(token);
+
+        includeParts.forEach(function(i) {
+            result += parts[i];
+        });
+    }
+
+    if(measure(result) > minimumMeasure) {
+        return result;
+    }
+
+    return null;
 }
 
 // step 1a as defined for the porter stemmer algorithm. 
 function step1a(token) {    
-    if(token.match(/(ss|i)es$/))
+    if(token.match(/(ss|i)es$/)) {
         return token.replace(/(ss|i)es$/, '$1');
- 
-    if(token.substr(-1) == 's' && token.substr(-2, 1) != 's' && token.length > 3)
+    }
+
+    if(token.substr(-1) == 's' && token.substr(-2, 1) != 's' && token.length > 2) {
         return token.replace(/s?$/, '');
-    
+    }
+
     return token;
 }
 
 // step 1b as defined for the porter stemmer algorithm. 
 function step1b(token) {   
     if(token.substr(-3) == 'eed') {
-    if(measure(token.substr(0, token.length - 3)) > 0)
+        if(measure(token.substr(0, token.length - 3)) > 0)
             return token.replace(/eed$/, 'ee');
     } else {
-    var result = attemptReplace(token, /ed|ing$/, '', function(token) {     
-        if(categorizeGroups(token).indexOf('V') >= 0) {
-        var result = attemptReplacePatterns(token, [['at', 'ate'],  ['bl', 'ble'], ['iz', 'ize']]);
-		if(result)
-		    return result;
-		else {
-		    if(endsWithDoublCons(token) && token.match(/[^lsz]$/))
-			return token.replace(/([^aeiou])\1$/, '$1');
-
-		    if(measure(token) == 1 && categorizeChars(token).substr(-3) == 'CVC' && token.match(/[^wxy]$/))
-			return token + 'e';                            
-		}
-
-		return token;
-	    }
-	    
-	    return null;
-	});
-	
-	if(result)
-	    return result;
+        var result = attemptReplace(token, /(ed|ing)$/, '', function(token) {
+            if(categorizeGroups(token).indexOf('V') >= 0) {
+                result = attemptReplacePatterns(token, [['at', '', 'ate'],  ['bl', '', 'ble'], ['iz', '', 'ize']]);
+
+                if(result != token) {
+        		    return result;
+        		} else {
+        		  if(endsWithDoublCons(token) && token.match(/[^lsz]$/)) {
+        			 return token.replace(/([^aeiou])\1$/, '$1');
+                    }
+
+        		  if(measure(token) == 1 && categorizeChars(token).substr(-3) == 'CVC' && token.match(/[^wxy]$/)) {
+        			 return token + 'e';
+                    }
+        		}                
+
+        		return token;
+    	    }
+    	    
+    	    return null;
+    	});
+    	
+    	if(result) {
+    	    return result;
+        }
     }
 
     return token;   
@@ -134,44 +155,48 @@ function step1b(token) {
 
 // step 1c as defined for the porter stemmer algorithm. 
 function step1c(token) {
-    if(categorizeGroups(token).substr(-2, 1) == 'V') {
-        if(token.substr(-1) == 'y')
-            return token.replace(/y$/, 'i');
+    var categorizedGroups = categorizeGroups(token);
+
+    if(token.substr(-1) == 'y' && categorizedGroups.substr(0, categorizedGroups.length - 1).indexOf('V') > -1) {
+        return token.replace(/y$/, 'i');
     }
-   
+
     return token;
 }
 
 // step 2 as defined for the porter stemmer algorithm. 
 function step2(token) {
-    return replacePatterns(token, [['ational', 'ate'], ['tional', 'tion'], ['enci', 'ence'], ['anci', 'ance'],
-        ['izer', 'ize'], ['abli', 'able'], ['alli', 'al'], ['entli', 'ent'], ['eli', 'e'],
-        ['ousli', 'ous'], ['ization', 'ize'], ['ation', 'ate'], ['ator', 'ate'],['alism', 'al'],
-        ['iveness', 'ive'], ['fulness', 'ful'], ['ousness', 'ous'], ['aliti', 'al'],
-        ['iviti', 'ive'], ['biliti', 'ble']], 0);
+    token = replacePatterns(token, [['ational', '', 'ate'], ['tional', '', 'tion'], ['enci', '', 'ence'], ['anci', '', 'ance'],
+        ['izer', '', 'ize'], ['abli', '', 'able'], ['bli', '', 'ble'], ['alli', '', 'al'], ['entli', '', 'ent'], ['eli', '', 'e'],
+        ['ousli', '', 'ous'], ['ization', '', 'ize'], ['ation', '', 'ate'], ['ator', '', 'ate'],['alism', '', 'al'],
+        ['iveness', '', 'ive'], ['fulness', '', 'ful'], ['ousness', '', 'ous'], ['aliti', '', 'al'],
+        ['iviti', '', 'ive'], ['biliti', '', 'ble'], ['logi', '', 'log']], 0);
+
+    return token;
 }
 
 // step 3 as defined for the porter stemmer algorithm. 
 function step3(token) {
-    return replacePatterns(token, [['icate', 'ic'], ['ative', ''], ['alize', 'al'],
-				   ['iciti', 'ic'], ['ical', 'ic'], ['ful', ''], ['ness', '']], 0); 
+    return replacePatterns(token, [['icate', '', 'ic'], ['ative', '', ''], ['alize', '', 'al'],
+				   ['iciti', '', 'ic'], ['ical', '', 'ic'], ['ful', '', ''], ['ness', '', '']], 0);
 }
 
 // step 4 as defined for the porter stemmer algorithm. 
 function step4(token) {
-    return replacePatterns(token, [['al', ''], ['ance', ''], ['ence', ''], ['er', ''], 
-        ['ic', ''], ['able', ''], ['ible', ''], ['ant', ''],
-        ['ement', ''], ['ment', ''], ['ent', ''], [/([st])ion/, '$1'], ['ou', ''], ['ism', ''],
-        ['ate', ''], ['iti', ''], ['ous', ''], ['ive', ''], 
-        ['ize', '']], 1);
+    return replaceRegex(token, /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/, [1], 1) || 
+        replaceRegex(token, /^(.+?)(s|t)(ion)$/, [1, 2], 1) ||
+        token; 
 }
 
 // step 5a as defined for the porter stemmer algorithm. 
 function step5a(token) {
-    var m = measure(token);
-    
-    if(token.length > 3 && ((m > 1 && token.substr(-1) == 'e') || (m == 1 && !(categorizeChars(token).substr(-4, 3) == 'CVC' && token.match(/[^wxy].$/)))))
-        return token.replace(/e$/, '');
+    var m = measure(token.replace(/e$/, ''));
+
+
+
+    if(m > 1 || (m == 1 && !(categorizeChars(token).substr(-4, 3) == 'CVC' && token.match(/[^wxy].$/)))) {
+        token = token.replace(/e$/, '');
+    }
 
     return token;
 }
@@ -179,8 +204,7 @@ function step5a(token) {
 // step 5b as defined for the porter stemmer algorithm. 
 function step5b(token) {
     if(measure(token) > 1) {
-        if(endsWithDoublCons(token) && token.substr(-2) == 'll')
-           return token.replace(/ll$/, 'l'); 
+       return token.replace(/ll$/, 'l'); 
     }
     
     return token;
@@ -189,12 +213,16 @@ function step5b(token) {
 var PorterStemmer = new Stemmer();
 module.exports = PorterStemmer;
 
+
 // perform full stemming algorithm on a single word
 PorterStemmer.stem = function(token) {
+    if(token.length < 3) return token;
     return step5b(step5a(step4(step3(step2(step1c(step1b(step1a(token.toLowerCase())))))))).toString();
 };
 
 //exports for tests
+PorterStemmer.categorizeGroups = categorizeGroups;
+PorterStemmer.measure = measure;
 PorterStemmer.step1a = step1a;
 PorterStemmer.step1b = step1b;
 PorterStemmer.step1c = step1c;

package/lib/natural/stemmers/porter_stemmer_fr.js

@@ -1,3 +1,5 @@
+'use strict';
+
 /*
 Copyright (c) 2014, Ismaël Héry
 
@@ -50,7 +52,8 @@ function stem(token) {
     return token;
 
   var regs = regions(token);
-  //console.log('regions:', regs)
+
+  var r1_txt, r2_txt, rv_txt;
   r1_txt = token.substring(regs.r1);
   r2_txt = token.substring(regs.r2);
   rv_txt = token.substring(regs.rv);
@@ -269,9 +272,11 @@ function stem(token) {
  * @return {Object}       Regions r1, r2, rv as offsets from the begining of the word
  */
 function regions(token) {
-  var r1 = r2 = rv = len = token.length;
+  var r1, r2, rv, len;
   var i;
 
+  r1 = r2 = rv = len = token.length;
+
   // R1 is the region after the first non-vowel following a vowel,
   for (var i = 0; i < len - 1 && r1 == len; i++) {
     if (isVowel(token[i]) && !isVowel(token[i + 1])) {
@@ -369,4 +374,4 @@ function isVowel(letter) {
 function endsin(token, suffix) {
   if (token.length < suffix.length) return false;
   return (token.slice(-suffix.length) == suffix);
-};
+};

package/lib/natural/stemmers/porter_stemmer_pt.js

@@ -0,0 +1,302 @@
+/*
+Copyright (c) 2015, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+module.exports = (function () {
+  'use strict';
+
+  var Stemmer     = require('./stemmer_pt'),
+    Token         = require('./token'),
+    PorterStemmer = new Stemmer();
+
+  /**
+   * Marks a region after the first non-vowel following a vowel, or the
+   * null region at the end of the word if there is no such non-vowel.
+   *
+   * @param {Object} token Token to stem.
+   * @param {Number} start Start index (defaults to 0).
+   * @param {Number}       Region start index.
+   */
+   var markRegionN = function (start) {
+    var index = start || 0,
+      length = this.string.length,
+      region = length;
+
+    while (index < length - 1 && region === length) {
+      if (this.hasVowelAtIndex(index) && !this.hasVowelAtIndex(index + 1)) {
+        region = index + 2;
+      }
+      index++;
+    }
+
+    return region;
+  };
+
+  /**
+   * Mark RV.
+   *
+   * @param  {Object} token Token to stem.
+   * @return {Number}       Region start index.
+   */
+  var markRegionV = function () {
+    var rv = this.string.length;
+
+    if (rv > 3) {
+      if (!this.hasVowelAtIndex(1)) {
+        rv = this.nextVowelIndex(2) + 1;
+
+      } else if (this.hasVowelAtIndex(0) && this.hasVowelAtIndex(1)) {
+        rv = this.nextConsonantIndex(2) + 1;
+
+      } else {
+        rv = 3;
+      }
+    }
+
+    return rv;
+  };
+
+  /**
+   * Prelude.
+   *
+   * Nasalised vowel forms should be treated as a vowel followed by a consonant.
+   *
+   * @param  {String} token Word to stem.
+   * @return {String}       Stemmed token.
+   */
+  function prelude (token) {
+    return token
+    .replaceAll('ã', 'a~')
+    .replaceAll('õ', 'o~');
+  }
+
+  /**
+   * Step 1: Standard suffix removal.
+   *
+   * This step should always be performed.
+   *
+   * @param  {Token} token Word to stem.
+   * @return {Token}       Stemmed token.
+   */
+  function standardSuffix (token) {
+
+    token.replaceSuffixInRegion([
+      'amentos', 'imentos', 'aço~es', 'adoras', 'adores', 'amento', 'imento',
+
+      'aça~o', 'adora', 'ância', 'antes', 'ismos', 'istas',
+
+      'ador', 'ante', 'ável', 'ezas', 'icas', 'icos', 'ismo', 'ista', 'ível',
+      'osas', 'osos',
+
+      'eza', 'ica', 'ico', 'osa', 'oso'
+
+      ], '', 'r2');
+
+    token.replaceSuffixInRegion(['logias', 'logia'], 'log', 'r2');
+
+    // token.replaceSuffixInRegion(['uço~es', 'uça~o'], 'u', 'r1');
+
+    token.replaceSuffixInRegion(['ências', 'ência'], 'ente', 'r2');
+
+    token.replaceSuffixInRegion([
+      'ativamente', 'icamente', 'ivamente', 'osamente', 'adamente'
+    ], '', 'r2');
+
+    token.replaceSuffixInRegion('amente', '', 'r1');
+
+    token.replaceSuffixInRegion([
+      'antemente', 'avelmente', 'ivelmente', 'mente'
+    ], '', 'r2');
+
+    token.replaceSuffixInRegion([
+      'abilidades', 'abilidade',
+      'icidades', 'icidade',
+      'ividades', 'ividade',
+      'idades', 'idade'
+    ], '', 'r2');
+
+    token.replaceSuffixInRegion([
+      'ativas', 'ativos', 'ativa', 'ativo',
+      'ivas', 'ivos', 'iva', 'ivo'
+    ], '', 'r2');
+
+    if (token.hasSuffix('eiras') || token.hasSuffix('eira')) {
+      token.replaceSuffixInRegion(['iras', 'ira'], 'ir', 'rv');
+    }
+
+    return token;
+  }
+
+  /**
+   * Step 2: Verb suffix removal.
+   *
+   * Perform this step if no ending was removed in step 1.
+   *
+   * @param  {Token} token   Token to stem.
+   * @return {Token}         Stemmed token.
+   */
+  function verbSuffix (token) {
+
+    token.replaceSuffixInRegion([
+      'aríamos', 'ássemos', 'eríamos', 'êssemos', 'iríamos', 'íssemos',
+
+      'áramos', 'aremos', 'aríeis', 'ásseis', 'ávamos', 'éramos', 'eremos',
+      'eríeis', 'ésseis', 'íramos', 'iremos', 'iríeis', 'ísseis',
+
+      'ara~o', 'ardes', 'areis', 'áreis', 'ariam', 'arias', 'armos', 'assem',
+      'asses', 'astes', 'áveis', 'era~o', 'erdes', 'ereis', 'éreis', 'eriam',
+      'erias', 'ermos', 'essem', 'esses', 'estes', 'íamos', 'ira~o', 'irdes',
+      'ireis', 'íreis', 'iriam', 'irias', 'irmos', 'issem', 'isses', 'istes',
+
+      'adas', 'ados', 'amos', 'ámos', 'ando', 'aram', 'aras', 'arás', 'arei',
+      'arem', 'ares', 'aria', 'asse', 'aste', 'avam', 'avas', 'emos', 'endo',
+      'eram', 'eras', 'erás', 'erei', 'erem', 'eres', 'eria', 'esse', 'este',
+      'idas', 'idos', 'íeis', 'imos', 'indo', 'iram', 'iras', 'irás', 'irei',
+      'irem', 'ires', 'iria', 'isse', 'iste',
+
+      'ada', 'ado', 'ais', 'ara', 'ará', 'ava', 'eis', 'era', 'erá', 'iam',
+      'ias', 'ida', 'ido', 'ira', 'irá',
+
+      'am', 'ar', 'as', 'ei', 'em', 'er', 'es', 'eu', 'ia', 'ir', 'is', 'iu', 'ou'
+
+    ], '', 'rv');
+
+    return token;
+  }
+
+  /**
+   * Step 3: Delete suffix i.
+   *
+   * Perform this step if the word was changed, in RV and preceded by c.
+   *
+   * @param  {Token} token   Token to stem.
+   * @return {Token}         Stemmed token.
+   */
+  function iPrecededByCSuffix (token) {
+
+    if (token.hasSuffix('ci')) {
+      token.replaceSuffixInRegion('i', '', 'rv');
+    }
+
+    return token;
+  }
+
+  /**
+   * Step 4: Residual suffix.
+   *
+   * Perform this step if steps 1 and 2 did not alter the word.
+   *
+   * @param  {Token} token Token to stem.
+   * @return {Token}       Stemmed token.
+   */
+  function residualSuffix (token) {
+
+    token.replaceSuffixInRegion(['os', 'a', 'i', 'o', 'á', 'í', 'ó'], '', 'rv');
+
+    return token;
+  }
+
+  /**
+   * Step 5: Residual form.
+   *
+   * This step should always be performed.
+   *
+   * @param  {Token} token Token to stem.
+   * @return {Token}       Stemmed token.
+   */
+  function residualForm (token) {
+
+    var tokenString = token.string;
+
+    if (token.hasSuffix('gue') || token.hasSuffix('gué') || token.hasSuffix('guê')) {
+      token.replaceSuffixInRegion(['ue', 'ué', 'uê'], '', 'rv');
+    }
+
+    if (token.hasSuffix('cie') || token.hasSuffix('cié') || token.hasSuffix('ciê')) {
+      token.replaceSuffixInRegion(['ie', 'ié', 'iê'], '', 'rv');
+    }
+
+    if (tokenString === token.string) {
+      token.replaceSuffixInRegion(['e', 'é', 'ê'], '', 'rv');
+    }
+
+    token.replaceSuffixInRegion('ç', 'c', 'all');
+
+    return token;
+  }
+
+  /**
+   * Postlude.
+   *
+   * Turns a~, o~ back into ã, õ.
+   *
+   * @param  {String} token Word to stem.
+   * @return {String}       Stemmed token.
+   */
+  function postlude (token) {
+    return token
+      .replaceAll('a~', 'ã')
+      .replaceAll('o~', 'õ');
+  }
+
+  /**
+   * Stems a word using a Porter stemmer algorithm.
+   *
+   * @param  {String} word Word to stem.
+   * @return {String}      Stemmed token.
+   */
+  PorterStemmer.stem = function (word) {
+    var token = new Token(word.toLowerCase()),
+      original;
+
+    token = prelude(token);
+
+    token.usingVowels('aeiouáéíóúâêôàãõ')
+      .markRegion('all', 0)
+      .markRegion('r1', null, markRegionN)
+      .markRegion('r2', token.regions.r1, markRegionN)
+      .markRegion('rv', null, markRegionV);
+
+    original = token.string;
+
+    // Always do step 1.
+    token = standardSuffix(token);
+
+    // Do step 2 if no ending was removed by step 1.
+    if (token.string === original) {
+      token = verbSuffix(token);
+    }
+
+    // If the last step to be obeyed — either step 1 or 2 — altered the word,
+    // do step 3. Alternatively, if neither steps 1 nor 2 altered the word, do
+    // step 4.
+    token = token.string !== original ? iPrecededByCSuffix(token) : residualSuffix(token);
+
+    // Always do step 5.
+    token = residualForm(token);
+
+    token = postlude(token);
+
+    return token.string;
+  };
+
+  return PorterStemmer;
+})();

package/lib/natural/stemmers/stemmer.js

@@ -38,14 +38,39 @@ module.exports = function() {
         stopwords.words = stopwords.words.concat(moreStopWords);
     };
 
+    stemmer.removeStopWord = function(stopWord) {
+        this.removeStopWords([stopWord])
+    };
+
+    stemmer.removeStopWords = function(moreStopWords) {
+        moreStopWords.forEach(function(stopWord){
+            var idx = stopwords.words.indexOf(stopWord);
+            if (idx >= 0) {
+                stopwords.words.splice(idx, 1);
+            }
+        });
+
+    };
+
+
     stemmer.tokenizeAndStem = function(text, keepStops) {
         var stemmedTokens = [];
-        
-        new Tokenizer().tokenize(text).forEach(function(token) {
-            if(keepStops || stopwords.words.indexOf(token) == -1)
+        var lowercaseText = text.toLowerCase();
+        var tokens = new Tokenizer().tokenize(lowercaseText);
+
+        if (keepStops) {
+            tokens.forEach(function(token) {
                 stemmedTokens.push(stemmer.stem(token));
-        });
-        
+            });
+        }
+
+        else {
+            tokens.forEach(function(token) {
+                if (stopwords.words.indexOf(token) == -1)
+                    stemmedTokens.push(stemmer.stem(token));
+            });
+        }
+
         return stemmedTokens;
     };
 
@@ -53,7 +78,7 @@ module.exports = function() {
         String.prototype.stem = function() {
             return stemmer.stem(this);
         };
-        
+
         String.prototype.tokenizeAndStem = function(keepStops) {
             return stemmer.tokenizeAndStem(this, keepStops);
         };