natural 0.1.27 → 0.2.1

package/lib/natural/stemmers/porter_stemmer.js

@@ -25,19 +25,19 @@ var Stemmer = require('./stemmer');
 // denote groups of consecutive consonants with a C and consecutive vowels
 // with a V.
 function categorizeGroups(token) {
-    return token.replace(/[^aeiou]+/g, 'C').replace(/[aeiouy]+/g, 'V');
+    return token.replace(/[^aeiouy]+y/g, 'CV').replace(/[aeiou]+/g, 'V').replace(/[^V]+/g, 'C');
 }
 
 // denote single consonants with a C and single vowels with a V
 function categorizeChars(token) {
-    return token.replace(/[^aeiou]/g, 'C').replace(/[aeiouy]/g, 'V');
+    return token.replace(/[^aeiouy]y/g, 'CV').replace(/[aeiou]/g, 'V').replace(/[^V]/g, 'C');
 }
 
 // calculate the "measure" M of a word. M is the count of VC sequences dropping
 // an initial C if it exists and a trailing V if it exists.
 function measure(token) {
     if(!token)
-	return -1;
+    	return -1;
 
     return categorizeGroups(token).replace(/^C/, '').replace(/V$/, '').length / 2;
 }
@@ -67,14 +67,12 @@ function attemptReplace(token, pattern, replacement, callback) {
 // attempt to replace a list of patterns/replacements on a token for a minimum
 // measure M.
 function attemptReplacePatterns(token, replacements, measureThreshold) {
-    var replacement = null;
+    var replacement = token;
 
-    for(var i = 0; i < replacements.length; i++) {
-	if(measureThreshold == null || measure(attemptReplace(token, replacements[i][0], '')) > measureThreshold)
-	    replacement = attemptReplace(token, replacements[i][0], replacements[i][1]);
-
-	if(replacement)
-	    break;
+    for(var i = 0; i < replacements.length; i++) {   
+    	if(measureThreshold == null || measure(attemptReplace(token, replacements[i][0], replacements[i][1])) > measureThreshold) {
+    	    replacement = attemptReplace(replacement, replacements[i][0], replacements[i][2]) || replacement;
+        }
     }
     
     return replacement;
@@ -83,51 +81,73 @@ function attemptReplacePatterns(token, replacements, measureThreshold) {
 // replace a list of patterns/replacements on a word. if no match is made return
 // the original token.
 function replacePatterns(token, replacements, measureThreshold) {
-    var result = attemptReplacePatterns(token, replacements, measureThreshold);
-    token = result == null ? token : result;
-    
-    return token;
+    return attemptReplacePatterns(token, replacements, measureThreshold) || token;
+}
+
+// TODO: this should replace all of the messy replacement stuff above
+function replaceRegex(token, regex, includeParts, minimumMeasure) {
+    var parts;
+    var result = '';
+
+    if(regex.test(token)) {
+        parts = regex.exec(token);
+
+        includeParts.forEach(function(i) {
+            result += parts[i];
+        });
+    }
+
+    if(measure(result) > minimumMeasure) {
+        return result;
+    }
+
+    return null;
 }
 
 // step 1a as defined for the porter stemmer algorithm. 
 function step1a(token) {    
-    if(token.match(/(ss|i)es$/))
+    if(token.match(/(ss|i)es$/)) {
         return token.replace(/(ss|i)es$/, '$1');
- 
-    if(token.substr(-1) == 's' && token.substr(-2, 1) != 's' && token.length > 3)
+    }
+
+    if(token.substr(-1) == 's' && token.substr(-2, 1) != 's' && token.length > 2) {
         return token.replace(/s?$/, '');
-    
+    }
+
     return token;
 }
 
 // step 1b as defined for the porter stemmer algorithm. 
 function step1b(token) {   
     if(token.substr(-3) == 'eed') {
-	if(measure(token.substr(0, token.length - 3)) > 0)
+        if(measure(token.substr(0, token.length - 3)) > 0)
             return token.replace(/eed$/, 'ee');
     } else {
-	var result = attemptReplace(token, /ed|ing$/, '', function(token) {	    
-	    if(categorizeGroups(token).indexOf('V') > 0) {
-		var result = attemptReplacePatterns(token, [['at', 'ate'],  ['bl', 'ble'], ['iz', 'ize']]);
-
-		if(result)
-		    return result;
-		else {
-		    if(endsWithDoublCons(token) && token.match(/[^lsz]$/))
-			return token.replace(/([^aeiou])\1$/, '$1');
-
-		    if(measure(token) == 1 && categorizeChars(token).substr(-3) == 'CVC' && token.match(/[^wxy]$/))
-			return token + 'e';                            
-		}
-
-		return token;
-	    }
-	    
-	    return null;
-	});
-	
-	if(result)
-	    return result;
+        var result = attemptReplace(token, /(ed|ing)$/, '', function(token) {
+            if(categorizeGroups(token).indexOf('V') >= 0) {
+                result = attemptReplacePatterns(token, [['at', '', 'ate'],  ['bl', '', 'ble'], ['iz', '', 'ize']]);
+
+                if(result != token) {
+        		    return result;
+        		} else {
+        		  if(endsWithDoublCons(token) && token.match(/[^lsz]$/)) {
+        			 return token.replace(/([^aeiou])\1$/, '$1');
+                    }
+
+        		  if(measure(token) == 1 && categorizeChars(token).substr(-3) == 'CVC' && token.match(/[^wxy]$/)) {
+        			 return token + 'e';
+                    }
+        		}                
+
+        		return token;
+    	    }
+    	    
+    	    return null;
+    	});
+    	
+    	if(result) {
+    	    return result;
+        }
     }
 
     return token;   
@@ -135,44 +155,48 @@ function step1b(token) {
 
 // step 1c as defined for the porter stemmer algorithm. 
 function step1c(token) {
-    if(categorizeGroups(token).substr(-2, 1) == 'V') {
-        if(token.substr(-1) == 'y')
-            return token.replace(/y$/, 'i');
+    var categorizedGroups = categorizeGroups(token);
+
+    if(token.substr(-1) == 'y' && categorizedGroups.substr(0, categorizedGroups.length - 1).indexOf('V') > -1) {
+        return token.replace(/y$/, 'i');
     }
-   
+
     return token;
 }
 
 // step 2 as defined for the porter stemmer algorithm. 
 function step2(token) {
-    return replacePatterns(token, [['ational', 'ate'], ['tional', 'tion'], ['enci', 'ence'], ['anci', 'ance'],
-        ['izer', 'ize'], ['abli', 'able'], ['alli', 'al'], ['entli', 'ent'], ['eli', 'e'],
-        ['ousli', 'ous'], ['ization', 'ize'], ['ation', 'ate'], ['ator', 'ate'],['alism', 'al'],
-        ['iveness', 'ive'], ['fulness', 'ful'], ['ousness', 'ous'], ['aliti', 'al'],
-        ['iviti', 'ive'], ['biliti', 'ble']], 0);
+    token = replacePatterns(token, [['ational', '', 'ate'], ['tional', '', 'tion'], ['enci', '', 'ence'], ['anci', '', 'ance'],
+        ['izer', '', 'ize'], ['abli', '', 'able'], ['bli', '', 'ble'], ['alli', '', 'al'], ['entli', '', 'ent'], ['eli', '', 'e'],
+        ['ousli', '', 'ous'], ['ization', '', 'ize'], ['ation', '', 'ate'], ['ator', '', 'ate'],['alism', '', 'al'],
+        ['iveness', '', 'ive'], ['fulness', '', 'ful'], ['ousness', '', 'ous'], ['aliti', '', 'al'],
+        ['iviti', '', 'ive'], ['biliti', '', 'ble'], ['logi', '', 'log']], 0);
+
+    return token;
 }
 
 // step 3 as defined for the porter stemmer algorithm. 
 function step3(token) {
-    return replacePatterns(token, [['icate', 'ic'], ['ative', ''], ['alize', 'al'],
-				   ['iciti', 'ic'], ['ical', 'ic'], ['ful', ''], ['ness', '']], 0); 
+    return replacePatterns(token, [['icate', '', 'ic'], ['ative', '', ''], ['alize', '', 'al'],
+				   ['iciti', '', 'ic'], ['ical', '', 'ic'], ['ful', '', ''], ['ness', '', '']], 0);
 }
 
 // step 4 as defined for the porter stemmer algorithm. 
 function step4(token) {
-    return replacePatterns(token, [['al', ''], ['ance', ''], ['ence', ''], ['er', ''], 
-        ['ic', ''], ['able', ''], ['ible', ''], ['ant', ''],
-        ['ement', ''], ['ment', ''], ['ent', ''], [/([st])ion/, '$1'], ['ou', ''], ['ism', ''],
-        ['ate', ''], ['iti', ''], ['ous', ''], ['ive', ''], 
-        ['ize', '']], 1);
+    return replaceRegex(token, /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/, [1], 1) || 
+        replaceRegex(token, /^(.+?)(s|t)(ion)$/, [1, 2], 1) ||
+        token; 
 }
 
 // step 5a as defined for the porter stemmer algorithm. 
 function step5a(token) {
-    var m = measure(token);
-    
-    if(token.length > 3 && ((m > 1 && token.substr(-1) == 'e') || (m == 1 && !(categorizeChars(token).substr(-4, 3) == 'CVC' && token.match(/[^wxy].$/)))))
-        return token.replace(/e$/, '');
+    var m = measure(token.replace(/e$/, ''));
+
+
+
+    if(m > 1 || (m == 1 && !(categorizeChars(token).substr(-4, 3) == 'CVC' && token.match(/[^wxy].$/)))) {
+        token = token.replace(/e$/, '');
+    }
 
     return token;
 }
@@ -180,8 +204,7 @@ function step5a(token) {
 // step 5b as defined for the porter stemmer algorithm. 
 function step5b(token) {
     if(measure(token) > 1) {
-        if(endsWithDoublCons(token) && token.substr(-2) == 'll')
-           return token.replace(/ll$/, 'l'); 
+       return token.replace(/ll$/, 'l'); 
     }
     
     return token;
@@ -190,12 +213,16 @@ function step5b(token) {
 var PorterStemmer = new Stemmer();
 module.exports = PorterStemmer;
 
+
 // perform full stemming algorithm on a single word
 PorterStemmer.stem = function(token) {
+    if(token.length < 3) return token;
     return step5b(step5a(step4(step3(step2(step1c(step1b(step1a(token.toLowerCase())))))))).toString();
 };
 
 //exports for tests
+PorterStemmer.categorizeGroups = categorizeGroups;
+PorterStemmer.measure = measure;
 PorterStemmer.step1a = step1a;
 PorterStemmer.step1b = step1b;
 PorterStemmer.step1c = step1c;

package/lib/natural/stemmers/porter_stemmer_es.js

@@ -67,7 +67,6 @@ function removeAccent(token) {
 	return str;
 };
 
-
 // perform full stemming algorithm on a single word
 PorterStemmer.stem = function(token) {
 	token = token.toLowerCase();
@@ -96,7 +95,7 @@ PorterStemmer.stem = function(token) {
 			if(isVowel(token[1])) {
 				// If the second letter is a consonant, RV is the region after the next following vowel
 				rv = getNextVowelPos(token, 2) +1;
-			} else if (isVowel(token[0]) && isVowel(token[1])) { 
+			} else if (isVowel(token[0]) && isVowel(token[1])) {
 				// or if the first two letters are vowels, RV is the region after the next consonant
 				rv = getNextConsonantPos(token, 2) + 1;
 			} else {
@@ -113,41 +112,41 @@ PorterStemmer.stem = function(token) {
 	var token_orig = token;
 
 	// Step 0: Attached pronoun
-	var pronoun_suf = new Array('me', 'se', 'sela', 'selo', 'selas', 'selos', 'la', 'le', 'lo', 'las', 'les', 'los', 'nos');	
-	var pronoun_suf_pre1 = new Array('éndo', 'ándo', 'ár', 'ér', 'ír');	
+	var pronoun_suf = new Array('me', 'se', 'sela', 'selo', 'selas', 'selos', 'la', 'le', 'lo', 'las', 'les', 'los', 'nos');
+	var pronoun_suf_pre1 = new Array('éndo', 'ándo', 'ár', 'ér', 'ír');
 	var pronoun_suf_pre2 = new Array('ando', 'iendo', 'ar', 'er', 'ir');
 	var suf = endsinArr(token, pronoun_suf);
 
-    
+
 	if (suf!='') {
 
 		var pre_suff = endsinArr(rv_txt.slice(0,-suf.length),pronoun_suf_pre1);
-		
+
 		if (pre_suff != '') {
-				
+
 				token = removeAccent(token.slice(0,-suf.length));
 		} else {
 			var pre_suff = endsinArr(rv_txt.slice(0,-suf.length),pronoun_suf_pre2);
-			
+
 			if (pre_suff != '' ||
-				(endsin(token, 'yendo' ) && 
+				(endsin(token, 'yendo' ) &&
 				(token.slice(-suf.length-6,1) == 'u'))) {
 				token = token.slice(0,-suf.length);
 			}
 		}
 	}
-		
+
 		if (token != token_orig) {
 			r1_txt = token.substring(r1-1);
 			r2_txt = token.substring(r2-1);
 			rv_txt = token.substring(rv-1);
 		}
 		var token_after0 = token;
-		
+
 		if ((suf = endsinArr(r2_txt, new Array('anza', 'anzas', 'ico', 'ica', 'icos', 'icas', 'ismo', 'ismos', 'able', 'ables', 'ible', 'ibles', 'ista', 'istas', 'oso', 'osa', 'osos', 'osas', 'amiento', 'amientos', 'imiento', 'imientos'))) != '') {
-			token = token.slice(0, -suf.length);	
+			token = token.slice(0, -suf.length);
 		} else if ((suf = endsinArr(r2_txt, new  Array('icadora', 'icador', 'icación', 'icadoras', 'icadores', 'icaciones', 'icante', 'icantes', 'icancia', 'icancias', 'adora', 'ador', 'ación', 'adoras', 'adores', 'aciones', 'ante', 'antes', 'ancia', 'ancias'))) != '') {
-			token = token.slice(0,  -suf.length);	
+			token = token.slice(0,  -suf.length);
 		} else if ((suf = endsinArr(r2_txt, new  Array('logía', 'logías'))) != '') {
 			token = token.slice(0,  -suf.length)+ 'log';
 		} else if ((suf =endsinArr(r2_txt, new  Array('ución', 'uciones'))) != '') {
@@ -172,37 +171,37 @@ PorterStemmer.stem = function(token) {
 			rv_txt = token.substring(rv-1);
 		}
 		var token_after1 = token;
-		
+
 		if (token_after0 == token_after1) {
-			// Do step 2a if no ending was removed by step 1. 
+			// Do step 2a if no ending was removed by step 1.
 			if ((suf = endsinArr(rv_txt, new Array('ya', 'ye', 'yan', 'yen', 'yeron', 'yendo', 'yo', 'yó', 'yas', 'yes', 'yais', 'yamos'))) != '' && (token.substring(suf.length-1,1) == 'u')) {
 				token = token.slice(0, -suf.length);
 			}
-			
+
 			if (token != token_after1) {
 				r1_txt = token.substring(r1-1);
 				r2_txt = token.substring(r2-1);
 				rv_txt = token.substring(rv-1);
 			}
 			var token_after2a = token;
-			
-			// Do Step 2b if step 2a was done, but failed to remove a suffix. 
+
+			// Do Step 2b if step 2a was done, but failed to remove a suffix.
 			if (token_after2a == token_after1) {
-				
+
 				if ((suf = endsinArr(rv_txt,new Array('en', 'es', 'éis', 'emos'))) != '') {
 					token = token.slice(0,-suf.length);
 					if (endsin(token, 'gu')) {
 						token = token.slice(0,-1);
 					}
 				} else if ((suf = endsinArr(rv_txt, new Array('arían', 'arías', 'arán', 'arás', 'aríais', 'aría', 'aréis', 'aríamos', 'aremos', 'ará', 'aré', 'erían', 'erías', 'erán', 'erás', 'eríais', 'ería', 'eréis', 'eríamos', 'eremos', 'erá', 'eré', 'irían', 'irías', 'irán', 'irás', 'iríais', 'iría', 'iréis', 'iríamos', 'iremos', 'irá', 'iré', 'aba', 'ada', 'ida', 'ía', 'ara', 'iera', 'ad', 'ed', 'id', 'ase', 'iese', 'aste', 'iste', 'an', 'aban', 'ían', 'aran', 'ieran', 'asen', 'iesen', 'aron', 'ieron', 'ado', 'ido', 'ando', 'iendo', 'ió', 'ar', 'er', 'ir', 'as', 'abas', 'adas', 'idas', 'ías', 'aras', 'ieras', 'ases', 'ieses', 'ís', 'áis', 'abais', 'íais', 'arais', 'ierais', '  aseis', 'ieseis', 'asteis', 'isteis', 'ados', 'idos', 'amos', 'ábamos', 'íamos', 'imos', 'áramos', 'iéramos', 'iésemos', 'ásemos'))) != '') {
-					
+
 					token = token.slice(0, -suf.length);
-					
+
 				}
 			}
 		}
 
-		// Always do step 3. 
+		// Always do step 3.
 		r1_txt = token.substring(r1-1);
 		r2_txt = token.substring(r2-1);
 		rv_txt = token.substring(rv-1);
@@ -216,7 +215,7 @@ PorterStemmer.stem = function(token) {
 				token = token.slice(0,-1);
 			}
 		}
-		
+
 		return removeAccent(token);
 
 };