natural 0.2.0 → 0.2.1

package/.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

package/README.md

@@ -184,7 +184,7 @@ tokenize and stem the text.
 
 ```javascript
 classifier.addDocument('i am long qqqq', 'buy');
-classifier.addDocument('buy the q''s', 'buy');
+classifier.addDocument('buy the q\'s', 'buy');
 classifier.addDocument('short gold', 'sell');
 classifier.addDocument('sell gold', 'sell');
 
@@ -208,8 +208,8 @@ You have access to the set of matched classes and the associated value from the
 Outputs:
 
 ```javascript
-[ { label: 'sell', value: 0.39999999999999997 },
-  { label: 'buy', value: 0.19999999999999998 } ]
+[ { label: 'buy', value: 0.39999999999999997 },
+  { label: 'sell', value: 0.19999999999999998 } ]
 ```
 
 From this:

package/lib/natural/inflectors/present_verb_inflector.js

@@ -51,6 +51,7 @@ var VerbInflector = function() {
     this.addIrregular("am", "are");    
     this.addIrregular("is", "are");
     this.addIrregular("was", "were");
+    this.addIrregular("has", "have");
     
     this.singularForms.regularForms.push([/ed$/i, 'ed']);
     this.singularForms.regularForms.push([/ss$/i, 'sses']);

package/lib/natural/phonetics/double_metaphone.js

@@ -66,7 +66,8 @@ function process(token, maxLength) {
     }
 
     function handleC() {
-        if(pos > 1 && !isVowel(token[pos - 2]) 
+
+        if(pos >= 1 && !isVowel(token[pos - 2]) 
                 && token[pos - 1] == 'A' && token[pos + 1] == 'H'
                     && (token[pos + 2] != 'I' && token[pos + 2] != 'I')
                         || subMatch(-2, 4, ['BACHER', 'MACHER'])) {
@@ -74,7 +75,9 @@ function process(token, maxLength) {
             pos++;
         } else if(pos == 0 && token.substring(1, 6) == 'EASAR') {
             add('S');
-            pos++;
+            add('S');
+            add('R');
+            pos += 6;
         } else if(token.substring(pos + 1, pos + 4) == 'HIA') {
             add('K');
             pos++;
@@ -84,7 +87,7 @@ function process(token, maxLength) {
                 pos++;
             } else if(pos == 0 
                         && (subMatch(1, 6, ['HARAC', 'HARIS']) 
-                            || subMatch(1, 3, ['HOR', 'HUM', 'HIA', 'HEM']))
+                            || subMatch(1, 4, ['HOR', 'HUM', 'HIA', 'HEM']))
                         && token.substring(pos + 1, pos + 5) != 'HORE') {
                 add('K');
                 pos++;
@@ -96,6 +99,7 @@ function process(token, maxLength) {
                         && subMatch(2, 3, ['B', 'F', 'H', 'L', 'M', 'N', 'R', 'V', 'W']))) {
                     add('K');
                 } else if(pos > 0) {
+
                     if(token.substring(0, 2) == 'MC') {
                         add('K');
                     } else {
@@ -415,6 +419,7 @@ function process(token, maxLength) {
     }
 
     while(pos < token.length) {
+
     	switch(token[pos]) {
 	        case 'A': case 'E': case 'I': case 'O': case 'U': case 'Y': 	        
 	        case 'Ê': case 'É': case 'É': case'À':
@@ -488,7 +493,7 @@ function process(token, maxLength) {
         }
 
     	pos++;
-    }    
+    }
 
     return [truncate(primary, maxLength), truncate(secondary, maxLength)];
 }

package/lib/natural/phonetics/metaphone.js

@@ -38,7 +38,11 @@ function dropBafterMAtEnd(token) {
 }
 
 function cTransform(token) {
+    
+
     token = token.replace(/([^s]|^)(c)(h)/g, '$1x$3').trim();
+
+
     token = token.replace(/cia/g, 'xia');
     token = token.replace(/c(i|e|y)/g, 's$1');
     token = token.replace(/c/g, 'k'); 

package/lib/natural/phonetics/soundex.js

@@ -47,7 +47,7 @@ function transformR(token) {
 }
 
 function condense(token) {
-    return token.replace(/(\d)[hw]?\1+/g, '$1').replace(/[hw]/g, '');
+    return token.replace(/(\d)?\1+/g, '$1');
 }
 
 function padRight0(token) {
@@ -57,16 +57,20 @@ function padRight0(token) {
         return token;
 }
 
+function transform(token) {
+    return transformLipps(transformThroats(
+        transformToungue(transformL(transformHum(transformR(token))))));
+}
+
 var SoundEx = new Phonetic();
 module.exports = SoundEx;
 
 SoundEx.process = function(token, maxLength) {
-    token = token.toLowerCase();
-    
-    return token.charAt(0).toUpperCase() + padRight0(condense(transformLipps(transformThroats(
-        transformToungue(transformL(transformHum(transformR(
-            token.substr(1, token.length - 1).replace(/[aeiouy]/g, '')))))))
-                )).substr(0, (maxLength && maxLength - 1) || 3);
+    token = token.toLowerCase();    
+    var transformed = condense(transform(token.substr(1, token.length - 1))); // anything that isn't a digit goes
+    // deal with duplicate INITIAL consonant SOUNDS
+    transformed = transformed.replace(new RegExp("^" + transform(token.charAt(0))), '');
+    return token.charAt(0).toUpperCase() + padRight0(transformed.replace(/\D/g, '')).substr(0, (maxLength && maxLength - 1) || 3);
 };
 
 // export for tests;

package/lib/natural/stemmers/porter_stemmer.js

@@ -25,19 +25,19 @@ var Stemmer = require('./stemmer');
 // denote groups of consecutive consonants with a C and consecutive vowels
 // with a V.
 function categorizeGroups(token) {
-    return token.replace(/[^aeiou]+/g, 'C').replace(/[aeiouy]+/g, 'V');
+    return token.replace(/[^aeiouy]+y/g, 'CV').replace(/[aeiou]+/g, 'V').replace(/[^V]+/g, 'C');
 }
 
 // denote single consonants with a C and single vowels with a V
 function categorizeChars(token) {
-    return token.replace(/[^aeiou]/g, 'C').replace(/[aeiouy]/g, 'V');
+    return token.replace(/[^aeiouy]y/g, 'CV').replace(/[aeiou]/g, 'V').replace(/[^V]/g, 'C');
 }
 
 // calculate the "measure" M of a word. M is the count of VC sequences dropping
 // an initial C if it exists and a trailing V if it exists.
 function measure(token) {
     if(!token)
-	return -1;
+    	return -1;
 
     return categorizeGroups(token).replace(/^C/, '').replace(/V$/, '').length / 2;
 }
@@ -67,14 +67,12 @@ function attemptReplace(token, pattern, replacement, callback) {
 // attempt to replace a list of patterns/replacements on a token for a minimum
 // measure M.
 function attemptReplacePatterns(token, replacements, measureThreshold) {
-    var replacement = null;
+    var replacement = token;
 
-    for(var i = 0; i < replacements.length; i++) {
-	if(measureThreshold == null || measure(attemptReplace(token, replacements[i][0], '')) > measureThreshold)
-	    replacement = attemptReplace(token, replacements[i][0], replacements[i][1]);
-
-	if(replacement)
-	    break;
+    for(var i = 0; i < replacements.length; i++) {   
+    	if(measureThreshold == null || measure(attemptReplace(token, replacements[i][0], replacements[i][1])) > measureThreshold) {
+    	    replacement = attemptReplace(replacement, replacements[i][0], replacements[i][2]) || replacement;
+        }
     }
     
     return replacement;
@@ -83,50 +81,73 @@ function attemptReplacePatterns(token, replacements, measureThreshold) {
 // replace a list of patterns/replacements on a word. if no match is made return
 // the original token.
 function replacePatterns(token, replacements, measureThreshold) {
-    var result = attemptReplacePatterns(token, replacements, measureThreshold);
-    token = result == null ? token : result;
-    
-    return token;
+    return attemptReplacePatterns(token, replacements, measureThreshold) || token;
+}
+
+// TODO: this should replace all of the messy replacement stuff above
+function replaceRegex(token, regex, includeParts, minimumMeasure) {
+    var parts;
+    var result = '';
+
+    if(regex.test(token)) {
+        parts = regex.exec(token);
+
+        includeParts.forEach(function(i) {
+            result += parts[i];
+        });
+    }
+
+    if(measure(result) > minimumMeasure) {
+        return result;
+    }
+
+    return null;
 }
 
 // step 1a as defined for the porter stemmer algorithm. 
 function step1a(token) {    
-    if(token.match(/(ss|i)es$/))
+    if(token.match(/(ss|i)es$/)) {
         return token.replace(/(ss|i)es$/, '$1');
- 
-    if(token.substr(-1) == 's' && token.substr(-2, 1) != 's' && token.length > 3)
+    }
+
+    if(token.substr(-1) == 's' && token.substr(-2, 1) != 's' && token.length > 2) {
         return token.replace(/s?$/, '');
-    
+    }
+
     return token;
 }
 
 // step 1b as defined for the porter stemmer algorithm. 
 function step1b(token) {   
     if(token.substr(-3) == 'eed') {
-    if(measure(token.substr(0, token.length - 3)) > 0)
+        if(measure(token.substr(0, token.length - 3)) > 0)
             return token.replace(/eed$/, 'ee');
     } else {
-    var result = attemptReplace(token, /ed|ing$/, '', function(token) {     
-        if(categorizeGroups(token).indexOf('V') >= 0) {
-        var result = attemptReplacePatterns(token, [['at', 'ate'],  ['bl', 'ble'], ['iz', 'ize']]);
-		if(result)
-		    return result;
-		else {
-		    if(endsWithDoublCons(token) && token.match(/[^lsz]$/))
-			return token.replace(/([^aeiou])\1$/, '$1');
-
-		    if(measure(token) == 1 && categorizeChars(token).substr(-3) == 'CVC' && token.match(/[^wxy]$/))
-			return token + 'e';                            
-		}
-
-		return token;
-	    }
-	    
-	    return null;
-	});
-	
-	if(result)
-	    return result;
+        var result = attemptReplace(token, /(ed|ing)$/, '', function(token) {
+            if(categorizeGroups(token).indexOf('V') >= 0) {
+                result = attemptReplacePatterns(token, [['at', '', 'ate'],  ['bl', '', 'ble'], ['iz', '', 'ize']]);
+
+                if(result != token) {
+        		    return result;
+        		} else {
+        		  if(endsWithDoublCons(token) && token.match(/[^lsz]$/)) {
+        			 return token.replace(/([^aeiou])\1$/, '$1');
+                    }
+
+        		  if(measure(token) == 1 && categorizeChars(token).substr(-3) == 'CVC' && token.match(/[^wxy]$/)) {
+        			 return token + 'e';
+                    }
+        		}                
+
+        		return token;
+    	    }
+    	    
+    	    return null;
+    	});
+    	
+    	if(result) {
+    	    return result;
+        }
     }
 
     return token;   
@@ -134,44 +155,48 @@ function step1b(token) {
 
 // step 1c as defined for the porter stemmer algorithm. 
 function step1c(token) {
-    if(categorizeGroups(token).substr(-2, 1) == 'V') {
-        if(token.substr(-1) == 'y')
-            return token.replace(/y$/, 'i');
+    var categorizedGroups = categorizeGroups(token);
+
+    if(token.substr(-1) == 'y' && categorizedGroups.substr(0, categorizedGroups.length - 1).indexOf('V') > -1) {
+        return token.replace(/y$/, 'i');
     }
-   
+
     return token;
 }
 
 // step 2 as defined for the porter stemmer algorithm. 
 function step2(token) {
-    return replacePatterns(token, [['ational', 'ate'], ['tional', 'tion'], ['enci', 'ence'], ['anci', 'ance'],
-        ['izer', 'ize'], ['abli', 'able'], ['alli', 'al'], ['entli', 'ent'], ['eli', 'e'],
-        ['ousli', 'ous'], ['ization', 'ize'], ['ation', 'ate'], ['ator', 'ate'],['alism', 'al'],
-        ['iveness', 'ive'], ['fulness', 'ful'], ['ousness', 'ous'], ['aliti', 'al'],
-        ['iviti', 'ive'], ['biliti', 'ble']], 0);
+    token = replacePatterns(token, [['ational', '', 'ate'], ['tional', '', 'tion'], ['enci', '', 'ence'], ['anci', '', 'ance'],
+        ['izer', '', 'ize'], ['abli', '', 'able'], ['bli', '', 'ble'], ['alli', '', 'al'], ['entli', '', 'ent'], ['eli', '', 'e'],
+        ['ousli', '', 'ous'], ['ization', '', 'ize'], ['ation', '', 'ate'], ['ator', '', 'ate'],['alism', '', 'al'],
+        ['iveness', '', 'ive'], ['fulness', '', 'ful'], ['ousness', '', 'ous'], ['aliti', '', 'al'],
+        ['iviti', '', 'ive'], ['biliti', '', 'ble'], ['logi', '', 'log']], 0);
+
+    return token;
 }
 
 // step 3 as defined for the porter stemmer algorithm. 
 function step3(token) {
-    return replacePatterns(token, [['icate', 'ic'], ['ative', ''], ['alize', 'al'],
-				   ['iciti', 'ic'], ['ical', 'ic'], ['ful', ''], ['ness', '']], 0); 
+    return replacePatterns(token, [['icate', '', 'ic'], ['ative', '', ''], ['alize', '', 'al'],
+				   ['iciti', '', 'ic'], ['ical', '', 'ic'], ['ful', '', ''], ['ness', '', '']], 0);
 }
 
 // step 4 as defined for the porter stemmer algorithm. 
 function step4(token) {
-    return replacePatterns(token, [['al', ''], ['ance', ''], ['ence', ''], ['er', ''], 
-        ['ic', ''], ['able', ''], ['ible', ''], ['ant', ''],
-        ['ement', ''], ['ment', ''], ['ent', ''], [/([st])ion/, '$1'], ['ou', ''], ['ism', ''],
-        ['ate', ''], ['iti', ''], ['ous', ''], ['ive', ''], 
-        ['ize', '']], 1);
+    return replaceRegex(token, /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/, [1], 1) || 
+        replaceRegex(token, /^(.+?)(s|t)(ion)$/, [1, 2], 1) ||
+        token; 
 }
 
 // step 5a as defined for the porter stemmer algorithm. 
 function step5a(token) {
-    var m = measure(token);
-    
-    if(token.length > 3 && ((m > 1 && token.substr(-1) == 'e') || (m == 1 && !(categorizeChars(token).substr(-4, 3) == 'CVC' && token.match(/[^wxy].$/)))))
-        return token.replace(/e$/, '');
+    var m = measure(token.replace(/e$/, ''));
+
+
+
+    if(m > 1 || (m == 1 && !(categorizeChars(token).substr(-4, 3) == 'CVC' && token.match(/[^wxy].$/)))) {
+        token = token.replace(/e$/, '');
+    }
 
     return token;
 }
@@ -179,8 +204,7 @@ function step5a(token) {
 // step 5b as defined for the porter stemmer algorithm. 
 function step5b(token) {
     if(measure(token) > 1) {
-        if(endsWithDoublCons(token) && token.substr(-2) == 'll')
-           return token.replace(/ll$/, 'l'); 
+       return token.replace(/ll$/, 'l'); 
     }
     
     return token;
@@ -189,12 +213,16 @@ function step5b(token) {
 var PorterStemmer = new Stemmer();
 module.exports = PorterStemmer;
 
+
 // perform full stemming algorithm on a single word
 PorterStemmer.stem = function(token) {
+    if(token.length < 3) return token;
     return step5b(step5a(step4(step3(step2(step1c(step1b(step1a(token.toLowerCase())))))))).toString();
 };
 
 //exports for tests
+PorterStemmer.categorizeGroups = categorizeGroups;
+PorterStemmer.measure = measure;
 PorterStemmer.step1a = step1a;
 PorterStemmer.step1b = step1b;
 PorterStemmer.step1c = step1c;