natural 0.5.0 → 0.5.1

package/.travis.yml

@@ -1,5 +1,4 @@
 language: node_js
 node_js:
-  - "0.8"
   - "0.10"
-before_install: 'npm install -g jasmine-node'
+before_install: 'npm install -g jasmine-node'

package/README.md

@@ -289,6 +289,15 @@ var restoredClassifier = natural.BayesClassifier.restore(JSON.parse(raw));
 console.log(restoredClassifier.classify('i should sell that'));
 ```
 
+__Note:__ if using the classifier for languages other than English you may need
+to pass in the stemmer to use. In fact, you can do this for any stemmer including
+alternate English stemmers. The default is the `PorterStemmer`.
+
+```javascript
+const PorterStemmerRu = require('./node_modules/natural/lib/natural/stemmers/porter_stemmer_ru');
+var classifier = new natural.BayesClassifier(PorterStemmerRu);
+```
+
 ## Phonetics
 
 Phonetic matching (sounds-like) matching can be done with the [SoundEx](http://en.wikipedia.org/wiki/Soundex),
@@ -954,7 +963,7 @@ This is best constructed with an array of tokens from a corpus, but a simple lis
 
 ```javascript
 var corpus = ['something', 'soothing'];
-var spellcheck = new Spellcheck(corpus);
+var spellcheck = new natural.Spellcheck(corpus);
 ```
 
 It uses the trie datastructure for fast boolean lookup of a word
@@ -977,19 +986,22 @@ algorithm. Transformation rules are specified in external files.
 
 ### Usage
 ```javascript
-var natural = require("./lib/natural");
+var natural = require("natural");
+var path = require("path");
 
-var base_folder = "some_path/lib/natural/brill_pos_tagger";
-var rulesFilename = base_folder + "/data/tr_from_posjs.txt";
-var lexiconFilename = base_folder + "/data/lexicon_from_posjs.json";
+var base_folder = path.join(path.dirname(require.resolve("natural")), "brill_pos_tagger");
+var rulesFilename = base_folder + "/data/English/tr_from_posjs.txt";
+var lexiconFilename = base_folder + "/data/English/lexicon_from_posjs.json";
 var defaultCategory = 'N';
 
 var lexicon = new natural.Lexicon(lexiconFilename, defaultCategory);
-var rules = new natural.Ruleset(rulesFilename);
+var rules = new natural.RuleSet(rulesFilename);
 var tagger = new natural.BrillPOSTagger(lexicon, rules);
 
 var sentence = ["I", "see", "the", "man", "with", "the", "telescope"];
 console.log(JSON.stringify(tagger.tag(sentence)));
+// [["I","NN"],["see","VB"],["the","DT"],["man","NN"],["with","IN"],["the","DT"],["telescope","NN"]]
+
 ```
 
 ### Lexicon

package/bayes_classifier.json

@@ -0,0 +1 @@
+{"classifier":{"classFeatures":{"computing":{"0":2,"1":2,"2":2,"3":3,"4":2,"5":2},"literature":{"2":2,"6":4,"7":2,"8":2}},"classTotals":{"computing":4,"literature":4},"totalExamples":7,"smoothing":1},"docs":[{"label":"computing","text":["fix","box"]},{"label":"computing","text":["write","code"]},{"label":"computing","text":["nasti","script","code"]},{"label":"literature","text":["write","book"]},{"label":"literature","text":["read","book"]},{"label":"literature","text":["studi","book"]}],"features":{"fix":1,"box":1,"write":2,"code":2,"nasti":1,"script":1,"book":3,"read":1,"studi":1},"stemmer":{},"lastAdded":6,"events":{"domain":null,"_events":{},"_eventsCount":0}}

package/lib/natural/classifiers/bayes_classifier.js

@@ -45,7 +45,7 @@ function restore(classifier, stemmer) {
 function load(filename, stemmer, callback) {
     Classifier.load(filename, function(err, classifier) {
         if (err) {
-            callback(err);
+            return callback(err);
         }
         else {
             callback(err, restore(classifier, stemmer));

package/lib/natural/classifiers/classifier.js

@@ -21,8 +21,10 @@ THE SOFTWARE.
 */
 
 var PorterStemmer = require('../stemmers/porter_stemmer'),
+Threads = require('webworker-threads'),
 util = require('util'),
-events = require('events');
+events = require('events')
+os = require('os');
 
 var Classifier = function(classifier, stemmer) {
     this.classifier = classifier;
@@ -34,15 +36,15 @@ var Classifier = function(classifier, stemmer) {
 };
 
 function addDocument(text, classification) {
-    
-    // Ignore further processing if classification is undefined 
+
+    // Ignore further processing if classification is undefined
     if(typeof classification === 'undefined') return;
-    
-    // If classification is type of string then make sure it's dosen't have blank space at both end  
+
+    // If classification is type of string then make sure it's dosen't have blank space at both end
     if(typeof classification === 'string'){
       classification = classification.trim();
     }
-  
+
     if(typeof text === 'string')
 	text = this.stemmer.tokenizeAndStem(text);
 
@@ -105,6 +107,28 @@ function textToFeatures(observation) {
     return features;
 }
 
+function docsToFeatures(docs) {
+    var parsedDocs = [];
+
+    for (var i = 0; i < docs.length; i++) {
+        var features = [];
+
+        for (var feature in FEATURES) {
+            if (docs[i].observation.indexOf(feature) > -1)
+                features.push(1);
+            else
+                features.push(0);
+        }
+
+        parsedDocs.push({
+            index: docs[i].index,
+            features: features
+        });
+    }
+
+    return JSON.stringify(parsedDocs);
+}
+
 function train() {
     var totalDocs = this.docs.length;
     for(var i = this.lastAdded; i < totalDocs; i++) {
@@ -117,12 +141,246 @@ function train() {
     this.classifier.train();
 }
 
+function trainParallel(numThreads, callback) {
+    if (!callback) {
+        callback = numThreads;
+        numThreads = undefined;
+    }
+
+    if (isNaN(numThreads)) {
+        numThreads = os.cpus().length;
+    }
+
+    var totalDocs = this.docs.length;
+    var threadPool = Threads.createPool(numThreads);
+    var docFeatures = {};
+    var finished = 0;
+    var self = this;
+
+    // Init pool; send the features array and the parsing function
+    threadPool.all.eval('var FEATURES = ' + JSON.stringify(this.features));
+    threadPool.all.eval(docsToFeatures);
+
+    // Convert docs to observation objects
+    var obsDocs = [];
+    for (var i = this.lastAdded; i < totalDocs; i++) {
+        var observation = this.docs[i].text;
+        if (typeof observation === 'string')
+            observation = this.stemmer.tokenizeAndStem(observation);
+        obsDocs.push({
+            index: i,
+            observation: observation
+        });
+    }
+
+    // Called when a batch completes processing
+    var onFeaturesResult = function(docs) {
+        setTimeout(function() {
+            self.events.emit('processedBatch', {
+                size: docs.length,
+                docs: totalDocs,
+                batches: numThreads,
+                index: finished
+            });
+        });
+
+        for (var j = 0; j < docs.length; j++) {
+            docFeatures[docs[j].index] = docs[j].features;
+        }
+    };
+
+    // Called when all batches finish processing
+    var onFinished = function(err) {
+        if (err) {
+            threadPool.destroy();
+            return callback(err);
+        }
+
+        for (var j = self.lastAdded; j < totalDocs; j++) {
+            self.classifier.addExample(docFeatures[j], self.docs[j].label);
+            self.events.emit('trainedWithDocument', {
+                index: j,
+                total: totalDocs,
+                doc: self.docs[j]
+            });
+            self.lastAdded++;
+        }
+
+        self.events.emit('doneTraining', true);
+        self.classifier.train();
+
+        threadPool.destroy();
+        callback(null);
+    };
+
+    // Split the docs and start processing
+    var batchSize = Math.ceil(obsDocs.length / numThreads);
+    var lastError;
+
+    for (var i = 0; i < numThreads; i++) {
+        var batchDocs = obsDocs.slice(i * batchSize, (i+1) * batchSize);
+        var batchJson = JSON.stringify(batchDocs);
+
+        threadPool.any.eval('docsToFeatures(' + batchJson + ')', function(err, docs) {
+            lastError = err || lastError;
+            finished++;
+
+            if (docs) {
+                docs = JSON.parse(docs);
+                onFeaturesResult(docs);
+            }
+
+            if (finished >= numThreads) {
+                onFinished(lastError);
+            }
+        });
+    }
+}
+
+function trainParallelBatches(options) {
+    var numThreads = options && options.numThreads;
+    var batchSize = options && options.batchSize;
+
+    if (isNaN(numThreads)) {
+        numThreads = os.cpus().length;
+    }
+
+    if (isNaN(batchSize)) {
+        batchSize = 2500;
+    }
+
+    var totalDocs = this.docs.length;
+    var threadPool = Threads.createPool(numThreads);
+    var docFeatures = {};
+    var finished = 0;
+    var self = this;
+
+    var abort = false;
+    var onError = function(err) {
+        if (!err || abort) return;
+        abort = true;
+        threadPool.destroy(true);
+        self.events.emit('doneTrainingError', err);
+    };
+
+    // Init pool; send the features array and the parsing function
+    var str = JSON.stringify(this.features);
+    threadPool.all.eval('var FEATURES = ' + str + ';', onError);
+    threadPool.all.eval(docsToFeatures, onError);
+
+    // Convert docs to observation objects
+    var obsDocs = [];
+    for (var i = this.lastAdded; i < totalDocs; i++) {
+        var observation = this.docs[i].text;
+        if (typeof observation === 'string')
+            observation = this.stemmer.tokenizeAndStem(observation);
+        obsDocs.push({
+            index: i,
+            observation: observation
+        });
+    }
+
+    // Split the docs in batches
+    var obsBatches = [];
+    var i = 0;
+    while (true) {
+        var batch = obsDocs.slice(i * batchSize, (i+1) * batchSize);
+        if (!batch || !batch.length) break;
+        obsBatches.push(batch);
+        i++;
+    }
+    obsDocs = null;
+    self.events.emit('startedTraining', {
+        docs: totalDocs,
+        batches: obsBatches.length
+    });
+
+    // Called when a batch completes processing
+    var onFeaturesResult = function(docs) {
+        self.events.emit('processedBatch', {
+            size: docs.length,
+            docs: totalDocs,
+            batches: obsBatches.length,
+            index: finished
+        });
+
+        for (var j = 0; j < docs.length; j++) {
+            docFeatures[docs[j].index] = docs[j].features;
+        }
+    };
+
+    // Called when all batches finish processing
+    var onFinished = function() {
+        threadPool.destroy(true);
+        abort = true;
+
+        for (var j = self.lastAdded; j < totalDocs; j++) {
+            self.classifier.addExample(docFeatures[j], self.docs[j].label);
+            self.events.emit('trainedWithDocument', {
+                index: j,
+                total: totalDocs,
+                doc: self.docs[j]
+            });
+            self.lastAdded++;
+        }
+
+        self.events.emit('doneTraining', true);
+        self.classifier.train();
+    };
+
+    // Called to send the next batch to be processed
+    var batchIndex = 0;
+    var sendNext = function() {
+        if (abort) return;
+        if (batchIndex >= obsBatches.length) {
+            return;
+        }
+
+        sendBatch(JSON.stringify(obsBatches[batchIndex]));
+        batchIndex++;
+    };
+
+    // Called to send a batch of docs to the threads
+    var sendBatch = function(batchJson) {
+        if (abort) return;
+        threadPool.any.eval('docsToFeatures(' + batchJson + ');', function(err, docs) {
+            if (err) {
+                return onError(err);
+            }
+
+            finished++;
+
+            if (docs) {
+                docs = JSON.parse(docs);
+                setTimeout(onFeaturesResult.bind(null, docs));
+            }
+
+            if (finished >= obsBatches.length) {
+                setTimeout(onFinished);
+            }
+
+            setTimeout(sendNext);
+        });
+    };
+
+    // Start processing
+    for (var i = 0; i < numThreads; i++) {
+        sendNext();
+    }
+}
+
 function retrain() {
   this.classifier = new (this.classifier.constructor)();
   this.lastAdded = 0;
   this.train();
 }
 
+function retrainParallel(numThreads, callback) {
+  this.classifier = new (this.classifier.constructor)();
+  this.lastAdded = 0;
+  this.trainParallel(numThreads, callback);
+}
+
 function getClassifications(observation) {
     return this.classifier.getClassifications(this.textToFeatures(observation));
 }
@@ -153,7 +411,7 @@ function load(filename, callback) {
 
     fs.readFile(filename, 'utf8', function(err, data) {
         var classifier;
-          
+
         if(!err) {
             classifier = JSON.parse(data);
         }
@@ -166,7 +424,10 @@ function load(filename, callback) {
 Classifier.prototype.addDocument = addDocument;
 Classifier.prototype.removeDocument = removeDocument;
 Classifier.prototype.train = train;
+Classifier.prototype.trainParallel = trainParallel;
+Classifier.prototype.trainParallelBatches = trainParallelBatches;
 Classifier.prototype.retrain = retrain;
+Classifier.prototype.retrainParallel = retrainParallel;
 Classifier.prototype.classify = classify;
 Classifier.prototype.textToFeatures = textToFeatures;
 Classifier.prototype.save = save;

package/lib/natural/distance/jaro-winkler_distance.js

@@ -27,70 +27,71 @@ THE SOFTWARE.
 // s1 is the first string to compare
 // s2 is the second string to compare
 function distance(s1, s2) {
-    if (typeof(s1) != "string" || typeof(s2) != "string") return 0;
-    if (s1.length == 0 || s2.length == 0) 
+    if (typeof(s1) !== "string" || typeof(s2) !== "string") {
         return 0;
+    }
+
+    if (s1.length === 0 || s2.length === 0) {
+        return 0;
+    }
+
     s1 = s1.toLowerCase(), s2 = s2.toLowerCase();
+
     var matchWindow = (Math.floor(Math.max(s1.length, s2.length) / 2.0)) - 1;
     var matches1 = new Array(s1.length);
     var matches2 = new Array(s2.length);
     var m = 0; // number of matches
     var t = 0; // number of transpositions
+    var i = 0; // index for string 1
+    var k = 0; // index for string 2
 
     //debug helpers
     //console.log("s1: " + s1 + "; s2: " + s2);
     //console.log(" - matchWindow: " + matchWindow);
 
-    // find matches
-    for (var i = 0; i < s1.length; i++) {
-	var matched = false;
-
-	// check for an exact match
-	if (s1[i] ==  s2[i]) {
-		matches1[i] = matches2[i] = matched = true;
-		m++
-	}
-
-	// check the "match window"
-	else {
-        	// this for loop is a little brutal
-        	for (k = (i <= matchWindow) ? 0 : i - matchWindow;
-        		(k <= i + matchWindow) && k < s2.length && !matched;
-			k++) {
-            		if (s1[i] == s2[k]) {
-                		if(!matches1[i] && !matches2[k]) {
-                	    		m++;
-               		}
-
-        	        matches1[i] = matches2[k] = matched = true;
-        	    }
-        	}
-	}
-    }
+    for (i = 0; i < s1.length; i++) { // loop to find matched characters
+        var start = Math.max(0, (i - matchWindow)); // use the higher of the window diff
+        var end = Math.min((i + matchWindow + 1), s2.length); // use the min of the window and string 2 length
 
-    if(m == 0)
-        return 0.0;
+        for (k = start; k < end; k++) { // iterate second string index
+            if (matches2[k]) { // if second string character already matched
+                continue;
+            }
+            if (s1[i] !== s2[k]) { // characters don't match
+                continue;
+            }
 
-    // count transpositions
-    var k = 0;
+            // assume match if the above 2 checks don't continue
+            matches1[i] = true;
+            matches2[k] = true;
+            m++;
+            break;
+        }
+    }
 
-    for(var i = 0; i < s1.length; i++) {
-    	if(matches1[k]) {
-    	    while(!matches2[k] && k < matches2.length)
-                k++;
-	        if(s1[i] != s2[k] &&  k < matches2.length)  {
-                t++;
-            }
+    // nothing matched
+    if (m === 0) {
+        return 0.0;
+    }
 
-    	    k++;
-    	}
+    k = 0; // reset string 2 index
+    for(i = 0; i < s1.length; i++) { // loop to find transpositions
+        if (!matches1[i]) { // non-matching character
+            continue;
+        }
+        while(!matches2[k]) { // move k index to the next match
+            k++;
+        }
+        if (s1[i] !== s2[k]) { // if the characters don't match, increase transposition
+            t++;
+        }
+        k++; // iterate k index normally
     }
-    
-    //debug helpers:
-    //console.log(" - matches: " + m);
-    //console.log(" - transpositions: " + t);
+
+    // transpositions divided by 2
     t = t / 2.0;
-    return (m / s1.length + m / s2.length + (m - t) / m) / 3;
+
+    return ((m / s1.length) + (m / s2.length) + ((m - t) / m)) / 3.0;
 }
 
 // Computes the Winkler distance between two string -- intrepreted from:
@@ -99,18 +100,18 @@ function distance(s1, s2) {
 // s2 is the second string to compare
 // dj is the Jaro Distance (if you've already computed it), leave blank and the method handles it
 function JaroWinklerDistance(s1, s2, dj) {
-		if (s1 == s2) {
-				return 1 
-		}
-		else {
-		    var jaro;
-		    (typeof(dj) == 'undefined')? jaro = distance(s1,s2) : jaro = dj;
-		    var p = 0.1; //
-		    var l = 0 // length of the matching prefix
-		    while(s1[l] == s2[l] && l < 4)
-		        l++;
-		    
-		    return jaro + l * p * (1 - jaro);
-		}
+    if (s1 === s2) {
+        return 1;
+    } else {
+        var jaro = (typeof(dj) === 'undefined') ? distance(s1,s2) : dj;
+        var p = 0.1; // default scaling factor
+        var l = 0 // length of the matching prefix
+        while(s1[l] === s2[l] && l < 4) {
+            l++;
+        }
+
+        return jaro + l * p * (1 - jaro);
+    }
 }
 module.exports = JaroWinklerDistance;
+

package/lib/natural/inflectors/noun_inflector.js

@@ -62,6 +62,7 @@ var NounInflector = function() {
     this.addIrregular("tooth", "teeth");
     this.addIrregular("goose", "geese");
     this.addIrregular("ephemeris", "ephemerides");
+    this.addIrregular("cloth", "clothes");
     
     // see if it is possible to unify the creation of both the singular and
     // plural regexes or maybe even just have one list. with a complete list

package/lib/natural/phonetics/double_metaphone.js

@@ -32,13 +32,13 @@ function isVowel(c) {
 function truncate(string, length) {
     if(string.length >= length)
         string = string.substring(0, length);
-        
+
     return string;
 }
 
 function process(token, maxLength) {
 	token = token.toUpperCase();
-	var primary = '', secondary = '';	
+	var primary = '', secondary = '';
     var pos = 0;
     maxLength == maxLength || 32;
 
@@ -67,9 +67,9 @@ function process(token, maxLength) {
 
     function handleC() {
 
-        if(pos >= 1 && !isVowel(token[pos - 2]) 
+        if(pos >= 1 && !isVowel(token[pos - 2])
                 && token[pos - 1] == 'A' && token[pos + 1] == 'H'
-                    && (token[pos + 2] != 'I' && token[pos + 2] != 'I')
+                    && token[pos + 2] != 'I'
                         || subMatch(-2, 4, ['BACHER', 'MACHER'])) {
             add('K');
             pos++;
@@ -85,8 +85,8 @@ function process(token, maxLength) {
             if(pos > 0 && token.substring(pos + 2, pos + 4) == 'AE') {
                 addSecondary('K', 'X');
                 pos++;
-            } else if(pos == 0 
-                        && (subMatch(1, 6, ['HARAC', 'HARIS']) 
+            } else if(pos == 0
+                        && (subMatch(1, 6, ['HARAC', 'HARIS'])
                             || subMatch(1, 4, ['HOR', 'HUM', 'HIA', 'HEM']))
                         && token.substring(pos + 1, pos + 5) != 'HORE') {
                 add('K');
@@ -95,7 +95,7 @@ function process(token, maxLength) {
                 if((subMatchAbsolute(0, 3, ['VAN', 'VON']) || token.substring(0,  3) == 'SCH')
                     || subMatch(-2, 4, ['ORCHES', 'ARCHIT', 'ORCHID'])
                     || subMatch(2, 3, ['T', 'S'])
-                    || ((subMatch(-1, 0, ['A', 'O', 'U', 'E']) || pos == 0) 
+                    || ((subMatch(-1, 0, ['A', 'O', 'U', 'E']) || pos == 0)
                         && subMatch(2, 3, ['B', 'F', 'H', 'L', 'M', 'N', 'R', 'V', 'W']))) {
                     add('K');
                 } else if(pos > 0) {
@@ -103,15 +103,15 @@ function process(token, maxLength) {
                     if(token.substring(0, 2) == 'MC') {
                         add('K');
                     } else {
-                        addSecondary('X', 'K');   
+                        addSecondary('X', 'K');
                     }
                 } else {
                     add('X');
                 }
 
                 pos++;
-            } 
-        } else if(token.substring(pos, pos + 2) == 'CZ' 
+            }
+        } else if(token.substring(pos, pos + 2) == 'CZ'
                 && token.substring(pos - 2, pos + 1) != 'WICZ') {
             addSecondary('S', 'X');
             pos++;
@@ -119,7 +119,7 @@ function process(token, maxLength) {
             add('X');
             pos += 2;
         } else if(token[pos + 1] == 'C' && pos != 1 && token[0] != 'M') {
-            if(['I', 'E', 'H'].indexOf(token[pos + 2]) > -1 
+            if(['I', 'E', 'H'].indexOf(token[pos + 2]) > -1
                     && token.substring(pos + 2, pos + 4) != 'HU') {
                 if(pos == 1 && token[pos - 1] == 'A'
                         || subMatch(-1, 4, ['UCCEE', 'UCCES'])) {
@@ -138,19 +138,19 @@ function process(token, maxLength) {
             pos++;
         } else if(['E', 'I', 'Y'].indexOf(token[pos + 1]) > -1) {
             if(subMatch(1, 3, ['IA', 'IE', 'IO'])) {
-                addSecondary('S', 'X');   
+                addSecondary('S', 'X');
             } else {
                 add('S');
             }
             pos++;
-        } else {            
+        } else {
             add('K');
             if(token[pos + 1] == ' ' && ['C', 'Q', 'G'].indexOf(token[pos + 2])) {
                 pos += 2;
             } else if(['C', 'K', 'Q'].indexOf(token[pos + 1]) > -1
                     && !subMatch(1, 3, ['CE', 'CI'])) {
                 pos++;
-            } 
+            }
         }
     }
 
@@ -165,7 +165,7 @@ function process(token, maxLength) {
     		}
 	    } else if(token[pos + 1] == 'T') {
     		add('T');
-	    	pos++;    		
+	    	pos++;
     	} else
     		addCompressedDouble('D', 'T');
     }
@@ -182,7 +182,7 @@ function process(token, maxLength) {
                     add('K');
                 }
                 pos++;
-            } else if(pos > 1 
+            } else if(pos > 1
                 && (['B', 'H', 'D'].indexOf(token[pos - 2]) > -1
                     || ['B', 'H', 'D'].indexOf(token[pos - 3]) > -1
                     || ['B', 'H'].indexOf(token[pos - 4]) > -1)) {
@@ -213,7 +213,7 @@ function process(token, maxLength) {
         } else if(token.substring(pos + 1, pos + 3) == 'LI' && !slavoGermanic) {
             addSecondary('KL', 'L');
             pos++;
-        } else if(pos == 0 && (token[pos + 1] == 'Y'                
+        } else if(pos == 0 && (token[pos + 1] == 'Y'
                 || subMatch(1, 3, ['ES', 'EP', 'EB', 'EL', 'EY', 'IB', 'IL', 'IN', 'IE', 'EI', 'ER']))) {
             addSecondary('K', 'J')
         } else {
@@ -226,22 +226,22 @@ function process(token, maxLength) {
 		if((pos == 0 || isVowel(token[pos - 1])) && isVowel(token[pos + 1])) {
 			add('H');
 			pos++;
-		}    	
-    }    
+		}
+    }
 
     function handleJ() {
         var jose = (token.substring(pos + 1, pos + 4) == 'OSE');
 
         if(san || jose) {
-            if((pos == 0 && token[pos + 4] == ' ') 
+            if((pos == 0 && token[pos + 4] == ' ')
                     || san) {
-                add('H');            
+                add('H');
             } else
                 add('J', 'H');
         } else {
             if(pos == 0/* && !jose*/) {
                 addSecondary('J', 'A');
-            } else if(isVowel(token[pos - 1]) && !slavoGermanic 
+            } else if(isVowel(token[pos - 1]) && !slavoGermanic
                     && (token[pos + 1] == 'A' || token[pos + 1] == 'O')) {
                 addSecondary('J', 'H');
             } else if(pos == token.length - 1) {
@@ -262,14 +262,14 @@ function process(token, maxLength) {
     			pos++;
     			return;
     		}
-    		pos++;	
+    		pos++;
     	}
     	add('L');
     }
 
     function handleM() {
     	addCompressedDouble('M');
-    	if(token[pos - 1] == 'U' && token[pos + 1] == 'B' && 
+    	if(token[pos - 1] == 'U' && token[pos + 1] == 'B' &&
     			((pos == token.length - 2  || token.substring(pos + 2, pos + 4) == 'ER')))
     		pos++;
     }
@@ -277,10 +277,10 @@ function process(token, maxLength) {
     function handleP() {
     	if(token[pos + 1] == 'H') {
     		add('F');
-    		pos++;	
+    		pos++;
     	} else {
     		addCompressedDouble('P');
-    		    		
+
 			if(token[pos + 1] == 'B')
     			pos++;
     	}
@@ -292,7 +292,7 @@ function process(token, maxLength) {
     			&& !subMatch(-4, -3, ['ME', 'MA'])) {
     		addSecondary('', 'R');
     	} else
-	    	addCompressedDouble('R');    		
+	    	addCompressedDouble('R');
     }
 
     function handleS() {
@@ -312,7 +312,7 @@ function process(token, maxLength) {
                 addSecondary('S', 'X');
             }
             pos++;
-        } else if((pos == 0 && ['M', 'N', 'L', 'W'].indexOf(token[pos + 1]) > -1) 
+        } else if((pos == 0 && ['M', 'N', 'L', 'W'].indexOf(token[pos + 1]) > -1)
                 || token[pos + 1] == 'Z') {
             addSecondary('S', 'X');
             if(token[pos + 1] == 'Z')
@@ -326,18 +326,18 @@ function process(token, maxLength) {
                 } else if(pos == 0 && !isVowel(token[3]) && token[3] != 'W') {
                     addSecondary('X', 'S');
                 } else {
-                    add('X');   
-                } 
+                    add('X');
+                }
             } else if(['I', 'E', 'Y'].indexOf(token[pos + 2]) > -1) {
                 add('S');
             } else {
                 add('SK');
             }
 
-            pos += 2;            
+            pos += 2;
         } else if(pos == token.length - 1
                 && subMatch(-2, 0, ['AI', 'OI'])) {
-            addSecondary('', 'S');            
+            addSecondary('', 'S');
         } else if(token[pos + 1] != 'L' && (
                 token[pos - 1] != 'A' && token[pos - 1] != 'I')) {
             addCompressedDouble('S');
@@ -353,12 +353,12 @@ function process(token, maxLength) {
         } else if(subMatch(1, 3, ['IA', 'CH'])) {
             add('X');
             pos += 2;
-        } else if(token[pos + 1] == 'H' 
+        } else if(token[pos + 1] == 'H'
                 || token.substring(1, 2) == 'TH') {
-            if(subMatch(2, 4, ['OM', 'AM']) 
+            if(subMatch(2, 4, ['OM', 'AM'])
                     || ['VAN ', 'VON '].indexOf(token.substring(0, 4)) > -1
                     || token.substring(0, 3) == 'SCH') {
-                add('T');            
+                add('T');
             } else
                 addSecondary('0', 'T');
             pos++;
@@ -373,7 +373,7 @@ function process(token, maxLength) {
     function handleX() {
     	if(pos == 0) {
     		add('S');
-    	} else if(!(pos == token.length - 1 
+    	} else if(!(pos == token.length - 1
 	    		&& (['IAU', 'EAU', 'IEU'].indexOf(token.substring(pos - 3, pos)) > -1
 	    			|| ['AU', 'OU'].indexOf(token.substring(pos - 2, pos)) > -1))) {
     		add('KS');
@@ -387,7 +387,7 @@ function process(token, maxLength) {
             } else if (isVowel(token[1])) {
                 addSecondary('A', 'F');
             }
-        } else if((pos == token.length - 1 && isVowel(token[pos - 1]) 
+        } else if((pos == token.length - 1 && isVowel(token[pos - 1])
                     || subMatch(-1, 4, ['EWSKI', 'EWSKY', 'OWSKI', 'OWSKY'])
                     || token.substring(0, 3) == 'SCH')) {
                 addSecondary('', 'F');
@@ -401,11 +401,11 @@ function process(token, maxLength) {
     function handleZ() {
         if(token[pos + 1] == 'H') {
             add('J');
-            pos++;            
-        } else if(subMatch(1, 3, ['ZO', 'ZI', 'ZA']) 
+            pos++;
+        } else if(subMatch(1, 3, ['ZO', 'ZI', 'ZA'])
                 || (slavoGermanic && pos > 0 && token[pos - 1] != 'T')) {
             addSecondary('S', 'TS');
-            pos++; 
+            pos++;
         } else
             addCompressedDouble('Z', 'S');
     }
@@ -421,7 +421,7 @@ function process(token, maxLength) {
     while(pos < token.length) {
 
     	switch(token[pos]) {
-	        case 'A': case 'E': case 'I': case 'O': case 'U': case 'Y': 	        
+	        case 'A': case 'E': case 'I': case 'O': case 'U': case 'Y':
 	        case 'Ê': case 'É': case 'É': case'À':
 		        if(pos == 0)
 		        	add('A');
@@ -502,7 +502,7 @@ function compare(stringA, stringB) {
     var encodingsA = process(stringA),
         encodingsB = process(stringB);
 
-    return encodingsA[0] == encodingsB[0] || 
+    return encodingsA[0] == encodingsB[0] ||
         encodingsA[1] == encodingsB[1];
 };
 

package/lr_classifier.json

@@ -0,0 +1 @@
+{"classifier":{"examples":{"computing":[[1,1,0,0,0,0,0,0,0],[0,0,1,1,0,0,0,0,0],[0,0,0,1,1,1,0,0,0]],"literature":[[0,0,1,0,0,0,1,0,0],[0,0,0,0,0,0,1,1,0],[0,0,0,0,0,0,1,0,1]]},"features":[],"featurePositions":{},"maxFeaturePosition":0,"classifications":["computing","literature"],"exampleCount":6,"theta":[{"elements":[2.0057203693422903,2.0057203693422903,0.0904766662626176,3.447364098636818,0.9696521506748157,0.9696521506748157,-4.840706080886848,-1.2267353995937322,-1.2267353995937322]},{"elements":[-2.0057203693422894,-2.0057203693422894,-0.09047666626261801,-3.4473640986368186,-0.9696521506748156,-0.9696521506748156,4.84070608088685,1.2267353995937313,1.2267353995937313]}]},"docs":[{"label":"computing","text":["fix","box"]},{"label":"computing","text":["write","code"]},{"label":"computing","text":["nasti","script","code"]},{"label":"literature","text":["write","book"]},{"label":"literature","text":["read","book"]},{"label":"literature","text":["studi","book"]}],"features":{"fix":1,"box":1,"write":2,"code":2,"nasti":1,"script":1,"book":3,"read":1,"studi":1},"stemmer":{},"lastAdded":6,"events":{"domain":null,"_events":{},"_eventsCount":0}}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -13,11 +13,13 @@
   "dependencies": {
     "apparatus": ">= 0.0.9",
     "sylvester": ">= 0.0.12",
-    "underscore": ">=1.3.1"
+    "underscore": ">=1.3.1",
+    "webworker-threads": ">=0.6.2"
   },
   "devDependencies": {
-    "jasmine-node": "~1.13.1",
-    "uubench": "0.0.x"
+    "uubench": "0.0.x",
+    "sinon": "^1.12.2",
+    "jasmine-node": "~1.13.1"
   },
   "scripts": {
     "test": "NODE_PATH=. node_modules/jasmine-node/bin/jasmine-node spec/"