natural 0.1.27 → 0.2.1

package/.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

package/README.md

@@ -18,15 +18,6 @@ will be in order. Thanks to Polyakov Vladimir, Russian stemming has been added!,
 
 Aside from this README, the only documentation is [this DZone article](http://www.dzone.com/links/r/using_natural_a_nlp_module_for_nodejs.html) and [here on my blog](http://www.chrisumbel.com/article/node_js_natural_language_porter_stemmer_lancaster_bayes_naive_metaphone_soundex), which is a bit older.
 
-Looking for Help Maintaining Natural!
--------------------------------------
-
-I'm having trouble devoting the time necessary to maintain natural. While I'm certainly not
-leaving the project I'd like someone to take over the day-to-day maintenance of dealing with
-issues, pull requests and driving the direction of the software moving forward. Please contact
-chris@chrisumbel.com if you're interested!
-
-
 Installation
 ------------
 
@@ -47,24 +38,24 @@ arrays of tokens:
 ```javascript
 var natural = require('natural'),
   tokenizer = new natural.WordTokenizer();
-console.log(tokenizer.tokenize("your dog has flees."));
-// [ 'your', 'dog', 'has', 'flees' ]
+console.log(tokenizer.tokenize("your dog has fleas."));
+// [ 'your', 'dog', 'has', 'fleas' ]
 ```
 
 The other tokenizers follow a similar pattern:
 
 ```javascript
 tokenizer = new natural.TreebankWordTokenizer();
-console.log(tokenizer.tokenize("my dog hasn't any flees."));
-// [ 'my', 'dog', 'has', 'n\'t', 'any', 'flees', '.' ]
+console.log(tokenizer.tokenize("my dog hasn't any fleas."));
+// [ 'my', 'dog', 'has', 'n\'t', 'any', 'fleas', '.' ]
 
 tokenizer = new natural.RegexpTokenizer({pattern: /\-/});
-console.log(tokenizer.tokenize("flee-dog"));
-// [ 'flee', 'dog' ]
+console.log(tokenizer.tokenize("flea-dog"));
+// [ 'flea', 'dog' ]
 
 tokenizer = new natural.WordPunctTokenizer();
-console.log(tokenizer.tokenize("my dog hasn't any flees."));
-// [ 'my',  'dog',  'hasn',  '\'',  't',  'any',  'flees',  '.' ]
+console.log(tokenizer.tokenize("my dog hasn't any fleas."));
+// [ 'my',  'dog',  'hasn',  '\'',  't',  'any',  'fleas',  '.' ]
 ```
 
 String Distance
@@ -193,7 +184,7 @@ tokenize and stem the text.
 
 ```javascript
 classifier.addDocument('i am long qqqq', 'buy');
-classifier.addDocument('buy the q''s', 'buy');
+classifier.addDocument('buy the q\'s', 'buy');
 classifier.addDocument('short gold', 'sell');
 classifier.addDocument('sell gold', 'sell');
 
@@ -217,8 +208,8 @@ You have access to the set of matched classes and the associated value from the
 Outputs:
 
 ```javascript
-[ { label: 'sell', value: 0.39999999999999997 },
-  { label: 'buy', value: 0.19999999999999998 } ]
+[ { label: 'buy', value: 0.39999999999999997 },
+  { label: 'sell', value: 0.19999999999999998 } ]
 ```
 
 From this:
@@ -492,6 +483,57 @@ The above outputs: `[ [ 'some', 'other', 'words', 'here' ],
   [ 'other', 'words', 'here', 'for' ],
   [ 'words', 'here', 'for', 'you' ] ]`
 
+### padding
+
+n-grams can also be returned with left or right padding by passing a start and/or end symbol to the bigrams, trigrams or ngrams.
+
+```javascript
+console.log(NGrams.ngrams('some other words here for you', 4, '[start]', '[end]'));
+```
+
+The above will output: 
+```
+[ [ '[start]', '[start]', '[start]', 'some' ],
+  [ '[start]', '[start]', 'some', 'other' ],
+  [ '[start]', 'some', 'other', 'words' ],
+  [ 'some', 'other', 'words', 'here' ],
+  [ 'other', 'words', 'here', 'for' ],
+  [ 'words', 'here', 'for', 'you' ],
+  [ 'here', 'for', 'you', '[end]' ],
+  [ 'for', 'you', '[end]', '[end]' ],
+  [ 'you', '[end]', '[end]', '[end]' ] ]
+```
+
+For only end symbols, pass `null` for the start symbol, for instance:
+```javascript
+console.log(NGrams.ngrams('some other words here for you', 4, null, '[end]'));
+```
+
+Will output: 
+```
+[ [ 'some', 'other', 'words', 'here' ],
+  [ 'other', 'words', 'here', 'for' ],
+  [ 'words', 'here', 'for', 'you' ],
+  [ 'here', 'for', 'you', '[end]' ],
+  [ 'for', 'you', '[end]', '[end]' ],
+  [ 'you', '[end]', '[end]', '[end]' ] ]
+```
+
+### NGramsZH
+
+For Chinese like languages, you can use NGramsZH to do a n-gram, and all apis are the same:
+
+```javascript
+var NGramsZH = natural.NGramsZH;
+console.log(NGramsZH.bigrams('中文测试'));
+console.log(NGramsZH.bigrams(['中',  '文',  '测', '试']));
+console.log(NGramsZH.trigrams('中文测试'));
+console.log(NGramsZH.trigrams(['中',  '文', '测',  '试']));
+console.log(NGramsZH.ngrams('一个中文测试', 4));
+console.log(NGramsZH.ngrams(['一', '个', '中', '文', '测',
+    '试'], 4));
+```
+
 tf-idf
 -----
 
@@ -555,7 +597,7 @@ tfidf.addFileSync('data_files/two.txt');
 
 Multiple terms can be measured as well, with their weights being added into
 a single measure value. The following example determines that the last document
-is the most relevent to the words "node" and "ruby".
+is the most relevant to the words "node" and "ruby".
 
 ```javascript
 var natural = require('natural'),
@@ -608,7 +650,7 @@ tfidf.listTerms(0 /*document index*/).forEach(function(item) {
 });
 ```
 
-A TfIdf instance can also be serialized and deserialzed for save and recall.
+A TfIdf instance can also be serialized and deserialized for save and recall.
 
 ```javascript
 var tfidf = new TfIdf();
@@ -625,8 +667,8 @@ Tries
 -----
 
 Tries are a very efficient data structure used for prefix-based searches. 
-Natural comes packaged with a basic Trie implementation wich can support match collection along a path,
-existance search and prefix search.
+Natural comes packaged with a basic Trie implementation which can support match collection along a path,
+existence search and prefix search.
 
 ### Building The Trie
 
@@ -677,6 +719,14 @@ trie.addString("est");
 console.log(trie.findMatchesOnPath("tester")); // ['tes', 'test'];
 ```
 
+### All Keys with Prefix
+
+This search will return all of the words in the Trie with the given prefix, or [ ] if not found.
+
+```javascript
+console.log(trie.keysWithPrefix("string")); // ["string1", "string2", "string3"]
+```
+
 ### Case-Sensitivity
 
 By default the trie is case-sensitive, you can use it in case-_in_sensitive mode by passing `false` 
@@ -691,6 +741,144 @@ ciTrie.contains("TEsT"); // true
 ```
 In the case of the searches which return strings, all strings returned will be in lower case if you are in case-_in_sensitive mode.
 
+EdgeWeightedDigraph
+-------------------
+EdgeWeightedDigraph represents a digraph, you can add an edge, get the number vertexes, edges, get all edges and use toString to print the Digraph.
+
+initialize a digraph:
+
+```javascript
+var EdgeWeightedDigraph = natural.EdgeWeightedDigraph;
+var digraph = new EdgeWeightedDigraph();
+digraph.add(5,4,0.35);
+digraph.add(5,1,0.32);
+digraph.add(1,3,0.29);
+digraph.add(6,2,0.40);
+digraph.add(3,6,0.52);
+digraph.add(6,4,0.93);
+```
+the api used is: add(from, to, weight).
+
+get the number of vertexes:
+
+```javascript
+console.log(digraph.v());
+```
+you will get 5.
+
+get the number of edges:
+
+```javascript
+console.log(digraph.e());
+```
+you will get 5.
+
+
+
+ShortestPathTree
+----------------
+
+ShortestPathTree represents a data type for solving the single-source shortest paths problem in
+edge-weighted directed acyclic graphs (DAGs).
+The edge weights can be positive, negative, or zero. There are three APIs:
+getDistTo(vertex),
+hasPathTo(vertex),
+pathTo(vertex).
+
+```javascript
+var ShortestPathTree = natural.ShortestPathTree;
+var spt = new ShortestPathTree(digraph, 5);
+```
+digraph is an instance of EdgeWeightedDigraph, the second param is the start vertex of DAG.
+
+### getDistTo(vertex)
+
+Will return the dist to vertex.
+
+```javascript
+console.log(spt.getDistTo(4));
+```
+the output will be: 0.35
+
+### hasDistTo(vertex)
+
+```javascript
+console.log(spt.hasDistTo(4));
+console.log(spt.hasDistTo(5));
+```
+
+output will be:
+
+```javascript
+true
+false
+```
+
+### pathTo(vertex)
+this will return a shortest path:
+
+```javascript
+console.log(spt.pathTo(4));
+```
+
+output will be:
+
+```javascript
+[5, 4]
+```
+
+LongestPathTree
+----------------
+
+LongestPathTree represents a data type for solving the single-source shortest paths problem in
+edge-weighted directed acyclic graphs (DAGs).
+The edge weights can be positive, negative, or zero. There are three APIs same as ShortestPathTree:
+getDistTo(vertex),
+hasPathTo(vertex),
+pathTo(vertex).
+
+```javascript
+var ShortestPathTree = natural.ShortestPathTree;
+var spt = new ShortestPathTree(digraph, 5);
+```
+digraph is an instance of EdgeWeightedDigraph, the second param is the start vertex of DAG.
+
+### getDistTo(vertex)
+
+Will return the dist to vertex.
+
+```javascript
+console.log(spt.getDistTo(4));
+```
+the output will be: 2.06
+
+### hasDistTo(vertex)
+
+```javascript
+console.log(spt.hasDistTo(4));
+console.log(spt.hasDistTo(5));
+```
+
+output will be:
+
+```javascript
+true
+false
+```
+
+### pathTo(vertex)
+this will return a shortest path:
+
+```javascript
+console.log(spt.pathTo(4));
+```
+
+output will be:
+
+```javascript
+[5, 1, 3, 6, 4]
+```
+
 WordNet
 -------
 
@@ -747,6 +935,31 @@ As of v0.1.11, WordNet data files are no longer automatically downloaded.
 
 Princeton University "About WordNet." WordNet. Princeton University. 2010. <http://wordnet.princeton.edu>
 
+Spellcheck
+------------
+A probabilistic spellchecker based on http://norvig.com/spell-correct.html
+
+This is best constructed with an array of tokens from a corpus, but a simple list of words from a dictionary will work. 
+
+```javascript
+var corpus = ['something', 'soothing'];
+var spellcheck = new Spellcheck(corpus);
+```
+
+It uses the trie datastructure for fast boolean lookup of a word
+
+```javascript
+spellcheck.isCorrect('cat'); // false
+```
+
+It suggests corrections (sorted by probability in descending order) that are up to a maximum edit distance away from the input word. According to Norvig, a max distance of 1 will cover 80% to 95% of spelling mistakes. After a distance of 2, it becomes very slow.
+
+```javascript
+spellcheck.getCorrections('soemthing', 1); // ['something']
+spellcheck.getCorrections('soemthing', 2); // ['something', 'soothing']
+```
+
+
 Development
 -----------
 When developing, please:

package/benchmarks/index.js

@@ -29,4 +29,4 @@ suite = new uubench.Suite({
 require('./metaphone');
 require('./soundex');
 
-suite.run();
+suite.run();

package/lib/natural/analyzers/sentence_analyzer.js

@@ -46,7 +46,7 @@ var Sentences = function(pos, callback) {
     this.posObj = pos;
     this.senType = null;
     callback(this);
-}
+};
 
 Sentences.prototype.part = function(callback) {
     var subject = [],
@@ -88,7 +88,7 @@ Sentences.prototype.part = function(callback) {
     }
     
     callback(this);	
-}
+};
 
 // Takes POS and removes IN to NN or NNS
 // Adds a PP for each prepositionPhrases
@@ -108,15 +108,15 @@ Sentences.prototype.prepositionPhrases = function() {
             remove = false;
         }
     }	
-}
+};
 
 Sentences.prototype.subjectToString = function() {
     return this.posObj.tags.map(function(t){ if (t.spos == "SP" || t.spos == "S" ) return t.token }).join(' ');
-}
+};
 
 Sentences.prototype.predicateToString = function() {
     return this.posObj.tags.map(function(t){ if (t.spos == "PP" || t.spos == "P" ) return t.token }).join(' ');
-}
+};
 
 Sentences.prototype.implicitYou = function() {
     for (var i = 0; i < this.posObj.tags.length;i++) {
@@ -126,22 +126,25 @@ Sentences.prototype.implicitYou = function() {
     }
     
     return false;
-}
+};
 
 Sentences.prototype.toString = function() {
     return this.posObj.tags.map(function(t){return t.token}).join(' ');
-}
+};
 
 // This is quick and incomplete.
 Sentences.prototype.type = function(callback) {
     var callback = callback || false;
-	
+
+    // Check for implicit you before popping a tag.
+    var implicitYou = this.implicitYou();
+
     // FIXME - punct seems useless
     var lastElement = this.posObj.punct();
     lastElement = (lastElement.length != 0) ? lastElement.pop() : this.posObj.tags.pop();
-	
+
     if (lastElement.pos !== ".") {
-        if (this.implicitYou()) {
+        if (implicitYou) {
             this.senType = "COMMAND";
         } else if (_(["WDT","WP","WP$","WRB"]).contains(this.posObj.tags[0].pos)) {
             // Sentences that start with: who, what where when why and how, then they are questions
@@ -158,8 +161,8 @@ Sentences.prototype.type = function(callback) {
     } else {
         switch(lastElement.token) {
             case "?": this.senType = "INTERROGATIVE"; break;
-            case "!": this.senType = (this.implicitYou()) ? "COMMAND":"EXCLAMATORY"; break;
-            case ".": this.senType = (this.implicitYou()) ? "COMMAND":"DECLARATIVE";	break;
+            case "!": this.senType = (implicitYou) ? "COMMAND":"EXCLAMATORY"; break;
+            case ".": this.senType = (implicitYou) ? "COMMAND":"DECLARATIVE";	break;
         }
     }
     
@@ -168,6 +171,6 @@ Sentences.prototype.type = function(callback) {
     } else {
         return this.senType;
     }
-}
+};
 
 module.exports = Sentences;

package/lib/natural/classifiers/bayes_classifier.js

@@ -25,8 +25,12 @@ util = require('util'),
 Classifier = require('./classifier'),
 ApparatusBayesClassifier = require('apparatus').BayesClassifier;
 
-var BayesClassifier = function(stemmer) {
-    Classifier.call(this, new ApparatusBayesClassifier(), stemmer);
+var BayesClassifier = function(stemmer, smoothing) {
+    var abc = new ApparatusBayesClassifier();
+    if (smoothing && isFinite(smoothing)) {
+        abc = new ApparatusBayesClassifier(smoothing);
+    }
+    Classifier.call(this, abc, stemmer);
 };
 
 util.inherits(BayesClassifier, Classifier);

package/lib/natural/distance/jaro-winkler_distance.js

@@ -99,13 +99,18 @@ function distance(s1, s2) {
 // s2 is the second string to compare
 // dj is the Jaro Distance (if you've already computed it), leave blank and the method handles it
 function JaroWinklerDistance(s1, s2, dj) {
-    var jaro;
-    (typeof(dj) == 'undefined')? jaro = distance(s1,s2) : jaro = dj;
-    var p = 0.1; //
-    var l = 0 // length of the matching prefix
-    while(s1[l] == s2[l] && l < 4)
-        l++;
-    
-    return jaro + l * p * (1 - jaro);
+		if (s1 == s2) {
+				return 1 
+		}
+		else {
+		    var jaro;
+		    (typeof(dj) == 'undefined')? jaro = distance(s1,s2) : jaro = dj;
+		    var p = 0.1; //
+		    var l = 0 // length of the matching prefix
+		    while(s1[l] == s2[l] && l < 4)
+		        l++;
+		    
+		    return jaro + l * p * (1 - jaro);
+		}
 }
 module.exports = JaroWinklerDistance;

package/lib/natural/index.js

@@ -26,10 +26,13 @@ exports.DoubleMetaphone = require('./phonetics/double_metaphone');
 exports.SoundExDM = require('./phonetics/dm_soundex');
 exports.PorterStemmer = require('./stemmers/porter_stemmer');
 exports.PorterStemmerFa = require('./stemmers/porter_stemmer_fa');
+exports.PorterStemmerFr = require('./stemmers/porter_stemmer_fr');
 exports.PorterStemmerRu = require('./stemmers/porter_stemmer_ru');
 exports.PorterStemmerEs = require('./stemmers/porter_stemmer_es');
 exports.PorterStemmerIt = require('./stemmers/porter_stemmer_it');
+exports.PorterStemmerNo = require('./stemmers/porter_stemmer_no');
 exports.LancasterStemmer = require('./stemmers/lancaster_stemmer');
+exports.StemmerFr = require('./stemmers/stemmer_fr');
 exports.StemmerPl = require('./stemmers/stemmer_pl');
 exports.StemmerJa = require('./stemmers/stemmer_ja');
 exports.AggressiveTokenizerNl = require('./tokenizers/aggressive_tokenizer_nl');
@@ -39,6 +42,7 @@ exports.AggressiveTokenizerEs = require('./tokenizers/aggressive_tokenizer_es');
 exports.AggressiveTokenizerIt = require('./tokenizers/aggressive_tokenizer_it');
 exports.AggressiveTokenizerPl = require('./tokenizers/aggressive_tokenizer_pl');
 exports.AggressiveTokenizerPt = require('./tokenizers/aggressive_tokenizer_pt');
+exports.AggressiveTokenizerNo = require('./tokenizers/aggressive_tokenizer_no');
 exports.AggressiveTokenizer = require('./tokenizers/aggressive_tokenizer');
 exports.RegexpTokenizer = require('./tokenizers/regexp_tokenizer').RegexpTokenizer;
 exports.WordTokenizer = require('./tokenizers/regexp_tokenizer').WordTokenizer;
@@ -57,7 +61,11 @@ exports.TfIdf = require('./tfidf/tfidf');
 exports.Trie = require('./trie/trie');
 exports.SentenceAnalyzer = require('./analyzers/sentence_analyzer');
 exports.stopwords = require('./util/stopwords').words;
+exports.ShortestPathTree = require('./util/shortest_path_tree');
+exports.LongestPathTree = require('./util/longest_path_tree');
+exports.EdgeWeightedDigraph = require('./util/edge_weighted_digraph');
 exports.NGrams = require('./ngrams/ngrams');
+exports.NGramsZH = require('./ngrams/ngrams_zh');
 exports.JaroWinklerDistance = require('./distance/jaro-winkler_distance');
 exports.LevenshteinDistance = require('./distance/levenshtein_distance');
 exports.DiceCoefficient = require('./distance/dice_coefficient');

package/lib/natural/inflectors/present_verb_inflector.js

@@ -51,6 +51,7 @@ var VerbInflector = function() {
     this.addIrregular("am", "are");    
     this.addIrregular("is", "are");
     this.addIrregular("was", "were");
+    this.addIrregular("has", "have");
     
     this.singularForms.regularForms.push([/ed$/i, 'ed']);
     this.singularForms.regularForms.push([/ss$/i, 'sses']);

package/lib/natural/inflectors/singular_plural_inflector.js

@@ -77,8 +77,7 @@ TenseInflector.prototype.restoreCase = function(token) {
 
 TenseInflector.prototype.izeRegulars = function(token, formSet) {
     token = token.toLowerCase();
-    
-    if(formSet.irregularForms[token])
+    if(formSet.irregularForms.hasOwnProperty(token) && formSet.irregularForms[token])
         return formSet.irregularForms[token];
 
     return false;