natural 0.1.28 → 0.4.0

package/lib/natural/stemmers/stemmer_pt.js

@@ -0,0 +1,65 @@
+/*
+Copyright (c) 2014, Ismaël Héry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+module.exports = function () {
+  'use strict';
+
+  var Stemmer = this,
+    stopwords = require('../util/stopwords_pt'),
+    Tokenizer = require('../tokenizers/aggressive_tokenizer_pt');
+
+  Stemmer.stem = function (token) {
+    return token;
+  };
+
+  Stemmer.addStopWords = function (word) {
+    stopwords.words.push(word);
+  };
+
+  Stemmer.addStopWords = function (words) {
+    stopwords.words = stopwords.words.concat(words);
+  };
+
+  Stemmer.tokenizeAndStem = function(text, keepStops) {
+    var stemmedTokens = [];
+
+    var tokenStemmer = function (token) {
+      if (keepStops || stopwords.words.indexOf(token.toLowerCase()) === -1) {
+        stemmedTokens.push(Stemmer.stem(token));
+      }
+    };
+
+    new Tokenizer().tokenize(text).forEach(tokenStemmer);
+
+    return stemmedTokens;
+  };
+
+  Stemmer.attach = function () {
+    String.prototype.stem = function () {
+      return Stemmer.stem(this);
+    };
+
+    String.prototype.tokenizeAndStem = function (keepStops) {
+      return Stemmer.tokenizeAndStem(this, keepStops);
+    };
+  };
+};

package/lib/natural/stemmers/token.js

@@ -0,0 +1,163 @@
+/*
+Copyright (c) 2015, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+module.exports = (function () {
+  'use strict';
+
+  /**
+   * Stemmer token constructor.
+   *
+   * @param {String} string Token string.
+   */
+  var Token = function (string) {
+    this.vowels   = '';
+    this.regions  = {};
+    this.string   = string;
+    this.original = string;
+  }
+
+  /**
+   * Set vowels.
+   *
+   * @param  {String|Array} vowels List of vowels.
+   * @return {Token}               Token instance.
+   */
+  Token.prototype.usingVowels = function (vowels) {
+    this.vowels = vowels;
+    return this;
+  };
+
+  /**
+   * Marks a region by defining its starting index or providing a callback
+   * function that does.
+   *
+   * @param  {String}       region   Region name.
+   * @param  {Array|Number} args     Callback arguments or region start index.
+   * @param  {Function}     callback Function that determines the start index (optional).
+   * @param  {Object}       context  Callback context (optional, defaults to this).
+   * @return {Token}                 Token instance.
+   */
+  Token.prototype.markRegion = function (region, args, callback, context) {
+    if (typeof callback === 'function') {
+      this.regions[region] = callback.apply(context || this, [].concat(args));
+
+    } else if (!isNaN(args)) {
+      this.regions[region] = args;
+    }
+
+    return this;
+  };
+
+  /**
+   * Replaces all instances of a string with another.
+   *
+   * @param  {String} find    String to be replaced.
+   * @param  {String} replace Replacement string.
+   * @return {Token}          Token instance.
+   */
+  Token.prototype.replaceAll = function (find, replace) {
+    this.string = this.string.split(find).join(replace);
+    return this;
+  };
+
+  /**
+   * Replaces the token suffix if in a region.
+   *
+   * @param  {String} suffix  Suffix to replace.
+   * @param  {String} replace Replacement string.
+   * @param  {String} region  Region name.
+   * @return {Token}          Token instance.
+   */
+  Token.prototype.replaceSuffixInRegion = function (suffix, replace, region) {
+    var suffixes = [].concat(suffix);
+    for (var i = 0; i < suffixes.length; i++) {
+      if (this.hasSuffixInRegion(suffixes[i], region)) {
+        this.string = this.string.slice(0, -suffixes[i].length) + replace;
+        return this;
+      }
+    }
+    return this;
+  };
+
+  /**
+   * Determines whether the token has a vowel at the provided index.
+   *
+   * @param  {Integer} index Character index.
+   * @return {Boolean}       Whether the token has a vowel at the provided index.
+   */
+  Token.prototype.hasVowelAtIndex = function (index) {
+    return this.vowels.indexOf(this.string[index]) !== -1;
+  };
+
+  /**
+   * Finds the next vowel in the token.
+   *
+   * @param  {Integer} start Starting index offset.
+   * @return {Integer}       Vowel index, or the end of the string.
+   */
+  Token.prototype.nextVowelIndex = function (start) {
+    var index = (start >= 0 && start < this.string.length) ? start : this.string.length;
+    while (index < this.string.length && !this.hasVowelAtIndex(index)) {
+      index++;
+    }
+    return index;
+  };
+
+  /**
+   * Finds the next consonant in the token.
+   *
+   * @param  {Integer} start Starting index offset.
+   * @return {Integer}       Consonant index, or the end of the string.
+   */
+  Token.prototype.nextConsonantIndex = function (start) {
+    var index = (start >= 0 && start < this.string.length) ? start : this.string.length;
+    while (index < this.string.length && this.hasVowelAtIndex(index)) {
+      index++;
+    }
+    return index;
+  };
+
+  /**
+   * Determines whether the token has the provided suffix.
+   * @param  {String}  suffix Suffix to match.
+   * @return {Boolean}        Whether the token string ends in suffix.
+   */
+  Token.prototype.hasSuffix = function (suffix) {
+    return this.string.slice(-suffix.length) === suffix;
+  };
+
+  /**
+   * Determines whether the token has the provided suffix within the specified
+   * region.
+   *
+   * @param  {String}  suffix Suffix to match.
+   * @param  {String}  region Region name.
+   * @return {Boolean}        Whether the token string ends in suffix.
+   */
+  Token.prototype.hasSuffixInRegion = function (suffix, region) {
+    var regionStart = this.regions[region] || 0,
+      suffixStart   = this.string.length - suffix.length;
+    return this.hasSuffix(suffix) && suffixStart >= regionStart;
+  };
+
+  return Token;
+})();

package/lib/natural/tfidf/tfidf.js

@@ -28,13 +28,13 @@ var _ = require("underscore")._,
 
 function buildDocument(text, key) {
     var stopOut;
-    
+
     if(typeof text === 'string') {
         text = tokenizer.tokenize(text.toLowerCase());
         stopOut = true;
     } else if(!_.isArray(text)) {
-        return text;
         stopOut = false;
+        return text;
     }
 
     return text.reduce(function(document, term) {
@@ -63,12 +63,33 @@ function TfIdf(deserialized) {
     this._idfCache = {};
 }
 
+// backwards compatibility for < node 0.10
+function isEncoding(encoding) {
+    if (typeof Buffer.isEncoding !== 'undefined')
+        return Buffer.isEncoding(encoding);
+    switch ((encoding + '').toLowerCase()) {
+        case 'hex':
+        case 'utf8':
+        case 'utf-8':
+        case 'ascii':
+        case 'binary':
+        case 'base64':
+        case 'ucs2':
+        case 'ucs-2':
+        case 'utf16le':
+        case 'utf-16le':
+        case 'raw':
+            return true;
+    }
+    return false;
+}
+
 module.exports = TfIdf;
 TfIdf.tf = tf;
 
 TfIdf.prototype.idf = function(term, force) {
 
-    // Lookup the term in the New term-IDF caching, 
+    // Lookup the term in the New term-IDF caching,
     // this will cut search times down exponentially on large document sets.
     if(this._idfCache[term] && this._idfCache.hasOwnProperty(term) && force !== true)
         return this._idfCache[term];
@@ -77,7 +98,7 @@ TfIdf.prototype.idf = function(term, force) {
         return count + (documentHasTerm(term, document) ? 1 : 0);
     }, 0);
 
-    var idf = Math.log((this.documents.length) / (docsWithTerm));
+    var idf = 1 + Math.log((this.documents.length) / ( 1 + docsWithTerm ));
 
     // Add the idf to the term cache and return it
     this._idfCache[term] = idf;
@@ -92,13 +113,12 @@ TfIdf.prototype.addDocument = function(document, key, restoreCache) {
     // make sure the cache is invalidated when new documents arrive
     if(restoreCache === true) {
         for(var term in this._idfCache) {
-            // invoking idf with the force option set will 
+            // invoking idf with the force option set will
             // force a recomputation of the idf, and it will
             // automatically refresh the cache value.
             this.idf(term, true);
         }
-    }
-    else {
+    }   else {
         this._idfCache = {};
     }
 };
@@ -107,15 +127,17 @@ TfIdf.prototype.addDocument = function(document, key, restoreCache) {
 // Otherwise, the cache will just be wiped clean
 TfIdf.prototype.addFileSync = function(path, encoding, key, restoreCache) {
     if(!encoding)
-        encoding = 'UTF-8';
-        
-    var document = fs.readFileSync(path, 'UTF-8');
+        encoding = 'utf8';
+    if(!isEncoding(encoding))
+        throw new Error('Invalid encoding: ' + encoding);
+
+    var document = fs.readFileSync(path, encoding);
     this.documents.push(buildDocument(document, key));
 
     // make sure the cache is invalidated when new documents arrive
     if(restoreCache === true) {
         for(var term in this._idfCache) {
-            // invoking idf with the force option set will 
+            // invoking idf with the force option set will
             // force a recomputation of the idf, and it will
             // automatically refresh the cache value.
             this.idf(term, true);
@@ -128,12 +150,14 @@ TfIdf.prototype.addFileSync = function(path, encoding, key, restoreCache) {
 
 TfIdf.prototype.tfidf = function(terms, d) {
     var _this = this;
-    
+
     if(!_.isArray(terms))
         terms = tokenizer.tokenize(terms.toString().toLowerCase());
 
     return terms.reduce(function(value, term) {
-        return value + (tf(term, _this.documents[d]) * _this.idf(term));
+        var idf = _this.idf(term);
+        idf = idf === Infinity ? 0 : idf;
+        return value + (tf(term, _this.documents[d]) * idf);
     }, 0.0);
 };
 
@@ -142,21 +166,28 @@ TfIdf.prototype.listTerms = function(d) {
 
     for(var term in this.documents[d]) {
         if(term != '__key')
-    	   terms.push({term: term, tfidf: this.tfidf(term, d)})
+           terms.push({term: term, tfidf: this.tfidf(term, d)});
     }
 
-    return terms.sort(function(x, y) { return y.tfidf - x.tfidf });
-}
+    return terms.sort(function(x, y) { return y.tfidf - x.tfidf; });
+};
 
 TfIdf.prototype.tfidfs = function(terms, callback) {
     var tfidfs = new Array(this.documents.length);
-    
+
     for(var i = 0; i < this.documents.length; i++) {
         tfidfs[i] = this.tfidf(terms, i);
-        
+
         if(callback)
             callback(i, tfidfs[i], this.documents[i].__key);
     }
 
     return tfidfs;
 };
+
+// Define a tokenizer other than the default "WordTokenizer"
+TfIdf.prototype.setTokenizer = function(t) {
+    if(!_.isFunction(t.tokenize))
+        throw new Error('Expected a valid Tokenizer');
+    tokenizer = t;
+};

package/lib/natural/tokenizers/regexp_tokenizer.js

@@ -62,7 +62,7 @@ exports.RegexpTokenizer = RegexpTokenizer;
  * 
  */
 var WordTokenizer = function(options) {
-    this._pattern = /\W+/;
+    this._pattern = /[^A-Za-zА-Яа-я0-9_]+/;
     RegexpTokenizer.call(this,options)
 };
 
@@ -74,11 +74,11 @@ exports.WordTokenizer = WordTokenizer;
  * non-alphabetic characters.  E.g.:
  *
  *      >>> WordPunctTokenizer().tokenize("She said 'hello'.")
- *      ['She', 'said', "'", 'hello', "'."]
+ *      ["She","said","'","hello","'","."]
  * 
  */
 var WordPunctTokenizer = function(options) {
-    this._pattern = new RegExp(/(\w+|\!|\'|\"")/i);
+    this._pattern = new RegExp(/(\w+|[а-я0-9_]+|\.|\!|\'|\"")/i);
     RegexpTokenizer.call(this,options)
 };
 

package/{examples/inflection/noun.js → lib/natural/tokenizers/sentence_tokenizer.js}

@@ -20,18 +20,22 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-var natural = require('natural'),
-    nounInflector = new natural.NounInflector();
+var Tokenizer = require('./tokenizer'),
+    util = require('util');
 
-plural = nounInflector.pluralize('radius');
-console.log(plural);
+var SentenceTokenizer = function() {
+    Tokenizer.call(this);
+};
+util.inherits(SentenceTokenizer, Tokenizer);
 
-plural = nounInflector.pluralize('beer');
-console.log(plural);
+SentenceTokenizer.prototype.tokenize = function(text) {
+    // break string up in to sentences based on punctation and quotation marks
+    var tokens = text.match(/([\"\'\‘\“\'\"\[\(\{\⟨][^\.\?\!]+[\.\?\!][\"\'\’\”\'\"\]\)\}\⟩]|[^\.\?\!]+[\.\?\!])\s?/g);
 
-singular = nounInflector.singularize('radii');
-console.log(singular);
+    // remove unecessary white space
+    tokens = tokens.map(Function.prototype.call, String.prototype.trim);
 
-singular = nounInflector.singularize('beers');
-console.log(singular);
+    return this.trim(tokens);
+};
 
+module.exports = SentenceTokenizer;

package/lib/natural/tokenizers/tokenizer_case.js

@@ -0,0 +1,58 @@
+/*
+ Copyright (c) 2011, Chris Umbel, Alex Langberg
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+var Tokenizer = require('./tokenizer'),
+  util = require('util'),
+  CaseTokenizer = function() {
+    Tokenizer.call(this);
+  };
+
+util.inherits(CaseTokenizer, Tokenizer);
+
+CaseTokenizer.prototype.attach = function() {
+  var self = this;
+
+  String.prototype.tokenize = function(preserveApostrophe) {
+    return self.tokenize(this, preserveApostrophe);
+  }
+};
+
+// Idea from Seagull: http://stackoverflow.com/a/26482650
+CaseTokenizer.prototype.tokenize = function(text, preserveApostrophe) {
+  var whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'];
+  var lower = text.toLowerCase();
+  var upper = text.toUpperCase();
+  var result = '';
+  var i;
+
+  for (i = 0; i < lower.length; ++i) {
+    if (lower[i] !== upper[i] || whitelist.indexOf(lower[i]) > -1 || (text[i] === '\'' && preserveApostrophe)) {
+      result += text[i];
+    } else {
+      result += ' ';
+    }
+  }
+
+  return this.trim(result.replace(/\s+/g, ' ').split(' '));
+};
+
+module.exports = CaseTokenizer;

package/lib/natural/util/bag.js

@@ -28,6 +28,7 @@ function Bag() {
 
 Bag.prototype.add = function(element) {
     this.dictionary.push(element);
+    this.nElement++;
     return this;
 };
 

package/lib/natural/util/stopwords_pt.js

@@ -0,0 +1,136 @@
+/*
+Copyright (c) 2011, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+// a list of commonly used words that have little meaning and can be excluded
+// from analysis.
+var words = [
+  'a',
+  'à',
+  'ao',
+  'aos',
+  'aquela',
+  'aquelas',
+  'aquele',
+  'aqueles',
+  'aquilo',
+  'as',
+  'às',
+  'até',
+  'com',
+  'como',
+  'da',
+  'das',
+  'de',
+  'dela',
+  'delas',
+  'dele',
+  'deles',
+  'depois',
+  'do',
+  'dos',
+  'e',
+  'ela',
+  'elas',
+  'ele',
+  'eles',
+  'em',
+  'entre',
+  'essa',
+  'essas',
+  'esse',
+  'esses',
+  'esta',
+  'estas',
+  'este',
+  'estes',
+  'eu',
+  'isso',
+  'isto',
+  'já',
+  'lhe',
+  'lhes',
+  'mais',
+  'mas',
+  'me',
+  'mesmo',
+  'meu',
+  'meus',
+  'minha',
+  'minhas',
+  'muito',
+  'muitos',
+  'na',
+  'não',
+  'nas',
+  'nem',
+  'no',
+  'nos',
+  'nós',
+  'nossa',
+  'nossas',
+  'nosso',
+  'nossos',
+  'num',
+  'nuns',
+  'numa',
+  'numas',
+  'o',
+  'os',
+  'ou',
+  'para',
+  'pela',
+  'pelas',
+  'pelo',
+  'pelos',
+  'por',
+  'quais',
+  'qual',
+  'quando',
+  'que',
+  'quem',
+  'se',
+  'sem',
+  'seu',
+  'seus',
+  'só',
+  'sua',
+  'suas',
+  'também',
+  'te',
+  'teu',
+  'teus',
+  'tu',
+  'tua',
+  'tuas',
+  'um',
+  'uma',
+  'umas',
+  'você',
+  'vocês',
+  'vos',
+  'vosso',
+  'vossos',
+  '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '_'
+];
+
+// tell the world about the noise words.
+exports.words = words;

package/lib/natural/wordnet/wordnet.js

@@ -137,9 +137,9 @@ function WordNet(dataDir) {
 
   if (!dataDir) {
     try {
-      var WNdb = require('WNdb');
+      var WNdb = require('wordnet-db');
     } catch(e) {
-      console.error("Please 'npm install WNdb' before using WordNet module or specify a dict directory.");
+      console.error("Please 'npm install wordnet-db' before using WordNet module or specify a dict directory.");
       throw e;
     }
     dataDir = WNdb.path;

package/lib/natural/wordnet/wordnet_file.js

@@ -37,7 +37,7 @@ function appendLineChar(fd, pos, buffPos, buff, callback) {
       console.log(err);
     else {
       if(buff[buffPos] == 10 || buffPos == buff.length)
-        callback(buff.slice(0, buffPos).toString('ASCII'));
+        callback(buff.slice(0, buffPos).toString('UTF-8'));
       else {
         appendLineChar(fd, pos + 1, buffPos + 1, buff, callback);
       }

package/lr_classifier.json

@@ -0,0 +1 @@
+{"classifier":{"examples":{"computing":[[1,1,0,0,0,0,0,0,0],[0,0,1,1,0,0,0,0,0],[0,0,0,1,1,1,0,0,0]],"literature":[[0,0,1,0,0,0,1,0,0],[0,0,0,0,0,0,1,1,0],[0,0,0,0,0,0,1,0,1]]},"features":[],"featurePositions":{},"maxFeaturePosition":0,"classifications":["computing","literature"],"exampleCount":6,"theta":[{"elements":[2.0057203693422903,2.0057203693422903,0.0904766662626176,3.447364098636818,0.9696521506748157,0.9696521506748157,-4.840706080886848,-1.2267353995937322,-1.2267353995937322]},{"elements":[-2.00572036934229,-2.00572036934229,-0.09047666626261795,-3.447364098636819,-0.9696521506748155,-0.9696521506748155,4.84070608088685,1.2267353995937313,1.2267353995937313]}]},"docs":[{"label":"computing","text":["fix","box"]},{"label":"computing","text":["write","code"]},{"label":"computing","text":["nasti","script","code"]},{"label":"literature","text":["write","book"]},{"label":"literature","text":["read","book"]},{"label":"literature","text":["studi","book"]}],"features":{"fix":1,"box":1,"write":2,"code":2,"nasti":1,"script":1,"book":3,"read":1,"studi":1},"stemmer":{},"lastAdded":6,"events":{"domain":null,"_events":{},"_maxListeners":10}}