natural 0.5.4 → 0.6.1

package/lib/natural/index.js

@@ -31,11 +31,16 @@ exports.PorterStemmerRu = require('./stemmers/porter_stemmer_ru');
 exports.PorterStemmerEs = require('./stemmers/porter_stemmer_es');
 exports.PorterStemmerIt = require('./stemmers/porter_stemmer_it');
 exports.PorterStemmerNo = require('./stemmers/porter_stemmer_no');
+exports.PorterStemmerSv = require('./stemmers/porter_stemmer_sv');
 exports.PorterStemmerPt = require('./stemmers/porter_stemmer_pt');
+exports.PorterStemmerNl = require('./stemmers/porter_stemmer_nl');
 exports.LancasterStemmer = require('./stemmers/lancaster_stemmer');
-exports.StemmerFr = require('./stemmers/stemmer_fr');
-exports.StemmerPl = require('./stemmers/stemmer_pl');
+// StemmerFr and StemmerPl are not stemmers. A Polish stemmer is
+// not available, and for French PorterStemmerFr should be used.
+//exports.StemmerFr = require('./stemmers/stemmer_fr');
+//exports.StemmerPl = require('./stemmers/stemmer_pl');
 exports.StemmerJa = require('./stemmers/stemmer_ja');
+exports.StemmerId = require('./stemmers/indonesian/stemmer_id');
 exports.AggressiveTokenizerNl = require('./tokenizers/aggressive_tokenizer_nl');
 exports.AggressiveTokenizerFa = require('./tokenizers/aggressive_tokenizer_fa');
 exports.AggressiveTokenizerFr = require('./tokenizers/aggressive_tokenizer_fr');
@@ -45,9 +50,12 @@ exports.AggressiveTokenizerIt = require('./tokenizers/aggressive_tokenizer_it');
 exports.AggressiveTokenizerPl = require('./tokenizers/aggressive_tokenizer_pl');
 exports.AggressiveTokenizerPt = require('./tokenizers/aggressive_tokenizer_pt');
 exports.AggressiveTokenizerNo = require('./tokenizers/aggressive_tokenizer_no');
+exports.AggressiveTokenizerSv = require('./tokenizers/aggressive_tokenizer_sv');
+exports.AggressiveTokenizerVi = require('./tokenizers/aggressive_tokenizer_vi');
 exports.AggressiveTokenizer = require('./tokenizers/aggressive_tokenizer');
 exports.CaseTokenizer = require('./tokenizers/tokenizer_case');
 exports.RegexpTokenizer = require('./tokenizers/regexp_tokenizer').RegexpTokenizer;
+exports.OrthographyTokenizer = require('./tokenizers/regexp_tokenizer').OrthographyTokenizer;
 exports.WordTokenizer = require('./tokenizers/regexp_tokenizer').WordTokenizer;
 exports.WordPunctTokenizer = require('./tokenizers/regexp_tokenizer').WordPunctTokenizer;
 exports.TreebankWordTokenizer = require('./tokenizers/treebank_word_tokenizer');
@@ -72,8 +80,10 @@ exports.EdgeWeightedDigraph = require('./util/edge_weighted_digraph');
 exports.NGrams = require('./ngrams/ngrams');
 exports.NGramsZH = require('./ngrams/ngrams_zh');
 exports.JaroWinklerDistance = require('./distance/jaro-winkler_distance');
-exports.LevenshteinDistance = require('./distance/levenshtein_distance');
+exports.LevenshteinDistance = require('./distance/levenshtein_distance').LevenshteinDistance;
+exports.DamerauLevenshteinDistance = require('./distance/levenshtein_distance').DamerauLevenshteinDistance;
 exports.DiceCoefficient = require('./distance/dice_coefficient');
+exports.HammingDistance = require('./distance/hamming_distance');
 exports.normalize = require('./normalizers/normalizer').normalize_tokens;
 exports.normalize_ja = require('./normalizers/normalizer_ja').normalize_ja;
 exports.removeDiacritics = require('./normalizers/remove_diacritics');
@@ -86,5 +96,16 @@ exports.RuleSet = require('./brill_pos_tagger/lib/RuleSet');
 exports.RuleTemplates = require('./brill_pos_tagger/lib/RuleTemplates');
 exports.RuleTemplate = require('./brill_pos_tagger/lib/RuleTemplate');
 exports.Corpus = require('./brill_pos_tagger/lib/Corpus');
-
-
+exports.MaxEntClassifier = require('./classifiers/maxent/Classifier');
+exports.Context = require('./classifiers/maxent/Context');
+exports.Feature = require('./classifiers/maxent/Feature');
+exports.FeatureSet = require('./classifiers/maxent/FeatureSet');
+exports.Sample = require('./classifiers/maxent/Sample');
+exports.Element = require('./classifiers/maxent/Element');
+exports.SE_Element = require('./classifiers/maxent/SimpleExample/SE_Element');
+exports.Sentence = require('./brill_pos_tagger/lib/Sentence');
+exports.GISScaler = require('./classifiers/maxent/GISScaler');
+exports.POS_Element = require('./classifiers/maxent/POS/POS_Element');
+exports.ME_Sentence = require('./classifiers/maxent/POS/ME_Sentence');
+exports.ME_Corpus = require('./classifiers/maxent/POS/ME_Corpus');
+exports.SentimentAnalyzer = require('./sentiment/SentimentAnalyzer');

package/lib/natural/inflectors/noun_inflector.js

@@ -26,11 +26,11 @@ var SingularPluralInflector = require('./singular_plural_inflector'),
 
 function attach() {
     var inflector = this;
-    
+
     String.prototype.singularizeNoun = function() {
         return inflector.singularize(this);
     }
-    
+
     String.prototype.pluralizeNoun = function() {
         return inflector.pluralize(this);
     }
@@ -44,9 +44,9 @@ var NounInflector = function() {
         'mackerel', 'mews', 'money', 'news', 'rice', 'rabies', 'salmon', 'series',
         'sheep', 'shrimp', 'species', 'swine', 'trout', 'tuna', 'whiting', 'wildebeest'
     ];
-    
+
     this.customPluralForms = [];
-    this.customSingularForms = [];    
+    this.customSingularForms = [];
     this.singularForms = new FormSet();
     this.pluralForms = new FormSet();
 
@@ -63,15 +63,17 @@ var NounInflector = function() {
     this.addIrregular("goose", "geese");
     this.addIrregular("ephemeris", "ephemerides");
     this.addIrregular("cloth", "clothes");
-    
+    this.addIrregular("hero", "heroes");
+    this.addIrregular("torso", "torsi");
+
     // see if it is possible to unify the creation of both the singular and
     // plural regexes or maybe even just have one list. with a complete list
-    // of rules it may only be possible for some regular forms, but worth a shot    
+    // of rules it may only be possible for some regular forms, but worth a shot
     this.pluralForms.regularForms.push([/y$/i, 'ies']);
     this.pluralForms.regularForms.push([/ife$/i, 'ives']);
-    this.pluralForms.regularForms.push([/(antenn|formul|nebul|vertebr|vit)a$/i, '$1ae']);    
-    this.pluralForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul)us$/i, '$1i']);    
-    this.pluralForms.regularForms.push([/(buffal|tomat|tornad)o$/i, '$1oes']);    
+    this.pluralForms.regularForms.push([/(antenn|formul|nebul|vertebr|vit)a$/i, '$1ae']);
+    this.pluralForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul|alumn|calcul|hippopotam|macrofung|phoet|syllab|troph)us$/i, '$1i']);
+    this.pluralForms.regularForms.push([/(buffal|tomat|tornad)o$/i, '$1oes']);
     this.pluralForms.regularForms.push([/(sis)$/i, 'ses']);
     this.pluralForms.regularForms.push([/(matr|vert|ind|cort)(ix|ex)$/i, '$1ices']);
     this.pluralForms.regularForms.push([/sses$/i, 'sses']);
@@ -82,7 +84,7 @@ var NounInflector = function() {
     this.singularForms.regularForms.push([/([^v])ies$/i, '$1y']);
     this.singularForms.regularForms.push([/ives$/i, 'ife']);
     this.singularForms.regularForms.push([/(antenn|formul|nebul|vertebr|vit)ae$/i, '$1a']);
-    this.singularForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul)(i)$/i, '$1us']);
+    this.singularForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul|alumn|calcul|hippopotam|macrofung|phoet|syllab|troph)(i)$/i, '$1us']);
     this.singularForms.regularForms.push([/(buffal|tomat|tornad)(oes)$/i, '$1o']);
     this.singularForms.regularForms.push([/(analy|naly|synop|parenthe|diagno|the)ses$/i, '$1sis']);
     this.singularForms.regularForms.push([/(vert|ind|cort)(ices)$/i, '$1ex']);
@@ -93,16 +95,16 @@ var NounInflector = function() {
     this.singularForms.regularForms.push([/men$/i, 'man']);
     this.singularForms.regularForms.push([/ss$/i, 'ss']);
     this.singularForms.regularForms.push([/s$/i, '']);
-    
+
     this.pluralize = function (token) {
         return this.ize(token, this.pluralForms, this.customPluralForms);
     };
-    
+
     this.singularize = function(token) {
         return this.ize(token, this.singularForms, this.customSingularForms);
     };
 };
 
 util.inherits(NounInflector, SingularPluralInflector);
-    
+
 module.exports = NounInflector;

package/lib/natural/ngrams/ngrams.js

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2011, Rob Ellis, Chris Umbel
+Copyright (c) 2011, 2018 Rob Ellis, Chris Umbel, Hugo W.L. ter Doest
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -22,32 +22,57 @@ THE SOFTWARE.
 
 var _ = require("underscore")._,
     Tokenizer = require('../tokenizers/regexp_tokenizer').WordTokenizer,
-    tokenizer = new Tokenizer();
+    tokenizer = new Tokenizer(),
+    frequencies = {},
+    nrOfNgrams = 0;
 
 exports.setTokenizer = function(t) {
     if(!_.isFunction(t.tokenize))
         throw new Error('Expected a valid Tokenizer');
     tokenizer = t;
+};
+
+exports.ngrams = function(sequence, n, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, n, startSymbol, endSymbol, stats);
+};
+
+exports.bigrams = function(sequence, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, 2, startSymbol, endSymbol, stats);
+};
+
+exports.trigrams = function(sequence, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, 3, startSymbol, endSymbol, stats);
+};
+
+exports.multrigrams = function(sequence, n, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, n, startSymbol, endSymbol, stats);
+};
+
+// Calculates a key (string) that can be used for a map
+function arrayToKey(arr) {
+  result = "(";
+  arr.forEach(function(x) {
+    result += x + ", ";
+  });
+  result = result.substr(0, result.length - 2) + ")";
+  return result;
+};
+
+// Updates the statistics for the new ngram
+function countNgrams(ngram) {
+  nrOfNgrams++;
+  var key = arrayToKey(ngram);
+  if (!frequencies[key]) {
+    frequencies[key] = 0;
+  }
+  frequencies[key]++;
 }
 
-exports.ngrams = function(sequence, n, startSymbol, endSymbol) {
-    return ngrams(sequence, n, startSymbol, endSymbol);
-}
-
-exports.bigrams = function(sequence, startSymbol, endSymbol) {
-    return ngrams(sequence, 2, startSymbol, endSymbol);
-}
-
-exports.trigrams = function(sequence, startSymbol, endSymbol) {
-    return ngrams(sequence, 3, startSymbol, endSymbol);
-}
-
-exports.multrigrams = function(sequence, n, startSymbol, endSymbol) {
-    return ngrams(sequence, n, startSymbol, endSymbol);
-}
-
-var ngrams = function(sequence, n, startSymbol, endSymbol) {
+// If stats is true, statistics will be returned
+var ngrams = function(sequence, n, startSymbol, endSymbol, stats) {
     var result = [];
+    frequencies = {};
+    nrOfNgrams = 0;
     
     if (!_(sequence).isArray()) {
         sequence = tokenizer.tokenize(sequence);
@@ -66,13 +91,21 @@ var ngrams = function(sequence, n, startSymbol, endSymbol) {
         // Create the left padding
         for(var p = n - 1 ; p > 0 ; p--) {
             // Create a tuple of (p) start symbols and (n - p) words
-            result.push(blanks.slice(0, p).concat(sequence.slice(0, n - p)));
+          var ngram = blanks.slice(0, p).concat(sequence.slice(0, n - p));
+          result.push(ngram);
+          if (stats) {
+            countNgrams(ngram);
+          }
         }
     }
 
     // Build the complete ngrams
     for (var i = 0; i < count; i++) {
-        result.push(sequence.slice(i, i + n));
+        var ngram = sequence.slice(i, i + n);
+        result.push(ngram);
+        if (stats) { 
+          countNgrams(ngram);
+        }
     }
 
     // Check for right padding
@@ -86,10 +119,35 @@ var ngrams = function(sequence, n, startSymbol, endSymbol) {
         // create the right padding
         for(var p = n - 1 ; p > 0 ; p--) {
             // Create a tuple of (p) start symbols and (n - p) words
-            result.push(sequence.slice(sequence.length - p, sequence.length).concat(blanks.slice(0, n - p)));
+          var ngram = sequence.slice(sequence.length - p, sequence.length).concat(blanks.slice(0, n - p));
+          result.push(ngram);
+          if (stats) {
+            countNgrams(ngram);
+          }
         }
     }
     
-    return result;
-}
-
+    if (stats) {
+      
+      // Count frequencies
+      var Nr = {};
+      Object.keys(frequencies).forEach(function(key) {
+        if (!Nr[frequencies[key]]) {
+          Nr[frequencies[key]] = 0;
+        }
+        Nr[frequencies[key]]++;
+      });
+      
+      // Return the ngrams AND statistics
+      return {
+        "ngrams": result,
+        "frequencies": frequencies,
+        "Nr": Nr,
+        "numberOfNgrams": nrOfNgrams
+      };
+      
+    }
+    else { // Do not break existing API of this module 
+      return result;
+    }
+};

package/lib/natural/normalizers/normalizer_sv.js

@@ -0,0 +1,41 @@
+/*
+ Copyright (c) 2017, Dogan Yazar
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+/**
+ * Remove commonly used diacritic marks from a string as these
+ * are not used in a consistent manner. Leave only ä, ö, å.
+ */
+var remove_diacritics = function(text) {
+    text = text.replace('à', 'a');
+    text = text.replace('À', 'A');
+    text = text.replace('á', 'a');
+    text = text.replace('Á', 'A');
+    text = text.replace('è', 'e');
+    text = text.replace('È', 'E');
+    text = text.replace('é', 'e');
+    text = text.replace('É', 'E');
+
+    return text;
+};
+
+// export the relevant stuff.
+exports.remove_diacritics = remove_diacritics;