natural 0.5.6 → 0.6.3

package/lib/natural/distance/jaro-winkler_distance.js

@@ -35,8 +35,6 @@ function distance(s1, s2) {
         return 0;
     }
 
-    s1 = s1.toLowerCase(), s2 = s2.toLowerCase();
-
     var matchWindow = (Math.floor(Math.max(s1.length, s2.length) / 2.0)) - 1;
     var matches1 = new Array(s1.length);
     var matches2 = new Array(s2.length);
@@ -83,6 +81,7 @@ function distance(s1, s2) {
             k++;
         }
         if (s1[i] !== s2[k]) { // if the characters don't match, increase transposition
+          // HtD: t is always less than the number of matches m, because transpositions are a subset of matches
             t++;
         }
         k++; // iterate k index normally
@@ -91,7 +90,8 @@ function distance(s1, s2) {
     // transpositions divided by 2
     t = t / 2.0;
 
-    return ((m / s1.length) + (m / s2.length) + ((m - t) / m)) / 3.0;
+    return ((m / s1.length) + (m / s2.length) + ((m - t) / m)) / 3.0; // HtD: therefore, m - t > 0, and m - t < m
+    // HtD: => return value is between 0 and 1
 }
 
 // Computes the Winkler distance between two string -- intrepreted from:
@@ -99,19 +99,29 @@ function distance(s1, s2) {
 // s1 is the first string to compare
 // s2 is the second string to compare
 // dj is the Jaro Distance (if you've already computed it), leave blank and the method handles it
-function JaroWinklerDistance(s1, s2, dj) {
+// ignoreCase: if true strings are first converted to lower case before comparison
+function JaroWinklerDistance(s1, s2, dj, ignoreCase) {
     if (s1 === s2) {
         return 1;
     } else {
-        var jaro = (typeof(dj) === 'undefined') ? distance(s1,s2) : dj;
+        if (ignoreCase) {
+          s1 = s1.toLowerCase();
+          s2 = s2.toLowerCase();
+        }
+
+        //console.log(news1);
+        //console.log(news2);
+
+        var jaro = (typeof(dj) === 'undefined') ? distance(s1, s2) : dj;
         var p = 0.1; // default scaling factor
         var l = 0 // length of the matching prefix
         while(s1[l] === s2[l] && l < 4) {
             l++;
         }
 
+        // HtD: 1 - jaro >= 0
         return jaro + l * p * (1 - jaro);
     }
 }
-module.exports = JaroWinklerDistance;
 
+module.exports = JaroWinklerDistance;

package/lib/natural/distance/levenshtein_distance.js

@@ -89,7 +89,7 @@ function getMinCostSubstring(distanceMatrix, source, target) {
 *  restricted: boolean (default: false)
 */
 function DamerauLevenshteinDistance(source, target, options) {
-    var damLevOptions = _.assign(
+    var damLevOptions = _.extend(
         { transposition_cost: 1, restricted: false },
         options || {},
         { damerau: true }
@@ -98,7 +98,7 @@ function DamerauLevenshteinDistance(source, target, options) {
 }
 
 function LevenshteinDistance(source, target, options) {
-    var levOptions = _.assign({}, options || {}, { damerau: false });
+    var levOptions = _.extend({}, options || {}, { damerau: false });
     return levenshteinDistance(source, target, levOptions);
 }
 

package/lib/natural/index.js

@@ -33,9 +33,12 @@ exports.PorterStemmerIt = require('./stemmers/porter_stemmer_it');
 exports.PorterStemmerNo = require('./stemmers/porter_stemmer_no');
 exports.PorterStemmerSv = require('./stemmers/porter_stemmer_sv');
 exports.PorterStemmerPt = require('./stemmers/porter_stemmer_pt');
+exports.PorterStemmerNl = require('./stemmers/porter_stemmer_nl');
 exports.LancasterStemmer = require('./stemmers/lancaster_stemmer');
-exports.StemmerFr = require('./stemmers/stemmer_fr');
-exports.StemmerPl = require('./stemmers/stemmer_pl');
+// StemmerFr and StemmerPl are not stemmers. A Polish stemmer is
+// not available, and for French PorterStemmerFr should be used.
+//exports.StemmerFr = require('./stemmers/stemmer_fr');
+//exports.StemmerPl = require('./stemmers/stemmer_pl');
 exports.StemmerJa = require('./stemmers/stemmer_ja');
 exports.StemmerId = require('./stemmers/indonesian/stemmer_id');
 exports.AggressiveTokenizerNl = require('./tokenizers/aggressive_tokenizer_nl');
@@ -48,9 +51,11 @@ exports.AggressiveTokenizerPl = require('./tokenizers/aggressive_tokenizer_pl');
 exports.AggressiveTokenizerPt = require('./tokenizers/aggressive_tokenizer_pt');
 exports.AggressiveTokenizerNo = require('./tokenizers/aggressive_tokenizer_no');
 exports.AggressiveTokenizerSv = require('./tokenizers/aggressive_tokenizer_sv');
+exports.AggressiveTokenizerVi = require('./tokenizers/aggressive_tokenizer_vi');
 exports.AggressiveTokenizer = require('./tokenizers/aggressive_tokenizer');
 exports.CaseTokenizer = require('./tokenizers/tokenizer_case');
 exports.RegexpTokenizer = require('./tokenizers/regexp_tokenizer').RegexpTokenizer;
+exports.OrthographyTokenizer = require('./tokenizers/regexp_tokenizer').OrthographyTokenizer;
 exports.WordTokenizer = require('./tokenizers/regexp_tokenizer').WordTokenizer;
 exports.WordPunctTokenizer = require('./tokenizers/regexp_tokenizer').WordPunctTokenizer;
 exports.TreebankWordTokenizer = require('./tokenizers/treebank_word_tokenizer');
@@ -78,6 +83,7 @@ exports.JaroWinklerDistance = require('./distance/jaro-winkler_distance');
 exports.LevenshteinDistance = require('./distance/levenshtein_distance').LevenshteinDistance;
 exports.DamerauLevenshteinDistance = require('./distance/levenshtein_distance').DamerauLevenshteinDistance;
 exports.DiceCoefficient = require('./distance/dice_coefficient');
+exports.HammingDistance = require('./distance/hamming_distance');
 exports.normalize = require('./normalizers/normalizer').normalize_tokens;
 exports.normalize_ja = require('./normalizers/normalizer_ja').normalize_ja;
 exports.removeDiacritics = require('./normalizers/remove_diacritics');
@@ -90,3 +96,16 @@ exports.RuleSet = require('./brill_pos_tagger/lib/RuleSet');
 exports.RuleTemplates = require('./brill_pos_tagger/lib/RuleTemplates');
 exports.RuleTemplate = require('./brill_pos_tagger/lib/RuleTemplate');
 exports.Corpus = require('./brill_pos_tagger/lib/Corpus');
+exports.MaxEntClassifier = require('./classifiers/maxent/Classifier');
+exports.Context = require('./classifiers/maxent/Context');
+exports.Feature = require('./classifiers/maxent/Feature');
+exports.FeatureSet = require('./classifiers/maxent/FeatureSet');
+exports.Sample = require('./classifiers/maxent/Sample');
+exports.Element = require('./classifiers/maxent/Element');
+exports.SE_Element = require('./classifiers/maxent/SimpleExample/SE_Element');
+exports.Sentence = require('./brill_pos_tagger/lib/Sentence');
+exports.GISScaler = require('./classifiers/maxent/GISScaler');
+exports.POS_Element = require('./classifiers/maxent/POS/POS_Element');
+exports.ME_Sentence = require('./classifiers/maxent/POS/ME_Sentence');
+exports.ME_Corpus = require('./classifiers/maxent/POS/ME_Corpus');
+exports.SentimentAnalyzer = require('./sentiment/SentimentAnalyzer');

package/lib/natural/inflectors/noun_inflector.js

@@ -38,7 +38,7 @@ function attach() {
 
 var NounInflector = function() {
     this.ambiguous = [
-        'bison', 'bream', 'carp', 'chassis', 'cod', 'corps', 'debris', 'deer',
+        'bison', 'bream', 'carp', 'chassis', 'christmas', 'cod', 'corps', 'debris', 'deer',
         'diabetes', 'equipment', 'elk', 'fish', 'flounder', 'gallows', 'graffiti',
         'headquarters', 'herpes', 'highjinks', 'homework', 'information',
         'mackerel', 'mews', 'money', 'news', 'rice', 'rabies', 'salmon', 'series',
@@ -64,14 +64,16 @@ var NounInflector = function() {
     this.addIrregular("ephemeris", "ephemerides");
     this.addIrregular("cloth", "clothes");
     this.addIrregular("hero", "heroes");
+    this.addIrregular("torso", "torsi");
 
     // see if it is possible to unify the creation of both the singular and
     // plural regexes or maybe even just have one list. with a complete list
     // of rules it may only be possible for some regular forms, but worth a shot
+    this.pluralForms.regularForms.push([/([aeiou]y)$/i, '$1s']);
     this.pluralForms.regularForms.push([/y$/i, 'ies']);
     this.pluralForms.regularForms.push([/ife$/i, 'ives']);
     this.pluralForms.regularForms.push([/(antenn|formul|nebul|vertebr|vit)a$/i, '$1ae']);
-    this.pluralForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul)us$/i, '$1i']);
+    this.pluralForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul|alumn|calcul|hippopotam|macrofung|phoet|syllab|troph)us$/i, '$1i']);
     this.pluralForms.regularForms.push([/(buffal|tomat|tornad)o$/i, '$1oes']);
     this.pluralForms.regularForms.push([/(sis)$/i, 'ses']);
     this.pluralForms.regularForms.push([/(matr|vert|ind|cort)(ix|ex)$/i, '$1ices']);
@@ -83,7 +85,7 @@ var NounInflector = function() {
     this.singularForms.regularForms.push([/([^v])ies$/i, '$1y']);
     this.singularForms.regularForms.push([/ives$/i, 'ife']);
     this.singularForms.regularForms.push([/(antenn|formul|nebul|vertebr|vit)ae$/i, '$1a']);
-    this.singularForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul)(i)$/i, '$1us']);
+    this.singularForms.regularForms.push([/(octop|vir|radi|nucle|fung|cact|stimul|alumn|calcul|hippopotam|macrofung|phoet|syllab|troph)(i)$/i, '$1us']);
     this.singularForms.regularForms.push([/(buffal|tomat|tornad)(oes)$/i, '$1o']);
     this.singularForms.regularForms.push([/(analy|naly|synop|parenthe|diagno|the)ses$/i, '$1sis']);
     this.singularForms.regularForms.push([/(vert|ind|cort)(ices)$/i, '$1ex']);

package/lib/natural/ngrams/ngrams.js

@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2011, Rob Ellis, Chris Umbel
+Copyright (c) 2011, 2018 Rob Ellis, Chris Umbel, Hugo W.L. ter Doest
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -22,32 +22,57 @@ THE SOFTWARE.
 
 var _ = require("underscore")._,
     Tokenizer = require('../tokenizers/regexp_tokenizer').WordTokenizer,
-    tokenizer = new Tokenizer();
+    tokenizer = new Tokenizer(),
+    frequencies = {},
+    nrOfNgrams = 0;
 
 exports.setTokenizer = function(t) {
     if(!_.isFunction(t.tokenize))
         throw new Error('Expected a valid Tokenizer');
     tokenizer = t;
+};
+
+exports.ngrams = function(sequence, n, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, n, startSymbol, endSymbol, stats);
+};
+
+exports.bigrams = function(sequence, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, 2, startSymbol, endSymbol, stats);
+};
+
+exports.trigrams = function(sequence, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, 3, startSymbol, endSymbol, stats);
+};
+
+exports.multrigrams = function(sequence, n, startSymbol, endSymbol, stats) {
+    return ngrams(sequence, n, startSymbol, endSymbol, stats);
+};
+
+// Calculates a key (string) that can be used for a map
+function arrayToKey(arr) {
+  result = "(";
+  arr.forEach(function(x) {
+    result += x + ", ";
+  });
+  result = result.substr(0, result.length - 2) + ")";
+  return result;
+};
+
+// Updates the statistics for the new ngram
+function countNgrams(ngram) {
+  nrOfNgrams++;
+  var key = arrayToKey(ngram);
+  if (!frequencies[key]) {
+    frequencies[key] = 0;
+  }
+  frequencies[key]++;
 }
 
-exports.ngrams = function(sequence, n, startSymbol, endSymbol) {
-    return ngrams(sequence, n, startSymbol, endSymbol);
-}
-
-exports.bigrams = function(sequence, startSymbol, endSymbol) {
-    return ngrams(sequence, 2, startSymbol, endSymbol);
-}
-
-exports.trigrams = function(sequence, startSymbol, endSymbol) {
-    return ngrams(sequence, 3, startSymbol, endSymbol);
-}
-
-exports.multrigrams = function(sequence, n, startSymbol, endSymbol) {
-    return ngrams(sequence, n, startSymbol, endSymbol);
-}
-
-var ngrams = function(sequence, n, startSymbol, endSymbol) {
+// If stats is true, statistics will be returned
+var ngrams = function(sequence, n, startSymbol, endSymbol, stats) {
     var result = [];
+    frequencies = {};
+    nrOfNgrams = 0;
     
     if (!_(sequence).isArray()) {
         sequence = tokenizer.tokenize(sequence);
@@ -66,13 +91,21 @@ var ngrams = function(sequence, n, startSymbol, endSymbol) {
         // Create the left padding
         for(var p = n - 1 ; p > 0 ; p--) {
             // Create a tuple of (p) start symbols and (n - p) words
-            result.push(blanks.slice(0, p).concat(sequence.slice(0, n - p)));
+          var ngram = blanks.slice(0, p).concat(sequence.slice(0, n - p));
+          result.push(ngram);
+          if (stats) {
+            countNgrams(ngram);
+          }
         }
     }
 
     // Build the complete ngrams
     for (var i = 0; i < count; i++) {
-        result.push(sequence.slice(i, i + n));
+        var ngram = sequence.slice(i, i + n);
+        result.push(ngram);
+        if (stats) { 
+          countNgrams(ngram);
+        }
     }
 
     // Check for right padding
@@ -86,10 +119,35 @@ var ngrams = function(sequence, n, startSymbol, endSymbol) {
         // create the right padding
         for(var p = n - 1 ; p > 0 ; p--) {
             // Create a tuple of (p) start symbols and (n - p) words
-            result.push(sequence.slice(sequence.length - p, sequence.length).concat(blanks.slice(0, n - p)));
+          var ngram = sequence.slice(sequence.length - p, sequence.length).concat(blanks.slice(0, n - p));
+          result.push(ngram);
+          if (stats) {
+            countNgrams(ngram);
+          }
         }
     }
     
-    return result;
-}
-
+    if (stats) {
+      
+      // Count frequencies
+      var Nr = {};
+      Object.keys(frequencies).forEach(function(key) {
+        if (!Nr[frequencies[key]]) {
+          Nr[frequencies[key]] = 0;
+        }
+        Nr[frequencies[key]]++;
+      });
+      
+      // Return the ngrams AND statistics
+      return {
+        "ngrams": result,
+        "frequencies": frequencies,
+        "Nr": Nr,
+        "numberOfNgrams": nrOfNgrams
+      };
+      
+    }
+    else { // Do not break existing API of this module 
+      return result;
+    }
+};