natural 0.5.4 → 0.6.1

package/lib/natural/classifiers/maxent/POS/POS_Element.js

@@ -0,0 +1,172 @@
+/*
+  Element class for POS tagging
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+var util = require('util');
+var Element = require('../Element');
+var Feature = require('../Feature');
+
+function POS_Element(a, b) {
+   POS_Element.super_.call(this, a, b);
+}
+
+util.inherits(POS_Element, Element);
+
+POS_Element.prototype.generateFeatures = function(featureSet) {
+  var context = this.b.data;
+  var tag = this.a;
+  var token = context.wordWindow["0"];
+
+
+  // Feature for the current word
+  function currentWord(x) {
+    if ((x.b.data.wordWindow["0"] === token) &&
+        (x.a === tag)) {
+        return 1;
+    }
+    return 0;
+  }
+  featureSet.addFeature(new Feature(currentWord, "wordFeature", ["0", token, "0", tag]));
+
+
+  // Feature for previous bigram (previous two tags), positions -2, -1
+  if (context.tagWindow["-2"]) {
+    var prevPrevTag = context.tagWindow["-2"];
+    var prevTag = context.tagWindow["-1"];
+    function prevBigram(x) {
+      if ((x.a === tag) &&
+          (x.b.data.tagWindow["-2"] === prevPrevTag) &&
+          (x.b.data.tagWindow["-1"] === prevTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(prevBigram, "prevBigram", ["0", tag, "-2", prevPrevTag, "-1", prevTag]));
+  }
+
+
+/*
+  // Feature for left bigram, positions -1, 0
+  if (context.tagWindow["-1"]) {
+    var prevTag = context.tagWindow["-1"];
+    function leftBigram(x) {
+      if ((x.b.data.tagWindow["-1"] === prevTag) &&
+          (x.a === tag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(leftBigram, "leftBigram", ["0", tag, "-1", prevTag]));
+  }
+*/
+
+/*
+
+  // Feature for right bigram, positions 0, 1
+  if (context.tagWindow["1"]) {
+    var nextTag = context.tagWindow["1"];
+    function rightBigram(x) {
+      if ((x.a === tag) &&
+          (x.b.data.tagWindow["1"] === nextTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(rightBigram, "rightBigram", ["0", tag, "1", nextTag]));
+  }
+*/
+/*
+  // Feature for next bigram (next two tags), positions 1 and 2
+  if (context.tagWindow["2"]) {
+    var nextTag = context.tagWindow["1"];
+    var nextNextTag = context.tagWindow["2"];
+    function nextBigram(x) {
+      if ((x.a === tag) &&
+          (x.b.data.tagWindow["1"] === nextTag) &&
+          (x.b.data.tagWindow["2"] === nextNextTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(nextBigram, "nextBigram", ["0", tag, "1", nextTag, "2", nextNextTag]));
+  }
+
+  // Feature that looks at the left bigram words
+  if (context.wordWindow["-1"]) {
+    var prevWord = context.wordWindow["-1"];
+    function leftBigramWords(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["0"] === token) &&
+          (x.b.data.wordWindow["-1"] === prevWord)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(leftBigramWords, "leftBigramWords", ["0", tag, "0", token, "-1", prevWord]));
+  }
+
+  // Feature that looks at the right bigram words
+  if (context.wordWindow["1"]) {
+    var nextWord = context.wordWindow["1"];
+    function rightBigramWords(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["0"] === token) &&
+          (x.b.data.wordWindow["1"] === nextWord)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(rightBigramWords, "rightBigramWords", ["0", tag, "0", token, "1", nextWord]));
+  }
+*/
+
+  // Feature that looks at the previous word and its category
+  if (context.wordWindow["-1"]) {
+    var prevWord = context.wordWindow["-1"];
+    var prevTag = context.tagWindow["-1"];
+    function prevWordAndCat(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["-1"] === prevWord) &&
+          (x.b.data.tagWindow["-1"] === prevTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(prevWordAndCat, "prevWordAndCat", ["0", tag, "-1", prevWord, "-1", prevTag]));
+  }
+
+
+/*
+  // Feature that looks at the next word and its category
+  if (context.wordWindow["1"]) {
+    var nextWord = context.wordWindow["1"];
+    var nextTag = context.tagWindow["1"];
+    function nextWordAndCat(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["1"] === nextWord) &&
+          (x.b.data.tagWindow["1"] === nextTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(nextWordAndCat, "nextWordAndCat", ["0", tag, "1", nextWord, "1", nextTag]));
+  }
+*/
+};
+
+module.exports = POS_Element;

package/lib/natural/classifiers/maxent/Sample.js

@@ -0,0 +1,141 @@
+/*
+    Sample space of observed events
+    Copyright (C) 2018 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var Context = require('./Context');
+
+var fs = require('fs');
+
+function Sample(elements) {
+  this.frequencyOfContext = {};
+  this.frequency = {};
+  this.classes = [];
+  if (elements) {
+    this.elements = elements;
+    this.analyse();
+  }
+  else {
+    this.elements = [];
+  }
+}
+
+// Extracts classes and frequencies
+Sample.prototype.analyse = function() {
+  var that = this;
+  this.elements.forEach(function(x) {
+    if (this.classes.indexOf(x.a) === -1) {
+      this.classes.push(x.a);
+    }
+    if (!that.frequencyOfContext[x.b.toString()]) {
+      that.frequencyOfContext[x.b.toString()] = 0;
+    }
+    that.frequencyOfContext[x.b.toString()]++;
+    if (!that.frequency[x.toString()]) {
+      that.frequency[x.toString()] = 0;
+    }
+    that.frequency[x.toString()]++;
+  });
+};
+
+Sample.prototype.addElement = function(x) {
+  this.elements.push(x);
+  // Update frequencies
+  if (!this.frequencyOfContext[x.b.toString()]) {
+    this.frequencyOfContext[x.b.toString()] = 0;
+  }
+  this.frequencyOfContext[x.b.toString()]++;
+  if (!this.frequency[x.toString()]) {
+    this.frequency[x.toString()] = 0;
+  }
+  this.frequency[x.toString()]++;
+  // Update classes
+  if (this.classes.indexOf(x.a) === -1) {
+    this.classes.push(x.a);
+  }
+};
+
+Sample.prototype.observedProbabilityOfContext = function(context) {
+  if (this.frequencyOfContext[context.toString()]) {
+    return this.frequencyOfContext[context.toString()] / this.elements.length;
+  }
+  else {
+    return 0;
+  }
+};
+
+Sample.prototype.observedProbability = function(x) {
+  if (this.frequency[x.toString()]) {
+    return this.frequency[x.toString()] / this.elements.length;
+  }
+  else {
+    return 0;
+  }
+};
+
+Sample.prototype.size = function() {
+  return this.elements.length;
+};
+
+Sample.prototype.getClasses = function() {
+  return this.classes;
+};
+
+Sample.prototype.generateFeatures = function(featureSet) {
+  this.elements.forEach(function(x) {
+    x.generateFeatures(featureSet);
+  });
+};
+
+Sample.prototype.save = function(filename, callback) {
+  var sample = this;
+  var data = JSON.stringify(this, null, 2);
+  fs.writeFile(filename, data, 'utf8', function(err) {
+      //console.log('Sample written')
+      if(callback) {
+          callback(err, err ? null : sample);
+      }
+  });
+};
+
+// Loads a sample from file and revives the right classes, i.e. Sample and
+// Element classes.
+Sample.prototype.load = function(filename, elementClass, callback) {
+  fs.readFile(filename, 'utf8', function(err, data) {
+
+    if(!err) {
+        var sampleData = JSON.parse(data);
+        var sample = new Sample();
+        sampleData.elements.forEach(function(elementData) {
+          var elt = new elementClass(elementData.a, new Context(elementData.b.data));
+          sample.addElement(elt);
+        });
+        if (!sample.frequency || !sample.frequencyOfContext) {
+          sample.analyse();
+        }
+        if (callback) {
+          callback(err, sample);
+        }
+    }
+    else {
+      if (callback) {
+        callback(err);
+      }
+    }
+  });
+};
+
+module.exports = Sample;

package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js

@@ -0,0 +1,49 @@
+/*
+  Simple Example Element class
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var util = require('util');
+
+var Element = require('../Element');
+var Feature = require('../Feature');
+
+function SE_Element(a, b) {
+   SE_Element.super_.call(this, a, b);
+}
+
+util.inherits(SE_Element, Element);
+
+SE_Element.prototype.generateFeatures = function(featureSet) {
+
+  function isZero(x) {
+    if ((x.a === "x") && (x.b.data === "0")) {
+      return 1;
+    }
+    return 0;
+  }
+  featureSet.addFeature(new Feature(isZero, "isZero", ["0"]));
+
+  function isOne(x) {
+    if ((x.a === "y") && (x.b.data === "1")) {
+      return 1;
+    }
+    return 0;
+  }
+  featureSet.addFeature(new Feature(isOne, "isOne", ["1"]));
+};
+
+module.exports = SE_Element;

package/lib/natural/distance/hamming_distance.js

@@ -0,0 +1,52 @@
+/*
+	Copyright (c) 2018, Shane Caldwell, Hugo ter Doest
+
+	Permission is hereby granted, free of charge, to any person obtaining a copy
+	of this software and associated documentation files (the "Software"), to deal
+	in the Software without restriction, including without limitation the rights
+	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+	copies of the Software, and to permit persons to whom the Software is
+	furnished to do so, subject to the following conditions:
+
+	The above copyright notice and this permission notice shall be included in
+	all copies or substantial portions of the Software.
+
+	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+	THE SOFTWARE.
+*/
+
+// Computes the Hamming distance between two string -- intrepreted from:
+// https://en.wikipedia.org/wiki/Hamming_distance
+// s1 is the first string to compare
+// s2 is the second string to compare
+// Strings should have equal length
+function HammingDistance(s1, s2, ignoreCase) {
+	// Return -1 if one of the parameters is not a string
+	if (typeof(s1) != "string" || typeof(s2) != "string") {
+		return -1;
+	}
+	// Return -1 the lengths of the strings differ
+	if (s1.length != s2.length) {
+		return -1;
+	}
+
+	if (ignoreCase) {
+		s1 = s1.toLowerCase();
+		s2 = s2.toLowerCase();
+	}
+
+  var diffs = 0;
+  for (var i = 0; i < s1.length; i++) {
+  	if (s1[i] != s2[i]) {
+  		diffs++;
+		}
+  }
+  return diffs;
+}
+
+module.exports = HammingDistance;

package/lib/natural/distance/jaro-winkler_distance.js

@@ -35,8 +35,6 @@ function distance(s1, s2) {
         return 0;
     }
 
-    s1 = s1.toLowerCase(), s2 = s2.toLowerCase();
-
     var matchWindow = (Math.floor(Math.max(s1.length, s2.length) / 2.0)) - 1;
     var matches1 = new Array(s1.length);
     var matches2 = new Array(s2.length);
@@ -83,6 +81,7 @@ function distance(s1, s2) {
             k++;
         }
         if (s1[i] !== s2[k]) { // if the characters don't match, increase transposition
+          // HtD: t is always less than the number of matches m, because transpositions are a subset of matches
             t++;
         }
         k++; // iterate k index normally
@@ -91,7 +90,8 @@ function distance(s1, s2) {
     // transpositions divided by 2
     t = t / 2.0;
 
-    return ((m / s1.length) + (m / s2.length) + ((m - t) / m)) / 3.0;
+    return ((m / s1.length) + (m / s2.length) + ((m - t) / m)) / 3.0; // HtD: therefore, m - t > 0, and m - t < m
+    // HtD: => return value is between 0 and 1
 }
 
 // Computes the Winkler distance between two string -- intrepreted from:
@@ -99,19 +99,29 @@ function distance(s1, s2) {
 // s1 is the first string to compare
 // s2 is the second string to compare
 // dj is the Jaro Distance (if you've already computed it), leave blank and the method handles it
-function JaroWinklerDistance(s1, s2, dj) {
+// ignoreCase: if true strings are first converted to lower case before comparison
+function JaroWinklerDistance(s1, s2, dj, ignoreCase) {
     if (s1 === s2) {
         return 1;
     } else {
-        var jaro = (typeof(dj) === 'undefined') ? distance(s1,s2) : dj;
+        if (ignoreCase) {
+          s1 = s1.toLowerCase();
+          s2 = s2.toLowerCase();
+        }
+
+        //console.log(news1);
+        //console.log(news2);
+
+        var jaro = (typeof(dj) === 'undefined') ? distance(s1, s2) : dj;
         var p = 0.1; // default scaling factor
         var l = 0 // length of the matching prefix
         while(s1[l] === s2[l] && l < 4) {
             l++;
         }
 
+        // HtD: 1 - jaro >= 0
         return jaro + l * p * (1 - jaro);
     }
 }
-module.exports = JaroWinklerDistance;
 
+module.exports = JaroWinklerDistance;

package/lib/natural/distance/levenshtein_distance.js

@@ -29,40 +29,193 @@ THE SOFTWARE.
  * Algorithm based from Speech and Language Processing - Daniel Jurafsky and James H. Martin.
  */
 
-function LevenshteinDistance (source, target, options) {
-    options = options || {};
+var _ = require('underscore');
+
+// Walk the path back from the matchEnd to the beginning of the match.
+// Do this by traversing the distanceMatrix as you would a linked list,
+// following going from cell child to parent until reach row 0.
+function _getMatchStart(distanceMatrix, matchEnd, sourceLength) {
+  var row = sourceLength;
+  var column = matchEnd;
+  var tmpRow;
+  var tmpColumn;
+
+  // match will be empty string
+  if (matchEnd === 0) { return 0; }
+
+  while(row > 1 && column > 1) {
+   tmpRow = row;
+   tmpColumn = column;
+   row = distanceMatrix[tmpRow][tmpColumn].parentCell.row;
+   column = distanceMatrix[tmpRow][tmpColumn].parentCell.column;
+  }
+
+  return column-1;
+}
+
+function getMinCostSubstring(distanceMatrix, source, target) {
+  var sourceLength = source.length;
+  var targetLength = target.length;
+  var minDistance = sourceLength + targetLength;
+  var matchEnd = targetLength;
+
+  // Find minimum value in last row of the cost matrix. This cell marks the
+  // end of the match string.
+  for (var column = 0; column <= targetLength; column++) {
+    if (minDistance > distanceMatrix[sourceLength][column].cost) {
+      minDistance = distanceMatrix[sourceLength][column].cost;
+      matchEnd = column;
+    }
+  }
+
+  matchStart = _getMatchStart(distanceMatrix, matchEnd, sourceLength);
+  return {substring: target.slice(matchStart, matchEnd), distance: minDistance};
+}
+
+/*
+* Returns the Damerau-Levenshtein distance between strings. Counts the distance
+* between two strings by returning the number of edit operations required to
+* convert `source` into `target`.
+*
+* Valid edit operations are:
+*  - transposition, insertion, deletion, and substitution
+*
+* Options:
+*  insertion_cost: (default: 1)
+*  deletion_cost: number (default: 1)
+*  substitution_cost: number (default: 1)
+*  transposition_cost: number (default: 1)
+*  search: boolean (default: false)
+*  restricted: boolean (default: false)
+*/
+function DamerauLevenshteinDistance(source, target, options) {
+    var damLevOptions = _.extend(
+        { transposition_cost: 1, restricted: false },
+        options || {},
+        { damerau: true }
+    );
+    return levenshteinDistance(source, target, damLevOptions);
+}
+
+function LevenshteinDistance(source, target, options) {
+    var levOptions = _.extend({}, options || {}, { damerau: false });
+    return levenshteinDistance(source, target, levOptions);
+}
+
+
+function levenshteinDistance (source, target, options) {
     if(isNaN(options.insertion_cost)) options.insertion_cost = 1;
     if(isNaN(options.deletion_cost)) options.deletion_cost = 1;
     if(isNaN(options.substitution_cost)) options.substitution_cost = 1;
 
+    if(typeof options.search !== 'boolean') options.search = false;
+
+    var isUnrestrictedDamerau = options.damerau && !options.restricted;
+    var isRestrictedDamerau = options.damerau && options.restricted;
+
+    if (isUnrestrictedDamerau) {
+        var lastRowMap = {};
+    }
+
     var sourceLength = source.length;
     var targetLength = target.length;
-    var distanceMatrix = [[0]];
+    var distanceMatrix = [[{cost: 0}]]; //the root, has no parent cell
 
     for (var row =  1; row <= sourceLength; row++) {
         distanceMatrix[row] = [];
-        distanceMatrix[row][0] = distanceMatrix[row-1][0] + options.deletion_cost;
+        distanceMatrix[row][0] = {cost: distanceMatrix[row-1][0].cost + options.deletion_cost, parentCell: {row: row-1, column: 0}};
     }
 
     for (var column = 1; column <= targetLength; column++) {
-        distanceMatrix[0][column] = distanceMatrix[0][column-1] + options.insertion_cost;
+        if (options.search) {
+          distanceMatrix[0][column] = {cost: 0};
+        } else {
+          distanceMatrix[0][column] = {cost: distanceMatrix[0][column-1].cost + options.insertion_cost, parentCell: {row: 0, column: column-1}};
+        }
     }
 
     for (var row = 1; row <= sourceLength; row++) {
+        if (isUnrestrictedDamerau) {
+            var lastColMatch = null;
+        }
         for (var column = 1; column <= targetLength; column++) {
-            var costToInsert = distanceMatrix[row][column-1] + options.insertion_cost;
-            var costToDelete = distanceMatrix[row-1][column] + options.deletion_cost;
+            var costToInsert = distanceMatrix[row][column-1].cost + options.insertion_cost;
+            var costToDelete = distanceMatrix[row-1][column].cost + options.deletion_cost;
 
             var sourceElement = source[row-1];
             var targetElement = target[column-1];
-            var costToSubstitute = distanceMatrix[row-1][column-1];
+            var costToSubstitute = distanceMatrix[row-1][column-1].cost;
             if (sourceElement !== targetElement) {
                 costToSubstitute = costToSubstitute + options.substitution_cost;
             }
-            distanceMatrix[row][column] = Math.min(costToInsert, costToDelete, costToSubstitute);
+
+            var possibleParents = [
+              {cost: costToInsert, coordinates: {row: row, column: column-1}},
+              {cost: costToDelete, coordinates: {row: row-1, column: column}},
+              {cost: costToSubstitute, coordinates: {row: row-1, column: column-1}}
+            ];
+
+            // We can add damerau to the possibleParents if the current
+            // target-letter has been encountered in our lastRowMap,
+            // and if there exists a previous column in this row where the
+            // row & column letters matched
+            var canDamerau = isUnrestrictedDamerau
+                && row > 1 && column > 1
+                && lastColMatch
+                && targetElement in lastRowMap;
+
+            if (canDamerau) {
+                var lastRowMatch = lastRowMap[targetElement];
+                var costBeforeTransposition =
+                    distanceMatrix[lastRowMatch - 1][lastColMatch - 1].cost;
+                var costToTranspose = costBeforeTransposition
+                    + ((row - lastRowMatch - 1) * options.deletion_cost)
+                    + ((column - lastColMatch - 1) * options.insertion_cost)
+                    + options.transposition_cost;
+                possibleParents.push({
+                    cost: costToTranspose,
+                    coordinates: {
+                        row: lastRowMatch - 1,
+                        column: lastColMatch - 1,
+                    },
+                });
+            }
+            // Source and target chars are 1-indexed in the distanceMatrix so previous
+            // source/target element is (col/row - 2)
+            var canDoRestrictedDamerau = isRestrictedDamerau
+                && row > 1 && column > 1
+                && sourceElement === target[column - 2]
+                && source[row - 2] === targetElement;
+
+            if (canDoRestrictedDamerau) {
+                var costBeforeTransposition = distanceMatrix[row - 2][column - 2].cost;
+                possibleParents.push({
+                    cost: costBeforeTransposition + options.transposition_cost,
+                    coordinates: { row: row - 2, column: column - 2 },
+                });
+            }
+
+            var minCostParent = _.min(possibleParents, function(p) { return p.cost; });
+
+            distanceMatrix[row][column] = {cost: minCostParent.cost, parentCell: minCostParent.coordinates};
+
+            if (isUnrestrictedDamerau) {
+                lastRowMap[sourceElement] = row;
+                if (sourceElement === targetElement) {
+                    lastColMatch = column;
+                }
+            }
         }
     }
-    return distanceMatrix[sourceLength][targetLength];
+
+    if (!options.search) {
+        return distanceMatrix[sourceLength][targetLength].cost;
+    }
+
+    return getMinCostSubstring(distanceMatrix, source, target);
 }
 
-module.exports = LevenshteinDistance;
+module.exports = {
+    LevenshteinDistance: LevenshteinDistance,
+    DamerauLevenshteinDistance: DamerauLevenshteinDistance,
+};