natural 0.5.6 → 0.6.3

package/lib/natural/classifiers/maxent/GISScaler.js

@@ -0,0 +1,148 @@
+/*
+    GISScaler class that finds parameters of features
+    Copyright (C) 2017 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var Element = require('./Element');
+var Feature = require('./Feature');
+var Distribution = require('./Distribution');
+
+// classes is an array of classes
+// features is an array of feature functions
+function GISScaler(featureSet, sample) {
+    this.featureSet = featureSet;
+    this.sample = sample;
+}
+
+// Returns true if a correction feature is necessary
+GISScaler.prototype.calculateMaxSumOfFeatures = function() {
+  var that = this;
+  this.C = 0;
+  this.featureSums = {};
+  var listOfSumValues = [];
+
+  // Since feature functions are derived from the sample
+  // we can use the sample to calculate the max sum
+  // We look at each sample element only once
+  this.sample.elements.forEach(function(x) {
+    if (!that.featureSums[x.toString()]) {
+      var sum = 0;
+      that.featureSet.getFeatures().forEach(function(f) {
+        sum += f.apply(x);
+      });
+      if (sum > that.C) {
+        that.C = sum;
+      }
+      that.featureSums[x.toString()] = sum;
+      listOfSumValues.push(sum);
+    }
+  });
+  //console.log("GISScaler:calculateMaxSumOfFeatures:maxSum is " + this.C);
+
+  // Check if a correction feature is necessary
+  listOfSumValues.sort(function(a, b) {
+    return a - b;
+  });
+  return(listOfSumValues[0] !== listOfSumValues[listOfSumValues.length - 1]);
+};
+
+GISScaler.prototype.addCorrectionFeature = function() {
+  if (this.calculateMaxSumOfFeatures()) {
+    //console.log("GISScaler:addCorrectionFeature:C is " + this.C);
+    var that = this;
+
+    function f(x) {
+      if (that.featureSums[x.toString()] !== undefined) {
+        return that.C - that.featureSums[x.toString()];
+      }
+      return 0;
+    }
+
+    var correctionFeature = new Feature(f, "Correction feature", []);
+    //console.log("GISScaler:addCorrectionFeature:correctionFeature " + JSON.stringify(correctionFeature));
+    this.featureSet.addFeature(correctionFeature);
+  }
+  else {
+    //console.log("Correction feature not needed");
+  }
+};
+
+// This is the Generalised Iterative Scaling algorithm
+// It ends if the improvement in likelihood of the distribution does not
+// improve more than minImprovement or if the maximum number of iterations is
+// reached.
+GISScaler.prototype.run = function(maxIterations, minImprovement) {
+  this.iteration = 0;
+  this.improvement = 0;
+
+  this.addCorrectionFeature();
+  // Build up the distribution p
+  var alpha = new Array(this.featureSet.size());
+  for (var i = 0; i < alpha.length; i++) {
+    alpha[i] = 1;
+  }
+  var p = new Distribution(alpha, this.featureSet, this.sample);
+  //console.log("Distribution created");
+  p.prepare();
+  //console.log("Distribution prepared");
+  var likelihood = p.logLikelihood();
+  var KLDistance = p.KullbackLieblerDistance();
+
+  var newAlpha = new Array(this.featureSet.size());
+  var observedExpectation = 0;
+  var expectationApprox = 0;
+  do {
+    //console.log("Iteration " + this.iteration + " - Log likelihood of sample: " + likelihood + " - Entropy: " + p.entropy());
+    for (var i = 0; i < this.featureSet.size(); i++) {
+      observedExpectation = this.featureSet.getFeatures()[i].observedExpectation(this.sample);
+      expectationApprox = this.featureSet.getFeatures()[i].expectationApprox(p, this.sample);
+      //console.log("Iteration " + this.iteration + " - Feature " + i);
+      newAlpha[i] = p.alpha[i] * Math.pow(observedExpectation / expectationApprox, 1 / this.C);
+
+      //console.log("GISScaler.run: old alpha[" + i + "]: " + p.alpha[i]);
+      //console.log("GISScaler.run: new alpha[" + i + "]: " + newAlpha[i]);
+    }
+
+    // Make the newly calculated parameters current parameters
+    newAlpha.forEach(function(newAlpha_j, j) {
+      p.alpha[j] = newAlpha_j;
+    });
+    // Recalculate a priori and a posteriori probabilities
+    p.prepare();
+
+    this.iteration++;
+    var newLikelihood = p.logLikelihood();
+    var newKLDistance = p.KullbackLieblerDistance();
+    this.improvement = KLDistance - newKLDistance;
+    //console.log("Iteration " + this.iteration + " - Old likelihood: " + likelihood + " - New likelihood: " + newLikelihood);
+    //console.log("Iteration " + this.iteration + " - Old KL: " + KLDistance + " - New KL: " + newKLDistance);
+
+    likelihood = newLikelihood;
+    KLDistance = newKLDistance;
+  } while ((this.iteration < maxIterations) && (this.improvement > minImprovement));
+  //} while (iteration < maxIterations);
+  /*
+  var that = this;
+  this.featureSet.getFeatures().forEach(function(f, j) {
+    console.log("Observed expectation of feature " + j + ": " + f.observedExpectation(that.sample) +
+      " - Expection of feature according to p: " + f.expectationApprox(p, that.sample));
+  });
+  */
+
+  return p;
+};
+
+module.exports = GISScaler;

package/lib/natural/classifiers/maxent/POS/ME_Corpus.js

@@ -0,0 +1,56 @@
+/*
+  Corpus class specific for MaxEnt modeling
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var util = require('util');
+var Sample = require('../Sample');
+var Corpus = require('../../../brill_pos_tagger/lib/Corpus');
+
+function ME_Corpus(data, BROWN, SentenceClass) {
+   ME_Corpus.super_.call(this, data, BROWN, SentenceClass);
+}
+
+util.inherits(ME_Corpus, Corpus);
+
+ME_Corpus.prototype.generateSample = function() {
+  var sample = new Sample([]);
+  this.sentences.forEach(function(sentence) {
+    sentence.generateSampleElements(sample);
+  });
+  return sample;
+};
+
+// Splits the corpus in a training and testing set.
+// percentageTrain is the size of the training corpus in percent
+// Returns an array with two elements: training corpus, testing corpus
+ME_Corpus.prototype.splitInTrainAndTest = function(percentageTrain) {
+  var corpusTrain = new ME_Corpus();
+  var corpusTest = new ME_Corpus();
+
+  var p = percentageTrain / 100;
+  this.sentences.forEach(function(sentence, i) {
+    if (Math.random() < p) {
+      corpusTrain.sentences.push(sentence);
+    }
+    else {
+      corpusTest.sentences.push(sentence);
+    }
+  });
+  return [corpusTrain, corpusTest];
+};
+
+module.exports = ME_Corpus;

package/lib/natural/classifiers/maxent/POS/ME_Sentence.js

@@ -0,0 +1,73 @@
+/*
+  Sentence class specific for MaxEnt modeling
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var util = require('util');
+var Context = require('../Context');
+var Sentence = require('../../../brill_pos_tagger/lib/Sentence');
+var Element = require('./POS_Element');
+
+function ME_Sentence() {
+   ME_Sentence.super_.call(this);
+}
+
+util.inherits(ME_Sentence, Sentence);
+
+ME_Sentence.prototype.generateSampleElements = function(sample) {
+  var sentence = this.taggedWords;
+  sentence.forEach(function(token, index) {
+    var x = new Element(
+      token.tag,
+      new Context({
+        wordWindow: {},
+        tagWindow: {}
+      })
+    );
+
+    // Current word and tag
+    x.b.data.wordWindow["0"] = token.token;
+    x.b.data.tagWindow["0"] = sentence[index].tag;
+
+    // Previous bigram
+    if (index > 1) {
+      x.b.data.tagWindow["-2"] = sentence[index - 2].tag;
+      x.b.data.wordWindow["-2"] = sentence[index - 2].token;
+    }
+
+    // Left bigram
+    if (index > 0) {
+      x.b.data.tagWindow["-1"] = sentence[index - 1].tag;
+      x.b.data.wordWindow["-1"] = sentence[index - 1].token;
+    }
+
+    // Right bigram
+    if (index < sentence.length - 1) {
+      x.b.data.tagWindow["1"] = sentence[index + 1].tag;
+      x.b.data.wordWindow["1"] = sentence[index + 1].token;
+    }
+
+    // Next bigram
+    if (index < sentence.length - 2) {
+      x.b.data.tagWindow["2"] = sentence[index + 2].tag;
+      x.b.data.wordWindow["2"] = sentence[index + 2].token;
+    }
+
+    sample.addElement(x);
+  });
+};
+
+module.exports = ME_Sentence;

package/lib/natural/classifiers/maxent/POS/POS_Element.js

@@ -0,0 +1,172 @@
+/*
+  Element class for POS tagging
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+var util = require('util');
+var Element = require('../Element');
+var Feature = require('../Feature');
+
+function POS_Element(a, b) {
+   POS_Element.super_.call(this, a, b);
+}
+
+util.inherits(POS_Element, Element);
+
+POS_Element.prototype.generateFeatures = function(featureSet) {
+  var context = this.b.data;
+  var tag = this.a;
+  var token = context.wordWindow["0"];
+
+
+  // Feature for the current word
+  function currentWord(x) {
+    if ((x.b.data.wordWindow["0"] === token) &&
+        (x.a === tag)) {
+        return 1;
+    }
+    return 0;
+  }
+  featureSet.addFeature(new Feature(currentWord, "wordFeature", ["0", token, "0", tag]));
+
+
+  // Feature for previous bigram (previous two tags), positions -2, -1
+  if (context.tagWindow["-2"]) {
+    var prevPrevTag = context.tagWindow["-2"];
+    var prevTag = context.tagWindow["-1"];
+    function prevBigram(x) {
+      if ((x.a === tag) &&
+          (x.b.data.tagWindow["-2"] === prevPrevTag) &&
+          (x.b.data.tagWindow["-1"] === prevTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(prevBigram, "prevBigram", ["0", tag, "-2", prevPrevTag, "-1", prevTag]));
+  }
+
+
+/*
+  // Feature for left bigram, positions -1, 0
+  if (context.tagWindow["-1"]) {
+    var prevTag = context.tagWindow["-1"];
+    function leftBigram(x) {
+      if ((x.b.data.tagWindow["-1"] === prevTag) &&
+          (x.a === tag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(leftBigram, "leftBigram", ["0", tag, "-1", prevTag]));
+  }
+*/
+
+/*
+
+  // Feature for right bigram, positions 0, 1
+  if (context.tagWindow["1"]) {
+    var nextTag = context.tagWindow["1"];
+    function rightBigram(x) {
+      if ((x.a === tag) &&
+          (x.b.data.tagWindow["1"] === nextTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(rightBigram, "rightBigram", ["0", tag, "1", nextTag]));
+  }
+*/
+/*
+  // Feature for next bigram (next two tags), positions 1 and 2
+  if (context.tagWindow["2"]) {
+    var nextTag = context.tagWindow["1"];
+    var nextNextTag = context.tagWindow["2"];
+    function nextBigram(x) {
+      if ((x.a === tag) &&
+          (x.b.data.tagWindow["1"] === nextTag) &&
+          (x.b.data.tagWindow["2"] === nextNextTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(nextBigram, "nextBigram", ["0", tag, "1", nextTag, "2", nextNextTag]));
+  }
+
+  // Feature that looks at the left bigram words
+  if (context.wordWindow["-1"]) {
+    var prevWord = context.wordWindow["-1"];
+    function leftBigramWords(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["0"] === token) &&
+          (x.b.data.wordWindow["-1"] === prevWord)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(leftBigramWords, "leftBigramWords", ["0", tag, "0", token, "-1", prevWord]));
+  }
+
+  // Feature that looks at the right bigram words
+  if (context.wordWindow["1"]) {
+    var nextWord = context.wordWindow["1"];
+    function rightBigramWords(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["0"] === token) &&
+          (x.b.data.wordWindow["1"] === nextWord)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(rightBigramWords, "rightBigramWords", ["0", tag, "0", token, "1", nextWord]));
+  }
+*/
+
+  // Feature that looks at the previous word and its category
+  if (context.wordWindow["-1"]) {
+    var prevWord = context.wordWindow["-1"];
+    var prevTag = context.tagWindow["-1"];
+    function prevWordAndCat(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["-1"] === prevWord) &&
+          (x.b.data.tagWindow["-1"] === prevTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(prevWordAndCat, "prevWordAndCat", ["0", tag, "-1", prevWord, "-1", prevTag]));
+  }
+
+
+/*
+  // Feature that looks at the next word and its category
+  if (context.wordWindow["1"]) {
+    var nextWord = context.wordWindow["1"];
+    var nextTag = context.tagWindow["1"];
+    function nextWordAndCat(x) {
+      if ((x.a === tag) &&
+          (x.b.data.wordWindow["1"] === nextWord) &&
+          (x.b.data.tagWindow["1"] === nextTag)) {
+          return 1;
+        }
+      return 0;
+    }
+    featureSet.addFeature(new Feature(nextWordAndCat, "nextWordAndCat", ["0", tag, "1", nextWord, "1", nextTag]));
+  }
+*/
+};
+
+module.exports = POS_Element;

package/lib/natural/classifiers/maxent/Sample.js

@@ -0,0 +1,141 @@
+/*
+    Sample space of observed events
+    Copyright (C) 2018 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var Context = require('./Context');
+
+var fs = require('fs');
+
+function Sample(elements) {
+  this.frequencyOfContext = {};
+  this.frequency = {};
+  this.classes = [];
+  if (elements) {
+    this.elements = elements;
+    this.analyse();
+  }
+  else {
+    this.elements = [];
+  }
+}
+
+// Extracts classes and frequencies
+Sample.prototype.analyse = function() {
+  var that = this;
+  this.elements.forEach(function(x) {
+    if (this.classes.indexOf(x.a) === -1) {
+      this.classes.push(x.a);
+    }
+    if (!that.frequencyOfContext[x.b.toString()]) {
+      that.frequencyOfContext[x.b.toString()] = 0;
+    }
+    that.frequencyOfContext[x.b.toString()]++;
+    if (!that.frequency[x.toString()]) {
+      that.frequency[x.toString()] = 0;
+    }
+    that.frequency[x.toString()]++;
+  });
+};
+
+Sample.prototype.addElement = function(x) {
+  this.elements.push(x);
+  // Update frequencies
+  if (!this.frequencyOfContext[x.b.toString()]) {
+    this.frequencyOfContext[x.b.toString()] = 0;
+  }
+  this.frequencyOfContext[x.b.toString()]++;
+  if (!this.frequency[x.toString()]) {
+    this.frequency[x.toString()] = 0;
+  }
+  this.frequency[x.toString()]++;
+  // Update classes
+  if (this.classes.indexOf(x.a) === -1) {
+    this.classes.push(x.a);
+  }
+};
+
+Sample.prototype.observedProbabilityOfContext = function(context) {
+  if (this.frequencyOfContext[context.toString()]) {
+    return this.frequencyOfContext[context.toString()] / this.elements.length;
+  }
+  else {
+    return 0;
+  }
+};
+
+Sample.prototype.observedProbability = function(x) {
+  if (this.frequency[x.toString()]) {
+    return this.frequency[x.toString()] / this.elements.length;
+  }
+  else {
+    return 0;
+  }
+};
+
+Sample.prototype.size = function() {
+  return this.elements.length;
+};
+
+Sample.prototype.getClasses = function() {
+  return this.classes;
+};
+
+Sample.prototype.generateFeatures = function(featureSet) {
+  this.elements.forEach(function(x) {
+    x.generateFeatures(featureSet);
+  });
+};
+
+Sample.prototype.save = function(filename, callback) {
+  var sample = this;
+  var data = JSON.stringify(this, null, 2);
+  fs.writeFile(filename, data, 'utf8', function(err) {
+      //console.log('Sample written')
+      if(callback) {
+          callback(err, err ? null : sample);
+      }
+  });
+};
+
+// Loads a sample from file and revives the right classes, i.e. Sample and
+// Element classes.
+Sample.prototype.load = function(filename, elementClass, callback) {
+  fs.readFile(filename, 'utf8', function(err, data) {
+
+    if(!err) {
+        var sampleData = JSON.parse(data);
+        var sample = new Sample();
+        sampleData.elements.forEach(function(elementData) {
+          var elt = new elementClass(elementData.a, new Context(elementData.b.data));
+          sample.addElement(elt);
+        });
+        if (!sample.frequency || !sample.frequencyOfContext) {
+          sample.analyse();
+        }
+        if (callback) {
+          callback(err, sample);
+        }
+    }
+    else {
+      if (callback) {
+        callback(err);
+      }
+    }
+  });
+};
+
+module.exports = Sample;

package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js

@@ -0,0 +1,49 @@
+/*
+  Simple Example Element class
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var util = require('util');
+
+var Element = require('../Element');
+var Feature = require('../Feature');
+
+function SE_Element(a, b) {
+   SE_Element.super_.call(this, a, b);
+}
+
+util.inherits(SE_Element, Element);
+
+SE_Element.prototype.generateFeatures = function(featureSet) {
+
+  function isZero(x) {
+    if ((x.a === "x") && (x.b.data === "0")) {
+      return 1;
+    }
+    return 0;
+  }
+  featureSet.addFeature(new Feature(isZero, "isZero", ["0"]));
+
+  function isOne(x) {
+    if ((x.a === "y") && (x.b.data === "1")) {
+      return 1;
+    }
+    return 0;
+  }
+  featureSet.addFeature(new Feature(isOne, "isOne", ["1"]));
+};
+
+module.exports = SE_Element;

package/lib/natural/distance/hamming_distance.js

@@ -0,0 +1,52 @@
+/*
+	Copyright (c) 2018, Shane Caldwell, Hugo ter Doest
+
+	Permission is hereby granted, free of charge, to any person obtaining a copy
+	of this software and associated documentation files (the "Software"), to deal
+	in the Software without restriction, including without limitation the rights
+	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+	copies of the Software, and to permit persons to whom the Software is
+	furnished to do so, subject to the following conditions:
+
+	The above copyright notice and this permission notice shall be included in
+	all copies or substantial portions of the Software.
+
+	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+	THE SOFTWARE.
+*/
+
+// Computes the Hamming distance between two string -- intrepreted from:
+// https://en.wikipedia.org/wiki/Hamming_distance
+// s1 is the first string to compare
+// s2 is the second string to compare
+// Strings should have equal length
+function HammingDistance(s1, s2, ignoreCase) {
+	// Return -1 if one of the parameters is not a string
+	if (typeof(s1) != "string" || typeof(s2) != "string") {
+		return -1;
+	}
+	// Return -1 the lengths of the strings differ
+	if (s1.length != s2.length) {
+		return -1;
+	}
+
+	if (ignoreCase) {
+		s1 = s1.toLowerCase();
+		s2 = s2.toLowerCase();
+	}
+
+  var diffs = 0;
+  for (var i = 0; i < s1.length; i++) {
+  	if (s1[i] != s2[i]) {
+  		diffs++;
+		}
+  }
+  return diffs;
+}
+
+module.exports = HammingDistance;