natural 0.5.4 → 0.6.1

package/lib/natural/classifiers/maxent/Context.js

@@ -0,0 +1,33 @@
+/*
+  Context class
+  Copyright (C) 2017 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var stringify = require('json-stable-stringify');
+
+function Context(data) {
+  this.data = data;
+}
+
+// Create a predictable key string for looking up in a hash
+Context.prototype.toString = function() {
+  if (!this.key) {
+    this.key = stringify(this.data);
+  }
+  return this.key;
+};
+
+module.exports = Context;

package/lib/natural/classifiers/maxent/Distribution.js

@@ -0,0 +1,145 @@
+/*
+    Distribution class for probability distributions
+    Copyright (C) 2017 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var Element = require('./Element');
+
+function Distribution(alpha, featureSet, sample) {
+  this.alpha = alpha;
+  this.featureSet = featureSet;
+  this.sample = sample;
+}
+
+// Returns the distribution as a string that can be stored for later usage
+Distribution.prototype.toString = function() {
+
+}
+
+Distribution.prototype.weight = function(x) {
+  var product = 1;
+  var that = this;
+
+  this.alpha.forEach(function(alpha_j, j) {
+    product *= Math.pow(alpha_j, that.featureSet.getFeatures()[j].apply(x));
+  });
+  return product;
+};
+
+Distribution.prototype.calculateAPriori = function(x) {
+  if (!this.aPriorisBeforeNormalisation[x.toString()]) {
+    this.aPriorisBeforeNormalisation[x.toString()] = this.weight(x);
+  }
+  return this.aPriorisBeforeNormalisation[x.toString()];// / this.aPrioriNormalisationConstant;
+};
+
+// Memoize a priori probabilities of sample elements
+Distribution.prototype.prepareWeights = function() {
+  this.aPriorisBeforeNormalisation = {};
+  this.aPrioriNormalisationConstant = 0;
+  var sum = 0;
+  var that = this;
+  this.sample.elements.forEach(function(x) {
+    that.aPriorisBeforeNormalisation[x.toString()] = that.weight(x);
+    sum += that.aPriorisBeforeNormalisation[x.toString()];
+  });
+  this.aPrioriNormalisationConstant = sum;
+};
+
+Distribution.prototype.calculateAPosteriori = function(x) {
+  if (!this.aPriorisBeforeNormalisation[x.toString()]) {
+    this.aPriorisBeforeNormalisation[x.toString()] = this.weight(x);
+  }
+  if (!this.aPosterioriNormalisationConstants[x.b.toString()]) {
+    this.aPosterioriNormalisationConstants[x.b.toString()] = this.aPosterioriNormalisation(x.b);
+  }
+  return this.aPriorisBeforeNormalisation[x] / this.aPosterioriNormalisationConstants[x.b.toString()];
+};
+
+Distribution.prototype.aPosterioriNormalisation = function(b) {
+  var sum = 0;
+
+  var that = this;
+  this.sample.getClasses().forEach(function(a) {
+    sum += that.weight(new Element(a, b));
+  });
+
+  return(sum);
+};
+
+// Memoize a posteriori probabilities of sample elements
+Distribution.prototype.prepareAPosterioris = function() {
+  this.aPosterioriNormalisationConstants = {};
+
+  var contextSeen = {};
+  var that = this;
+  this.sample.elements.forEach(function(sampleElement) {
+    var context = sampleElement.b;
+    if (!contextSeen[context]) {
+      contextSeen[context] = true;
+      that.aPosterioriNormalisationConstants[context] =
+        that.aPosterioriNormalisation(context);
+    }
+  });
+};
+
+// Memoize all probabilities of sample elements
+Distribution.prototype.prepare = function() {
+  this.prepareWeights();
+  //console.log("Weights prepared");
+  this.prepareAPosterioris();
+};
+
+// Relative entropy between observered distribution and derived distribution
+Distribution.prototype.KullbackLieblerDistance = function() {
+  var sum = 0;
+  var that = this;
+  this.sample.elements.forEach(function(x) {
+    sum += that.sample.observedProbability(x) * Math.log(that.sample.observedProbability(x) / that.calculateAPriori(x));
+  });
+  return sum;
+};
+
+Distribution.prototype.logLikelihood = function() {
+  var sum = 0;
+  var that = this;
+  this.sample.elements.forEach(function(x) {
+    sum += that.sample.observedProbability(x) * Math.log(that.calculateAPriori(x));
+  });
+  return sum;
+};
+
+Distribution.prototype.entropy = function() {
+  var sum = 0;
+  var that = this;
+  this.sample.elements.forEach(function(x) {
+    var p = that.calculateAPriori(x);
+    sum += p * Math.log(p);
+  });
+  return sum;
+};
+
+Distribution.prototype.checkSum = function() {
+  var sum = 0;
+  var that = this;
+  this.sample.elements.forEach(function(x) {
+      sum += that.calculateAPriori(x);
+  });
+  //console.log("Distribution.checkSum is " + sum);
+  return sum;
+}
+
+module.exports = Distribution;

package/lib/natural/classifiers/maxent/Element.js

@@ -0,0 +1,34 @@
+/*
+    Element class for elements in the event space
+    Copyright (C) 2017 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var Feature = require('./Feature');
+
+// a is class and b is context
+function Element(a, b) {
+  this.a = a;
+  this.b = b;
+}
+
+Element.prototype.toString = function() {
+  if (!this.key) {
+    this.key =  this.a + this.b.toString();
+  }
+  return this.key;
+};
+
+module.exports = Element;

package/lib/natural/classifiers/maxent/Feature.js

@@ -0,0 +1,88 @@
+/*
+    Feature class for features that fire (or don't) on combinations of context
+    and class
+    Copyright (C) 2017 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+function Feature(f, name, parameters) {
+  this.evaluate = f;
+  this.name = name;
+  this.parameters = parameters;
+
+  var tmp = "";
+  parameters.forEach(function(par) {
+    tmp += par + "|";
+  });
+  this.parametersKey = tmp.substr(0, tmp.length - 1);
+}
+
+Feature.prototype.apply = function(x) {
+  return this.evaluate(x);
+};
+
+Feature.prototype.expectationApprox = function(p, sample) {
+  var totalSum = 0;
+  var that = this;
+  var sum = 0;
+  var seen = {};
+  var A = sample.getClasses();
+  sample.elements.forEach(function(sampleElement) {
+    var b_i = sampleElement.b;
+
+    if (!seen[b_i.toString()]) {
+      seen[b_i.toString()] = true;
+      var Element = require('./Element');
+
+      A.forEach(function(a) {
+        var x = new Element(a, b_i);
+        sum += sample.observedProbabilityOfContext(b_i) * p.calculateAPosteriori(x) * that.apply(x);
+      });
+    }
+  });
+  return sum;
+};
+
+// Diect calculation of expected value of this feature according to distribution p
+// In real-life applications with a lot of features this is not tractable
+Feature.prototype.expectation = function(p, A, B) {
+  var sum = 0;
+  var that = this;
+  A.forEach(function(a) {
+    B.forEach(function(b) {
+        var x = new Element(a, b);
+        sum += (p.calculateAPriori(x) * that.apply(x));
+    });
+  });
+  return sum;
+};
+
+// Observed expectation of this feature in the sample
+Feature.prototype.observedExpectation = function(sample) {
+  if (this.observedExpect) {
+    return this.observedExpect;
+  }
+  var N = sample.size();
+  var sum = 0;
+  var that = this;
+  sample.elements.forEach(function(x) {
+    sum += that.apply(x);
+  });
+  this.observedExpect = sum / N;
+  return this.observedExpect;
+};
+
+module.exports = Feature;

package/lib/natural/classifiers/maxent/FeatureSet.js

@@ -0,0 +1,67 @@
+/*
+  Feature set class for administrating a set of unique feature
+  Copyright (C) 2017 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+function FeatureSet() {
+  this.features = [];
+  this.map = {};
+}
+
+// Returns true if the feature did not exist and was added
+FeatureSet.prototype.addFeature = function(feature) {
+  if (!this.featureExists(feature)) {
+    this.map[feature.name +" | " + feature.parametersKey] = true;
+    this.features.push(feature);
+    //console.log("FeatureSet.addFeature: feature added: " + feature.name + " - " + feature.parametersKey);
+    return true;
+  }
+  else {
+    return false;
+  }
+};
+
+FeatureSet.prototype.featureExists = function(feature) {
+  if (this.map[feature.name +" | " + feature.parametersKey]) {
+    //console.log("FeatureSet.featureExists: feature already exists: " +
+    //  feature.name + " - " + feature.parameters);
+    return true;
+  }
+  else {
+    return false;
+  }
+};
+
+// Returns an array of features
+// If the available array this.features is up to date it is returned immediately
+FeatureSet.prototype.getFeatures = function() {
+  return this.features;
+};
+
+FeatureSet.prototype.size = function() {
+  return this.features.length;
+};
+
+FeatureSet.prototype.prettyPrint = function() {
+  var s = "";
+  Object.keys(this.map).forEach(function(key) {
+    s += key + "\n";
+  });
+  return s;
+};
+
+module.exports = FeatureSet;

package/lib/natural/classifiers/maxent/GISScaler.js

@@ -0,0 +1,148 @@
+/*
+    GISScaler class that finds parameters of features
+    Copyright (C) 2017 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var Element = require('./Element');
+var Feature = require('./Feature');
+var Distribution = require('./Distribution');
+
+// classes is an array of classes
+// features is an array of feature functions
+function GISScaler(featureSet, sample) {
+    this.featureSet = featureSet;
+    this.sample = sample;
+}
+
+// Returns true if a correction feature is necessary
+GISScaler.prototype.calculateMaxSumOfFeatures = function() {
+  var that = this;
+  this.C = 0;
+  this.featureSums = {};
+  var listOfSumValues = [];
+
+  // Since feature functions are derived from the sample
+  // we can use the sample to calculate the max sum
+  // We look at each sample element only once
+  this.sample.elements.forEach(function(x) {
+    if (!that.featureSums[x.toString()]) {
+      var sum = 0;
+      that.featureSet.getFeatures().forEach(function(f) {
+        sum += f.apply(x);
+      });
+      if (sum > that.C) {
+        that.C = sum;
+      }
+      that.featureSums[x.toString()] = sum;
+      listOfSumValues.push(sum);
+    }
+  });
+  //console.log("GISScaler:calculateMaxSumOfFeatures:maxSum is " + this.C);
+
+  // Check if a correction feature is necessary
+  listOfSumValues.sort(function(a, b) {
+    return a - b;
+  });
+  return(listOfSumValues[0] !== listOfSumValues[listOfSumValues.length - 1]);
+};
+
+GISScaler.prototype.addCorrectionFeature = function() {
+  if (this.calculateMaxSumOfFeatures()) {
+    //console.log("GISScaler:addCorrectionFeature:C is " + this.C);
+    var that = this;
+
+    function f(x) {
+      if (that.featureSums[x.toString()] !== undefined) {
+        return that.C - that.featureSums[x.toString()];
+      }
+      return 0;
+    }
+
+    var correctionFeature = new Feature(f, "Correction feature", []);
+    //console.log("GISScaler:addCorrectionFeature:correctionFeature " + JSON.stringify(correctionFeature));
+    this.featureSet.addFeature(correctionFeature);
+  }
+  else {
+    //console.log("Correction feature not needed");
+  }
+};
+
+// This is the Generalised Iterative Scaling algorithm
+// It ends if the improvement in likelihood of the distribution does not
+// improve more than minImprovement or if the maximum number of iterations is
+// reached.
+GISScaler.prototype.run = function(maxIterations, minImprovement) {
+  this.iteration = 0;
+  this.improvement = 0;
+
+  this.addCorrectionFeature();
+  // Build up the distribution p
+  var alpha = new Array(this.featureSet.size());
+  for (var i = 0; i < alpha.length; i++) {
+    alpha[i] = 1;
+  }
+  var p = new Distribution(alpha, this.featureSet, this.sample);
+  //console.log("Distribution created");
+  p.prepare();
+  //console.log("Distribution prepared");
+  var likelihood = p.logLikelihood();
+  var KLDistance = p.KullbackLieblerDistance();
+
+  var newAlpha = new Array(this.featureSet.size());
+  var observedExpectation = 0;
+  var expectationApprox = 0;
+  do {
+    //console.log("Iteration " + this.iteration + " - Log likelihood of sample: " + likelihood + " - Entropy: " + p.entropy());
+    for (var i = 0; i < this.featureSet.size(); i++) {
+      observedExpectation = this.featureSet.getFeatures()[i].observedExpectation(this.sample);
+      expectationApprox = this.featureSet.getFeatures()[i].expectationApprox(p, this.sample);
+      //console.log("Iteration " + this.iteration + " - Feature " + i);
+      newAlpha[i] = p.alpha[i] * Math.pow(observedExpectation / expectationApprox, 1 / this.C);
+
+      //console.log("GISScaler.run: old alpha[" + i + "]: " + p.alpha[i]);
+      //console.log("GISScaler.run: new alpha[" + i + "]: " + newAlpha[i]);
+    }
+
+    // Make the newly calculated parameters current parameters
+    newAlpha.forEach(function(newAlpha_j, j) {
+      p.alpha[j] = newAlpha_j;
+    });
+    // Recalculate a priori and a posteriori probabilities
+    p.prepare();
+
+    this.iteration++;
+    var newLikelihood = p.logLikelihood();
+    var newKLDistance = p.KullbackLieblerDistance();
+    this.improvement = KLDistance - newKLDistance;
+    //console.log("Iteration " + this.iteration + " - Old likelihood: " + likelihood + " - New likelihood: " + newLikelihood);
+    //console.log("Iteration " + this.iteration + " - Old KL: " + KLDistance + " - New KL: " + newKLDistance);
+
+    likelihood = newLikelihood;
+    KLDistance = newKLDistance;
+  } while ((this.iteration < maxIterations) && (this.improvement > minImprovement));
+  //} while (iteration < maxIterations);
+  /*
+  var that = this;
+  this.featureSet.getFeatures().forEach(function(f, j) {
+    console.log("Observed expectation of feature " + j + ": " + f.observedExpectation(that.sample) +
+      " - Expection of feature according to p: " + f.expectationApprox(p, that.sample));
+  });
+  */
+
+  return p;
+};
+
+module.exports = GISScaler;

package/lib/natural/classifiers/maxent/POS/ME_Corpus.js

@@ -0,0 +1,56 @@
+/*
+  Corpus class specific for MaxEnt modeling
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var util = require('util');
+var Sample = require('../Sample');
+var Corpus = require('../../../brill_pos_tagger/lib/Corpus');
+
+function ME_Corpus(data, BROWN, SentenceClass) {
+   ME_Corpus.super_.call(this, data, BROWN, SentenceClass);
+}
+
+util.inherits(ME_Corpus, Corpus);
+
+ME_Corpus.prototype.generateSample = function() {
+  var sample = new Sample([]);
+  this.sentences.forEach(function(sentence) {
+    sentence.generateSampleElements(sample);
+  });
+  return sample;
+};
+
+// Splits the corpus in a training and testing set.
+// percentageTrain is the size of the training corpus in percent
+// Returns an array with two elements: training corpus, testing corpus
+ME_Corpus.prototype.splitInTrainAndTest = function(percentageTrain) {
+  var corpusTrain = new ME_Corpus();
+  var corpusTest = new ME_Corpus();
+
+  var p = percentageTrain / 100;
+  this.sentences.forEach(function(sentence, i) {
+    if (Math.random() < p) {
+      corpusTrain.sentences.push(sentence);
+    }
+    else {
+      corpusTest.sentences.push(sentence);
+    }
+  });
+  return [corpusTrain, corpusTest];
+};
+
+module.exports = ME_Corpus;

package/lib/natural/classifiers/maxent/POS/ME_Sentence.js

@@ -0,0 +1,73 @@
+/*
+  Sentence class specific for MaxEnt modeling
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var util = require('util');
+var Context = require('../Context');
+var Sentence = require('../../../brill_pos_tagger/lib/Sentence');
+var Element = require('./POS_Element');
+
+function ME_Sentence() {
+   ME_Sentence.super_.call(this);
+}
+
+util.inherits(ME_Sentence, Sentence);
+
+ME_Sentence.prototype.generateSampleElements = function(sample) {
+  var sentence = this.taggedWords;
+  sentence.forEach(function(token, index) {
+    var x = new Element(
+      token.tag,
+      new Context({
+        wordWindow: {},
+        tagWindow: {}
+      })
+    );
+
+    // Current word and tag
+    x.b.data.wordWindow["0"] = token.token;
+    x.b.data.tagWindow["0"] = sentence[index].tag;
+
+    // Previous bigram
+    if (index > 1) {
+      x.b.data.tagWindow["-2"] = sentence[index - 2].tag;
+      x.b.data.wordWindow["-2"] = sentence[index - 2].token;
+    }
+
+    // Left bigram
+    if (index > 0) {
+      x.b.data.tagWindow["-1"] = sentence[index - 1].tag;
+      x.b.data.wordWindow["-1"] = sentence[index - 1].token;
+    }
+
+    // Right bigram
+    if (index < sentence.length - 1) {
+      x.b.data.tagWindow["1"] = sentence[index + 1].tag;
+      x.b.data.wordWindow["1"] = sentence[index + 1].token;
+    }
+
+    // Next bigram
+    if (index < sentence.length - 2) {
+      x.b.data.tagWindow["2"] = sentence[index + 2].tag;
+      x.b.data.wordWindow["2"] = sentence[index + 2].token;
+    }
+
+    sample.addElement(x);
+  });
+};
+
+module.exports = ME_Sentence;