natural 0.5.6 → 0.6.3

package/classifier.json

@@ -0,0 +1,418 @@
+{
+  "features": {
+    "features": [
+      {
+        "name": "isZero",
+        "parameters": [
+          "0"
+        ],
+        "parametersKey": "0",
+        "observedExpect": 0.3
+      },
+      {
+        "name": "isOne",
+        "parameters": [
+          "1"
+        ],
+        "parametersKey": "1",
+        "observedExpect": 0.3
+      },
+      {
+        "name": "Correction feature",
+        "parameters": [],
+        "parametersKey": "",
+        "observedExpect": 0.4
+      }
+    ],
+    "map": {
+      "isZero | 0": true,
+      "isOne | 1": true,
+      "Correction feature | ": true
+    }
+  },
+  "sample": {
+    "frequencyOfContext": {
+      "\"0\"": 6,
+      "\"1\"": 4
+    },
+    "frequency": {
+      "x\"0\"": 3,
+      "y\"0\"": 3,
+      "x\"1\"": 1,
+      "y\"1\"": 3
+    },
+    "classes": [
+      "x",
+      "y"
+    ],
+    "elements": [
+      {
+        "a": "x",
+        "b": {
+          "data": "0",
+          "key": "\"0\""
+        },
+        "key": "x\"0\""
+      },
+      {
+        "a": "x",
+        "b": {
+          "data": "0",
+          "key": "\"0\""
+        },
+        "key": "x\"0\""
+      },
+      {
+        "a": "x",
+        "b": {
+          "data": "0",
+          "key": "\"0\""
+        },
+        "key": "x\"0\""
+      },
+      {
+        "a": "y",
+        "b": {
+          "data": "0",
+          "key": "\"0\""
+        },
+        "key": "y\"0\""
+      },
+      {
+        "a": "y",
+        "b": {
+          "data": "0",
+          "key": "\"0\""
+        },
+        "key": "y\"0\""
+      },
+      {
+        "a": "y",
+        "b": {
+          "data": "0",
+          "key": "\"0\""
+        },
+        "key": "y\"0\""
+      },
+      {
+        "a": "x",
+        "b": {
+          "data": "1",
+          "key": "\"1\""
+        },
+        "key": "x\"1\""
+      },
+      {
+        "a": "y",
+        "b": {
+          "data": "1",
+          "key": "\"1\""
+        },
+        "key": "y\"1\""
+      },
+      {
+        "a": "y",
+        "b": {
+          "data": "1",
+          "key": "\"1\""
+        },
+        "key": "y\"1\""
+      },
+      {
+        "a": "y",
+        "b": {
+          "data": "1",
+          "key": "\"1\""
+        },
+        "key": "y\"1\""
+      }
+    ]
+  },
+  "scaler": {
+    "featureSet": {
+      "features": [
+        {
+          "name": "isZero",
+          "parameters": [
+            "0"
+          ],
+          "parametersKey": "0",
+          "observedExpect": 0.3
+        },
+        {
+          "name": "isOne",
+          "parameters": [
+            "1"
+          ],
+          "parametersKey": "1",
+          "observedExpect": 0.3
+        },
+        {
+          "name": "Correction feature",
+          "parameters": [],
+          "parametersKey": "",
+          "observedExpect": 0.4
+        }
+      ],
+      "map": {
+        "isZero | 0": true,
+        "isOne | 1": true,
+        "Correction feature | ": true
+      }
+    },
+    "sample": {
+      "frequencyOfContext": {
+        "\"0\"": 6,
+        "\"1\"": 4
+      },
+      "frequency": {
+        "x\"0\"": 3,
+        "y\"0\"": 3,
+        "x\"1\"": 1,
+        "y\"1\"": 3
+      },
+      "classes": [
+        "x",
+        "y"
+      ],
+      "elements": [
+        {
+          "a": "x",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "x\"0\""
+        },
+        {
+          "a": "x",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "x\"0\""
+        },
+        {
+          "a": "x",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "x\"0\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "y\"0\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "y\"0\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "y\"0\""
+        },
+        {
+          "a": "x",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "x\"1\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "y\"1\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "y\"1\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "y\"1\""
+        }
+      ]
+    },
+    "iteration": 3,
+    "improvement": 0.008269762368828815,
+    "C": 1,
+    "featureSums": {
+      "x\"0\"": 1,
+      "y\"0\"": 0,
+      "x\"1\"": 0,
+      "y\"1\"": 1
+    }
+  },
+  "p": {
+    "alpha": [
+      0.8442857142857142,
+      1.8851785714285711,
+      0.777505117061545
+    ],
+    "featureSet": {
+      "features": [
+        {
+          "name": "isZero",
+          "parameters": [
+            "0"
+          ],
+          "parametersKey": "0",
+          "observedExpect": 0.3
+        },
+        {
+          "name": "isOne",
+          "parameters": [
+            "1"
+          ],
+          "parametersKey": "1",
+          "observedExpect": 0.3
+        },
+        {
+          "name": "Correction feature",
+          "parameters": [],
+          "parametersKey": "",
+          "observedExpect": 0.4
+        }
+      ],
+      "map": {
+        "isZero | 0": true,
+        "isOne | 1": true,
+        "Correction feature | ": true
+      }
+    },
+    "sample": {
+      "frequencyOfContext": {
+        "\"0\"": 6,
+        "\"1\"": 4
+      },
+      "frequency": {
+        "x\"0\"": 3,
+        "y\"0\"": 3,
+        "x\"1\"": 1,
+        "y\"1\"": 3
+      },
+      "classes": [
+        "x",
+        "y"
+      ],
+      "elements": [
+        {
+          "a": "x",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "x\"0\""
+        },
+        {
+          "a": "x",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "x\"0\""
+        },
+        {
+          "a": "x",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "x\"0\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "y\"0\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "y\"0\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "0",
+            "key": "\"0\""
+          },
+          "key": "y\"0\""
+        },
+        {
+          "a": "x",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "x\"1\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "y\"1\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "y\"1\""
+        },
+        {
+          "a": "y",
+          "b": {
+            "data": "1",
+            "key": "\"1\""
+          },
+          "key": "y\"1\""
+        }
+      ]
+    },
+    "aPriorisBeforeNormalisation": {
+      "x\"0\"": 0.8442857142857142,
+      "y\"0\"": 0.777505117061545,
+      "x\"1\"": 0.777505117061545,
+      "y\"1\"": 1.8851785714285711
+    },
+    "aPrioriNormalisationConstant": 11.298413325389037,
+    "aPosterioriNormalisationConstants": {
+      "\"0\"": 1.6217908313472593,
+      "\"1\"": 2.662683688490116
+    }
+  }
+}

package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tagger.js

@@ -19,6 +19,7 @@
 var fs = require("fs");
 
 var TF_Parser = require('./TF_Parser');
+var Sentence = require('./Sentence');
 
 function Brill_POS_Tagger(lexicon, ruleSet) {
   this.lexicon = lexicon;
@@ -30,18 +31,17 @@ function Brill_POS_Tagger(lexicon, ruleSet) {
 // the word itself followed by its lexical category
 Brill_POS_Tagger.prototype.tag = function(sentence) {
   var taggedSentence = this.tagWithLexicon(sentence);
+  //console.log(taggedSentence);
   return this.applyRules(taggedSentence);
 };
 
 Brill_POS_Tagger.prototype.tagWithLexicon = function(sentence) {
-  var taggedSentence = new Array(sentence.length);
+  var taggedSentence = new Sentence();
 
   var that = this;
   sentence.forEach(function(word, index) {
-    taggedSentence[index] = [];
-    taggedSentence[index][0] = word;
     var categories = that.lexicon.tagWord(word);
-    taggedSentence[index][1] = categories[0];
+    taggedSentence.addTaggedWord(word, categories[0]);
   });
   return(taggedSentence);
 };
@@ -50,14 +50,13 @@ Brill_POS_Tagger.prototype.tagWithLexicon = function(sentence) {
 // taggedSentence is an array of tagged words.
 // A tagged word is an array consisting of the word itself followed by its lexical category.
 // Returns an array of tagged words as well
-Brill_POS_Tagger.prototype.applyRules = function(taggedSentence) {
-  // Apply transformation rules
-  for (var i = 0, size = taggedSentence.length; i < size; i++) {
+Brill_POS_Tagger.prototype.applyRules = function(sentence) {
+  for (var i = 0, size = sentence.taggedWords.length; i < size; i++) {
     this.ruleSet.getRules().forEach(function(rule) {
-      rule.apply(taggedSentence, i);
+      rule.apply(sentence, i);
     });
   }
-  return(taggedSentence);
+  return sentence;
 };
 
 module.exports = Brill_POS_Tagger;

package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tester.js

@@ -27,16 +27,16 @@ Brill_POS_Tester.prototype.test = function(corpus, tagger) {
 
   // Tag the corpus using the tagger
   corpus.sentences.forEach(function(sentence) {
-    var s = sentence.map(function(token) {
+    var s = sentence.taggedWords.map(function(token) {
       return token.token;
     });
 
     // Use the lexicon to tag the sentence
     var taggedSentence = tagger.tagWithLexicon(s);
     // Count the right tags
-    sentence.forEach(function(token, i) {
+    sentence.taggedWords.forEach(function(token, i) {
       totalWords++;
-      if (token.tag === taggedSentence[i][1]) {
+      if (token.tag === taggedSentence.taggedWords[i].tag) {
         correctTagsLexicon++;
       }
     });
@@ -44,8 +44,8 @@ Brill_POS_Tester.prototype.test = function(corpus, tagger) {
     // Use the rule set to tag the sentence
     var taggedSentenceAfterRules = tagger.applyRules(taggedSentence);
     // Count the right tags
-    sentence.forEach(function(token, i) {
-      if (token.tag === taggedSentenceAfterRules[i][1]) {
+    sentence.taggedWords.forEach(function(token, i) {
+      if (token.tag === taggedSentenceAfterRules.taggedWords[i].tag) {
         correctTagsAfterRules++;
       }
     });

package/lib/natural/brill_pos_tagger/lib/Brill_POS_Trainer.js

@@ -27,6 +27,7 @@
 
 var TransformationRule = require("./TransformationRule");
 var RuleSet = require("./RuleSet");
+var Sentence = require('./Sentence');
 
 // Training continues as long as there are rules with a positive score
 // that have not been selected before
@@ -156,7 +157,7 @@ Brill_POS_Trainer.prototype.disconnectSiteFromRule = function(i, j, rule) {
 // Adjusts the score of the rule at position i, j of the corpus
 Brill_POS_Trainer.prototype.scoreRule = function(rule, i, j) {
   //logger.debug("Brill_POS_Trainer.prototype.scoreRule: entry");
-  var token = this.corpus.sentences[i][j];
+  var token = this.corpus.sentences[i].taggedWords[j];
   var rightTag = token.tag;
   var oldTag = token.testTag;
   var newTag = token.newTag;
@@ -198,7 +199,7 @@ Brill_POS_Trainer.prototype.scoreRule = function(rule, i, j) {
 // Generate positive rules for this given site using templates
 Brill_POS_Trainer.prototype.generatePositiveRules = function(i, j) {
   var sentence = this.corpus.sentences[i];
-  var token = sentence[j];
+  var token = sentence.taggedWords[j];
   // A positive rule should trigger on the currently assigned testTag
   var oldTag = token.testTag;
   //logger.debug("Brill_POS_Trainer.prototype.generatePositiveRules: oldTag " + oldTag);
@@ -242,7 +243,7 @@ Brill_POS_Trainer.prototype.scanForPositiveRules = function() {
   //logger.debug("Brill_POS_Trainer.prototype.scanForPositiveRules: entry");
   var that = this;
   this.corpus.sentences.forEach(function(sentence, i) {
-    sentence.forEach(function(token, j) {
+    sentence.taggedWords.forEach(function(token, j) {
       //logger.debug("Brill_POS_Trainer.prototype.scanForPositiveRules: sentence no " + i);
       var newRules = that.generatePositiveRules(i, j);
       newRules.getRules().forEach(function(rule) {
@@ -265,10 +266,13 @@ Brill_POS_Trainer.prototype.scanForSites = function() {
     if (i % 100 === 0) {
       //logger.info("Brill_POS_Trainer.prototype.scanForSites: sentence " + i);
     }
-    var taggedSentence = sentence.map(function(token) {
-      return [token.token, token.testTag];
+
+    var taggedSentence = new Sentence();
+    sentence.taggedWords.forEach(function(wordObject) {
+      taggedSentence.addTaggedWord(wordObject.token, wordObject.testTag);
     });
-    sentence.forEach(function(token, j) {
+
+    sentence.taggedWords.forEach(function(token, j) {
       that.positiveRules.getRules().forEach(function(rule) {
         if (rule.isApplicableAt(sentence, taggedSentence, j)) {
           that.associateSiteWithRule(i, j, rule);

package/lib/natural/brill_pos_tagger/lib/Corpus.js

@@ -1,6 +1,6 @@
 /*
   Corpus class for parsing and analysing corpora
-  Copyright (C) 2017 Hugo W.L. ter Doest
+  Copyright (C) 2018 Hugo W.L. ter Doest
 
   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-//var log4js = require('log4js');
-//var logger = log4js.getLogger();
-//logger.setLevel('INFO');
-
+var Sample = require('../../classifiers/maxent/Sample');
+var ElementClass = require('../../classifiers/maxent/POS/POS_Element');
 var Lexicon = require('./Lexicon');
+
 const BROWN = 1;
 
 // sentences: an array of annotated sentences
 // A sentence is an array of annotated tokens
 // A token is an object with (token, tag, testTag, ruleList)
-function Corpus(data, typeOfCorpus) {
+function Corpus(data, typeOfCorpus, SentenceClass) {
   this.wordCount = 0;
   this.sentences = [];
   if (data && typeOfCorpus) {
     // For other types of corpora add a case here and supply a parsing method
     switch (typeOfCorpus) {
       case BROWN:
-        this.parseBrownCorpus(data);
+        this.parseBrownCorpus(data, SentenceClass);
         break;
       default:
         // Assume it is an array of tagged sentences
@@ -45,7 +44,7 @@ function Corpus(data, typeOfCorpus) {
 // data is raw text
 // A corpus parsing method should split the corpus in sentences each of which
 // consist of an array of tokens.
-Corpus.prototype.parseBrownCorpus = function(data) {
+Corpus.prototype.parseBrownCorpus = function(data, SentenceClass) {
   var that = this;
 
   var lines = data.split('\n');
@@ -53,16 +52,13 @@ Corpus.prototype.parseBrownCorpus = function(data) {
     var trimmedLine = line.trim();
     // Only parse lines that contain characters
     if (trimmedLine != "") {
-      var taggedSentence = [];
+      var taggedSentence = new SentenceClass();
       var tokens = line.trim().split(/\s+/);
       tokens.forEach(function (token) {
         that.wordCount++;
         // Create a tagged sentences consisting of tokens
         var wordPlusTag = token.split('_');
-        taggedSentence.push({
-          "token": wordPlusTag[0],
-          "tag": wordPlusTag[1]
-        });
+        taggedSentence.addTaggedWord(wordPlusTag[0], wordPlusTag[1]);
       });
 
       // Add the sentence to the corpus
@@ -96,7 +92,7 @@ Corpus.prototype.splitInTrainAndTest = function(percentageTrain) {
 };
 
 // Analyses the corpus:
-// - registers used PS tags
+// - registers used POS tags
 // - records the frequency of POS tag for each word
 Corpus.prototype.analyse = function() {
   this.tagFrequencies = {};
@@ -105,7 +101,7 @@ Corpus.prototype.analyse = function() {
 
   var that = this;
   this.sentences.forEach(function(sentence) {
-    sentence.forEach(function(token) {
+    sentence.taggedWords.forEach(function(token) {
       that.wordCount++;
 
       // Register the tags used in the corpus
@@ -156,7 +152,7 @@ Corpus.prototype.buildLexicon = function() {
 
 Corpus.prototype.tag = function(lexicon) {
   this.sentences.forEach(function(sentence) {
-    sentence.forEach(function(token) {
+    sentence.taggedWords.forEach(function(token) {
       // tagWord returns a list of categories, take the first category
       token.testTag = lexicon.tagWord(token.token)[0];
     });
@@ -171,6 +167,15 @@ Corpus.prototype.nrWords = function() {
   return this.wordCount;
 };
 
+Corpus.prototype.generateFeatures = function() {
+  var features = [];
+  this.sentences.forEach(function(sentence) {
+    features = sentence.generateFeatures(features);
+  });
+  //console.log(JSON.stringify(features));
+  return features;
+};
+
 Corpus.prototype.prettyPrint = function() {
   this.sentences.forEach(function(sentence, index) {
     //logger.debug("sentence no " + index + "\n" +