natural 0.2.0 → 0.2.1

package/lib/natural/parsers/DoubleDottedItem.js

@@ -1,226 +0,0 @@
-/*
-    DoubleDottedItem class
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-var _ = require('underscore');
-var typeOf = require('typeof');
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('ERROR');
-
-// Creates an item; 
-// left_dot is an index in the RHS of the rule as well as right_dot
-// from is the starting point in the sentence
-// Data structure is prepared for InfoVis
-function DoubleDottedItem(rule, left_dot, right_dot, from, to) {
-
-  // An identifier is constructed from rule, dots, from and to
-  this.id = "DoubleDottedItem(" + rule.lhs + "->" + rule.rhs + ", " + left_dot + ", " + right_dot + ", " + from + ", " + to +")";
-  logger.debug("Enter DoubleDottedItem: " + this.id);
-  this.name = rule.lhs;
-  this.children = [];
-
-  this.data = {};
-  this.data.rule = rule;
-  this.data.left_dot = left_dot;
-  this.data.right_dot = right_dot;
-  this.data.from = from;
-  this.data.to = to;
-  logger.debug("Exit DoubleDottedItem: created item: " + this.id);
-}
-
-module.exports = DoubleDottedItem;
-
-var GoalItem = require('./GoalItem');
-var CYK_Item = require('./CYK_Item');
-
-DoubleDottedItem.prototype.copy = function () {
-  logger.debug("Enter DoubleDottedItem.copy(): " + this.id);
-  var new_item = new DoubleDottedItem(this.data.rule, this.data.left_dot, this.data.right_dot, this.data.from, this.data.to);
-  new_item.children = this.children.slice();
-  logger.debug("Exit DoubleDottedItem.copy: " + new_item.id);
-  return(new_item);
-};
-
-DoubleDottedItem.prototype.recognise_left = function(cyk_item) {
-  logger.debug("Enter " + this.id + ".recognise_left(" + cyk_item.id + ")");
-  this.data.left_dot--;
-  this.data.from = cyk_item.data.from;
-  this.children.unshift(cyk_item);
-  this.id = "DoubleDottedItem(" + this.data.rule.lhs + "->" + this.data.rule.rhs + ", " + 
-            this.data.left_dot + ", " + this.data.right_dot + ", " + this.data.from + ", " + this.data.to +")";
-  logger.debug("Exit DoubleDottedItem.recognise_left: " + this.id);
-};
-
-DoubleDottedItem.prototype.recognise_right = function(cyk_item) {
-  logger.debug("Enter " + this.id + ".recognise_right(" + cyk_item.id + ")");
-  this.data.right_dot++;
-  this.data.to = cyk_item.data.to;
-  this.children.push(cyk_item);
-  this.id = "DoubleDottedItem(" + this.data.rule.lhs + "->" + this.data.rule.rhs + ", " + 
-            this.data.left_dot + ", " + this.data.right_dot + ", " + this.data.from + ", " + this.data.to +")";
-  logger.debug("Exit DoubleDottedItem.recognise_right: " + this.id);
-};
-
-// Checks if an item is incomplete
-DoubleDottedItem.prototype.is_incomplete = function () {
-  return(!this.is_complete());
-};
-
-// Checks if an item is complete
-DoubleDottedItem.prototype.is_complete = function () {
-  return((this.data.left_dot === 0) && (this.data.right_dot === this.data.rule.rhs.length));
-};
-
-DoubleDottedItem.prototype.combine_with_chart = function(chart, agenda, grammar) {
-  var nr_items_added = 0;
-  
-  logger.debug("Enter DoubleDottedItem.combine_with_chart:" + this.id);
-  nr_items_added += this.pre_complete(chart, agenda, grammar);
-  nr_items_added += this.left_predict(chart, agenda, grammar);
-  nr_items_added += this.right_predict(chart, agenda, grammar);
-  nr_items_added += this.left_complete(chart, agenda, grammar);
-  nr_items_added += this.right_complete(chart, agenda, grammar);
-  logger.debug("Exit DoubleDottedItem.combine_with_chart: number of items added: " + nr_items_added);
-};
-
-DoubleDottedItem.prototype.pre_complete = function (chart, agenda, grammar) {
-  var nr_items_added = 0;
-
-  logger.debug("Enter DoubleDottedItem.pre_complete:" + this.id);
-  if (this.is_complete()) {
-    var new_item = new CYK_Item(this.data.rule, this.data.from, this.data.to);
-    new_item.children = this.children.slice();
-    nr_items_added += agenda.add_item(new_item, chart);
-    logger.debug("DoubleDottedItem.pre_complete: " + this.id + " |- " + new_item.id);
-  }
-  logger.debug("Exit DoubleDottedItem.pre_complete: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-DoubleDottedItem.prototype.left_predict = function (chart, agenda, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("Enter DoubleDottedItem.left_predict:" + this.id);
-  if (this.data.left_dot > 0) {
-    // Get goal items that end at the "to" of the current item
-    var items = chart.get_items_to(this.data.to);
-    items.forEach(function(item) {
-      if (typeOf(item) === "goalitem") { // try to combine with the current item
-        if (grammar.is_headcorner_of(that.data.rule.lhs, item.nonterminal)) {
-          for (var i = item.data.from; i <= that.data.from; i++) {
-            for (var j = i; j <= that.data.from; j++) {
-              var new_goal = new GoalItem(i, j, that.data.rule.rhs[that.data.left_dot-1]);
-              nr_items_added += agenda.add_item(new_goal, chart);
-              logger.debug("DoubleDottedItem.left_predict: " + item.id + ", " + that.id + " |- " + new_goal.id);
-            }
-          }
-        }
-      }
-    });
-  }
-  logger.debug("Exit DoubleDottedItem.left_predict: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-DoubleDottedItem.prototype.right_predict = function (chart, agenda, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("Enter DoubleDottedItem.right_predict:" + this.id);
-  if (this.data.right_dot < this.data.rule.rhs.length) {
-    // Get goal items that start at the "from" of the current item
-    var items = chart.get_items_from(this.data.from);
-    items.forEach(function(item) {
-      if (typeOf(item) === "goalitem") { // try to combine with the current item
-        if (grammar.is_headcorner_of(that.data.rule.lhs, item.data.nonterminal)) {
-          for (var j = that.data.to; j <= item.data.to; j++) {
-            for (var k = j; k <= item.data.to; k++) {
-              var new_goal = new GoalItem(j, k, that.data.rule.rhs[that.data.right_dot]);
-              nr_items_added += agenda.add_item(new_goal, chart);
-              logger.debug("DoubleDottedItem.right_predict: " + item.id + ", " + that.id + " |- " + new_goal.id);
-            }
-          }
-        }
-      }
-    });
-  }
-  logger.debug("Exit DoubleDottedItem.right_predict: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-DoubleDottedItem.prototype.left_complete = function (chart, agenda, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("Enter DoubleDottedItem.left_complete:" + this.id);
-  if  (this.data.left_dot > 0) {
-    // Get goal items that end at the "to" of the current item
-    var items1 = chart.get_items_to(this.data.to);
-    items1.forEach(function(item1) {
-      if (typeOf(item1) === "goalitem") { 
-        // find completed items to combine with
-        var items2 = chart.get_items_from_to(item1.data.from, that.data.from);
-        items2.forEach(function(item2) {
-          if ((typeOf(item2) === "cyk_item") &&
-              (that.data.rule.rhs[that.data.left_dot-1] === item2.data.rule.lhs) &&
-              grammar.is_headcorner_of(that.data.rule.lhs, item1.data.nonterminal)) {
-            var new_item = that.copy();
-            new_item.recognise_left(item2);
-            nr_items_added += agenda.add_item(new_item, chart);
-            logger.debug("DoubleDottedItem.left_complete: " + item1.id +", " + item2.id + ", " + that.id + " |- " + new_item.id);
-          }
-        });
-      }
-    });
-  }
-  logger.debug("Exit DoubleDottedItem.left_complete: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-DoubleDottedItem.prototype.right_complete = function (chart, agenda, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("Enter DoubleDottedItem.right_complete:" + this.id);
-  if (this.data.right_dot < this.data.rule.rhs.length) {
-    // Get goal items that start at the "from" of the current item
-    var items1 = chart.get_items_from(this.data.from);
-    items1.forEach(function(item1) {
-      if (typeOf(item1) === "goalitem") { 
-        // find completed items to combine with
-        var items2 = chart.get_items_from_to(that.data.to, item1.data.to);
-        items2.forEach(function(item2) {
-          if (typeOf(item2) === "cyk_item") {
-            if ((that.data.rule.rhs[that.data.right_dot] === item2.data.rule.lhs) &&
-                grammar.is_headcorner_of(that.data.rule.lhs, item1.data.nonterminal)) {
-              new_item = that.copy();
-              new_item.recognise_right(item2);
-              nr_items_added += agenda.add_item(new_item, chart);
-              logger.debug("DoubleDottedItem.right_complete: " + item1.id +", " + that.id + ", " + item2.id + " |- " + new_item.id);
-            }
-          }
-        });
-      }
-    });
-  }
-  logger.debug("Exit DoubleDottedItem.right_complete: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-module.exports = DoubleDottedItem;

package/lib/natural/parsers/EarleyItem.js

@@ -1,155 +0,0 @@
-/*
-    Single dotted items for Earley and Left-Corner parsing
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-var _ = require('underscore');
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('DEBUG');
-
-// Creates an item; dot is an index in the RHS of the rule, 
-// from is the starting point in the sentence
-// Data structure is prepared for InfoVis
-function EarleyItem(rule, dot, from, to) {
-  // A unique identifier is constructed from rule, dot and from
-  this.id = "(" + rule.lhs + "->" + rule.rhs + ", " + dot + ", " + from + ", " + to +")";
-  logger.debug("EarleyItem: " + this.id);
-  this.name = rule.lhs;
-  this.children = [];
-
-  this.data = {};
-  this.data.rule = rule;
-  this.data.dot = dot;
-  this.data.from = from;
-  this.data.to = to;
-}
-
-EarleyItem.prototype.set_children = function(children) {
-  logger.debug("Enter EarleyItem.set_children: " + children);
-  this.children = children;
-  logger.debug("Exit EarleyItem.set_children");
-};
-
-EarleyItem.prototype.append_child = function(child) {
-  logger.debug("Enter EarleyItem.append_child: " + child);
-  this.children.push(child);
-  logger.debug("Exit EarleyItem.append_child");
-};
-
-// Checks if an item is incomplete
-EarleyItem.prototype.is_incomplete = function () {
-  logger.debug("EarleyItem.is_incomplete: " + this.id + (this.data.dot < this.data.rule.rhs.length));
-  return(this.data.dot < this.data.rule.rhs.length);
-};
-
-// Checks if an item is complete
-EarleyItem.prototype.is_complete = function () {
-  logger.debug("EarleyItem.is complete: " + this.id + (this.data.dot === this.data.rule.rhs.length));
-  return(this.data.dot === this.data.rule.rhs.length);
-};
-
-// Introduces new items for the next nonterminal to be recognised
-EarleyItem.prototype.predictor = function(chart, grammar) { 
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("EarleyItem.predictor: " + that.id);
-  if (this.is_incomplete() && grammar.is_nonterminal(this.data.rule.rhs[this.data.dot])) {
-    // Get all rules with lhs B
-    var rules_with_lhs_B = grammar.rules_with_lhs(this.data.rule.rhs[this.data.dot]);
-    // for each rule with LHS B create an item
-    rules_with_lhs_B.forEach(function(rule) {
-      var newitem = new EarleyItem(rule, 0, that.data.to, that.data.to);
-      nr_items_added += chart.add_item(newitem);
-      logger.debug("EarleyItem.predictor: " + JSON.stringify(rule) + ", " + that.id + " |- " + newitem.id);
-    });
-  }
-  logger.debug("EarleyItem.predictor: added " + nr_items_added + " items");
-  // Return number of items added
-  return(nr_items_added);
-};
-
-// LC method is based on the improved left-corner algorithm in 
-// Improved Left-Corner Chart Parsing for Large Context-Free Grammars, Robert C. Moore
-// IWPT2000
-// Predictor based on rule 4a on page 5
-EarleyItem.prototype.lc_predictor = function(chart, grammar) {
-  that = this;
-  var nr_items_added = 0;
-
-  logger.debug("EarleyItem.lc_predictor: " + this.id);
-  if (this.is_complete()) {
-    // Get the productions rules that have the LHS of item as left-most daughter
-    grammar.rules_with_leftmost_daughter(this.data.rule.lhs).forEach(function(rule) {
-      // Get the items that end at the given item
-      chart.get_items_to(that.data.from).forEach(function(item2) {
-        if (item2.is_incomplete() && grammar.is_leftcorner_of(rule.lhs, item2.data.rule.rhs[item2.data.dot])) {
-          var newitem = new EarleyItem(rule, 1, that.data.from, that.data.to);
-          newitem.append_child(that);
-          nr_items_added += chart.add_item(newitem);
-          logger.debug("EarleyItem.lc_predictor: " + JSON.stringify(rule) + ", " + that.id + ", " + item2.id + " |- " + newitem.id);
-        }
-      });
-    });
-  }
-  logger.debug("EarleyItem.lc_predictor: added " + nr_items_added + " items");
-  return(nr_items_added);
-};
-
-// item is complete
-// Shifts the dot to the right for items in chart[k]
-EarleyItem.prototype.completer = function(chart, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("EarleyItem.completer: " + this.id);
-  if (this.is_complete()) {
-    var items = chart.get_items_to(this.data.from);
-    items.forEach(function(item2) {
-      if (item2.is_incomplete() && (that.data.rule.lhs === item2.data.rule.rhs[item2.data.dot])) {
-        var new_item = new EarleyItem(item2.data.rule, item2.data.dot + 1, item2.data.from, that.data.to);
-        // Make a copy of the children of item2, otherwise two items refer to the same set of children
-        new_item.children = item2.children.slice();
-        new_item.append_child(that);
-        nr_items_added += chart.add_item(new_item);
-        logger.debug("EarleyItem.completer: " + that.id + ", " + item2.id + " |- " + new_item.id);
-      }
-    });
-  }
-  logger.debug("EarleyItem.completer: added " + nr_items_added + " items");
-  return(nr_items_added);
-};
-
-EarleyItem.prototype.create_parse_tree = function() {
-  logger.debug("Enter EarleyItem.create_parse_tree: " + this.id);
-  var subtree = this.data.rule.lhs;
-  if (this.children.length === 0) {
-    subtree += "(" + this.data.rule.rhs + ")";
-  }
-  else {
-    subtree += "(";
-    var i;
-    for (i = 0; i < this.children.length; i++) {
-      subtree +=  this.children[i].create_parse_tree() + (i < this.children.length - 1 ? "," : "");
-    }
-    subtree += ")";
-  }
-  logger.debug("Exit EarleyItem.create_parse_tree: " + subtree);
-  return(subtree);
-};
-
-module.exports = EarleyItem;

package/lib/natural/parsers/EarleyParser.js

@@ -1,58 +0,0 @@
-/*
-    Earley Chart parser
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('DEBUG');
-
-var EarleyItem = require('./EarleyItem');
-var Chart = require('./Chart');
-var ChartParser = require("./ChartParser");
-
-function EarleyChartParser(grammar) {
-  logger.debug("EarleyChartParser");
-  this.grammar = grammar;
-}
-
-EarleyChartParser.prototype = Object.create(ChartParser.prototype);
-
-// The main algorithm of the chart parser. Sentence is an array of words
-// Analyses the sentence from left to right
-// If you need more methods to be called then override this method
-EarleyChartParser.prototype.parse = function(tagged_sentence) {
-  that = this;
-
-  logger.debug("Enter EarleyChartParser.parse" + tagged_sentence);
-  this.initialise(tagged_sentence);
-  var i;
-  for (i = 0; i <= tagged_sentence.length; i++) {
-    var items_added;
-    do {
-      items_added = 0;
-      var items = this.chart.get_items_to(i);
-      items.forEach(function(item) {
-        items_added += item.completer(that.chart, that.grammar);
-        items_added += item.predictor(that.chart, that.grammar);
-      });
-    } while (items_added);
-  }
-  logger.debug("Enter EarleyChartParser.parse" + this.chart);
-  return this.chart;
-};
-
-module.exports = EarleyChartParser;

package/lib/natural/parsers/FunctionWordTagger.js

@@ -1,84 +0,0 @@
-/*
-    A part-of-speech tagger for English function words
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-var fs = require('fs');
-
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('ERROR');
-
-// Files that contain function words; files are located in ../data
-files_with_function_words = [
-  "function_words_adverb.txt",
-  "function_words_indefinite_pronoun.txt",
-  "function_words_possessive_pronoun.txt",
-  "function_words_relative_pronoun.txt",
-  "function_words_conjunction.txt",
-  "function_words_interrogative_pronoun.txt",
-  "function_words_preposition.txt",
-  "function_words_demonstrative_pronoun.txt",
-  "function_words_particle.txt",
-  "function_words_reciprocal_pronoun.txt",
-  "function_words_determiner.txt",
-  "function_words_personal_pronoun.txt",
-  "function_words_reflexive_pronoun.txt"];
-  
-path_to_lexicon = "../data/";
-
-// Returns an array of category for word
-FunctionWordTagger.prototype.tag_word = function(word) {
-  return(this.lexicon[word]);
-};
-
-// Parses the list of words; first lines is the POS category
-FunctionWordTagger.prototype.read_words = function(text) {
-  var words = text.split('\n');
-  var category = words.shift();
-  var that = this;
-  
-  logger.debug("Enter FunctionWordTagger.read_words");
-  words.forEach(function(word) {
-    if (!that.lexicon[word]) {
-      that.lexicon[word] = [category];
-    }
-    else {
-      that.lexicon[word].push(category);
-    }
-    logger.debug("FunctionWordTagger.read_words: lexicon entry " + word + " -> " + that.lexicon[word]);
-  });
-  logger.debug("Exit FunctionWordTagger.read_words");
-};
-
-// Constructor: reads files with function words and calls the callback with itself 
-function FunctionWordTagger(callback) {
-  var that = this;
-  var nr_files = files_with_function_words.length;
-
-  this.lexicon = {};
-  files_with_function_words.forEach(function(filename){
-    fs.readFile(path_to_lexicon + filename, 'utf8', function (error, text) {
-      that.read_words(text);
-      nr_files--;
-      if (nr_files === 0) {
-        callback(that);
-      }
-    });
-  });
-}
-
-module.exports = FunctionWordTagger;