natural 0.2.0 → 0.2.1

package/lib/natural/parsers/Agenda.js

@@ -1,68 +0,0 @@
-/*
-    Agenda class
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-// For deep comparing items including children
-var _ = require('underscore');
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('ERROR');
-
-// Constructor
-function Agenda() {
-  logger.debug("Agenda");
-  this.agenda = [];
-}
-
-// Adds an item to the agenda if it is nog already on  chart or agenda
-Agenda.prototype.add_item = function(item, chart) {
-  var item_found = false;
-  var nr_items = 0;
-  
-  logger.debug("Enter Agenda.add_item( "+ item.id + ")" + "; size of the agenda: " + this.agenda.length);
-  if (chart.is_not_on_chart(item)) {
-    this.agenda.some(function(item2) {
-      if (_.isEqual(item, item2)) {
-        item_found = true;
-        return(true);
-      }
-    });
-    
-    if (!item_found) {
-      this.agenda.push(item);
-      logger.debug("Agenda.add_item; item added: " + item.id);
-      nr_items = 1;
-    }
-  }
-  logger.debug("Exit Agenda.add_item; size of the agenda: " + this.agenda.length);
-  return(nr_items);
-};
-
-// Removes an item from the agenda and returns it
-Agenda.prototype.get_item = function() {
-  var item = this.agenda.pop();
-  logger.debug("Agenda.get_item: " + (item ? item.id : "agenda is empty"));
-  return(item);
-};
-
-// Checks if the agenda is empty
-Agenda.prototype.is_non_empty = function() {
-  logger.debug("Agenda.is_non_empty: " + (this.agenda !== []));
-  return(this.agenda !== []);
-};
-
-module.exports = Agenda;

package/lib/natural/parsers/CYKParser.js

@@ -1,87 +0,0 @@
-/*
-    Cocke Younger Kasami (CYK) chart parser
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('DEBUG');
-
-var Item = require('./CYK_Item');
-var Chart = require('./Chart');
-var ChartParser = require('./ChartParser');
-
-// Constructor for parser object
-// grammar is an object as defined in CFG.js
-function CYK_ChartParser(grammar) {
-  logger.debug("Enter CYK_ChartParser: " + JSON.stringify(grammar));
-  this.grammar = grammar;
-}
-
-CYK_ChartParser.prototype = Object.create(ChartParser.prototype);
-
-
-CYK_ChartParser.prototype.initialise = function(tagged_sentence) {
-  logger.debug("Enter CYK_ChartParser.initialise");
-  this.tagged_sentence = tagged_sentence;
-  this.N = tagged_sentence.length;
-  this.chart = new Chart(this.N);
-  for (var i = 0; i < this.N; i++) {
-    // Create a terminal item of the form Terminal -> *empty*
-    var term_item = new Item({'lhs': tagged_sentence[i][0], 'rhs': []}, i, i+1);
-    // Create a tag item of the form Categorie -> Terminal
-    for (var j = 1; j < tagged_sentence[i].length; j++) {
-      var tag_item = new Item({'lhs': this.tagged_sentence[i][j], 'rhs': [this.tagged_sentence[i][0]]}, i, i + 1);
-      tag_item.children.push(term_item);
-      this.chart.add_item(tag_item);
-      logger.debug("CYK_Parser.initialise: |- " + tag_item.id);
-    }
-  }
-  logger.debug("Exit CYK_ChartParser.initialise");
-};
-
-// This is the CYK chart parser
-// sentence is a tagged sentence of the form [[word, category], [word, category], ...]
-CYK_ChartParser.prototype.parse = function(tagged_sentence) {
-  var that = this;
-
-  logger.debug("Enter CYK_ChartParser.parse: " + JSON.stringify(tagged_sentence, null, 2));
-  this.initialise(tagged_sentence);
-  for (var i = 2; i <= this.N; ++i) { // Length of span
-    for (var j = 0; j <= this.N - i; ++j) { // Start of span
-      for (var k = i - 1; k >= 0; --k) { // Partition of span
-        var items1 = that.chart.get_items_from_to(j, j + k);
-        var items2 = that.chart.get_items_from_to(j + k, j + i);
-        items1.forEach(function(item1) {
-          items2.forEach(function(item2) {
-            var matching_rules = that.grammar.get_rules_with_rhs(item1.data.rule.lhs, item2.data.rule.lhs);
-            matching_rules.forEach(function(rule) {
-              var item = new Item(rule, item1.data.from, item2.data.to);
-              item.add_child(item1);
-              item.add_child(item2);
-              that.chart.add_item(item);
-              logger.debug("CYK_ChartParser.parse: " + item1.id + ", " + item2.id + " |- " + item.id);
-            });
-          });
-        });
-      }
-    }
-  }
-  logger.debug("Exit CYK_ChartParser.parse: " + JSON.stringify(this.chart, null, 2));
-  return this.chart;
-};
-
-module.exports = CYK_ChartParser;

package/lib/natural/parsers/CYK_Item.js

@@ -1,159 +0,0 @@
-/*
-    CYK item class for CYK and Head-Corner Parsing
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-// NB: CYK items are completely recognised items of the form [X, i, j] meaning ai...aj can be generated from X
-
-var typeOf = require('typeof');
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('ERROR');
-
-
-function CYK_Item(rule, i, j) {
-  logger.debug("CYK_Item: " + rule + ", " + i + ", " + j);
-  this.id = "CYK(" + rule.lhs + ", " + i + ", " + j + ")";
-  this.name = rule.lhs;
-  this.children = [];
-  this.data = {};
-  this.data.from = i;
-  this.data.to = j;
-  this.data.rule = rule;
-}
-
-module.exports = CYK_Item;
-
-var DoubleDottedItem = require('./DoubleDottedItem');
-
-CYK_Item.prototype.add_child = function(child) {
-  logger.debug("CYK_Item.add_child: " + child.id);
-  this.children.push(child);
-};
-
-CYK_Item.prototype.is_complete = function() {
-  logger.debug("CYK_Item.is_complete: CYK items always are complete!");
-  return(true);
-};
-
-CYK_Item.prototype.combine_with_chart = function(chart, agenda, grammar) {
-  var nr_items_added = 0;
-  
-  logger.debug("Enter CYK_Item.combine_with_chart:" + this.id);
-  nr_items_added += this.hc_predict(chart, agenda, grammar);
-  nr_items_added += this.left_complete(chart, agenda, grammar);
-  nr_items_added += this.right_complete(chart, agenda, grammar);
-  logger.debug("Exit CYK_Item.combine_with_chart: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-// Adds double dotted items to the chart based on goal and CYK items
-CYK_Item.prototype.hc_predict = function(chart, agenda, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-  
-  logger.debug("Enter CYK_Item.hc_predict: " + this.id);
-  var items = chart.get_items_from_to(this.data.from, this.data.to);
-  items.forEach(function(item) {
-    if (typeOf(item) === "goalitem") {
-      // Create head-corner items
-      var rules = grammar.get_rules_with_head(that.data.rule.lhs);
-      rules.forEach(function(rule) {
-        if (grammar.is_headcorner_of(rule.lhs, item.data.nonterminal)) {
-          var new_item = new DoubleDottedItem(rule, rule.head, rule.head+1, that.data.from, that.data.to);
-          new_item.children.push(that);
-          nr_items_added += agenda.add_item(new_item, chart);
-          logger.debug("CYK_Item.hc_predict: " + item.id +", " + that.id + " |- " + new_item.id);
-        }
-      });
-    }
-  });
-  logger.debug("Exit CYK_Item.hc_predict: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-CYK_Item.prototype.left_complete = function (chart, agenda, grammar) {
-  var nr_items_added = 0;
-  var that = this;
-  
-  logger.debug("Enter CYK_Item.left_complete: " + this.id);
-  var goals = chart.get_items_from(this.data.from);
-  var doubledotteditems = chart.get_items_from(this.data.to);
-  doubledotteditems.forEach(function(doubledotteditem) {
-    if ((typeOf(doubledotteditem) === "doubledotteditem") && (doubledotteditem.data.left_dot > 0)) {
-      goals.forEach(function(goal) {
-        if (typeOf(goal) === "goalitem") {
-          logger.debug("CYK_Item.left_complete: trying" + goal.id + ", " + that.id + ", " + doubledotteditem);
-          if ((doubledotteditem.data.rule.rhs[doubledotteditem.data.left_dot-1] === that.data.rule.lhs) &&
-              grammar.is_headcorner_of(doubledotteditem.data.rule.lhs, goal.data.nonterminal) ) {
-            new_item = doubledotteditem.copy();
-            new_item.recognise_left(that);
-            nr_items_added += agenda.add_item(new_item, chart);
-            logger.debug("CYK_Item.left_complete: " + goal.id + ", " + that.id + ", " + doubledotteditem + " |- " + new_item.id);
-          }
-        }
-      });
-    }
-  });
-  logger.debug("Exit CYK_Item.left_complete: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-CYK_Item.prototype.right_complete = function (chart, agenda, grammar) {
-  var that = this;
-  var nr_items_added = 0;
-
-  logger.debug("Enter CYK_Item.right_complete: " + this.id);
-  var goals = chart.get_items_to(this.data.to);
-  var doubledotteditems = chart.get_items_to(this.data.from);
-  doubledotteditems.forEach(function(doubledotteditem) {
-    if (typeOf(doubledotteditem) === "doubledotteditem") {
-      goals.forEach(function(goal) {
-        if (typeOf(goal) === "goalitem") {
-          logger.debug("CYK_Item.right_complete: trying: " + goal.id + ", " + doubledotteditem + ", " + that.id);
-          if ((doubledotteditem.data.rule.rhs[doubledotteditem.data.right_dot] === that.data.rule.lhs) &&
-              grammar.is_headcorner_of(doubledotteditem.data.rule.lhs, goal.data.nonterminal) ) {
-            new_item = doubledotteditem.copy();
-            new_item.recognise_right(that);
-            nr_items_added += agenda.add_item(new_item, chart);
-            logger.debug("CYK_Item.right_complete: " + goal.id + ", " + doubledotteditem + ", " + that.id + " |- " + new_item.id);
-          }
-        }
-      });
-    }
-  });
-  logger.debug("Exit CYK_Item.right_complete: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-CYK_Item.prototype.create_parse_tree = function() {
-  logger.debug("Enter CYK_Item.create_parse_tree: " + this.id);
-  var subtree = this.data.rule.lhs;
-  if (this.children.length === 0) {
-    subtree += "(" + this.data.rule.rhs + ")";
-  }
-  else {
-    subtree += "(";
-    var i;
-    for (i = 0; i < this.children.length; i++) {
-      subtree +=  this.children[i].create_parse_tree() + (i < this.children.length - 1 ? "," : "");
-    }
-    subtree += ")";
-  }
-  logger.debug("Exit CYK_Item.create_parse_tree: " + subtree);
-  return(subtree);
-};
-

package/lib/natural/parsers/Chart.js

@@ -1,211 +0,0 @@
-/*
-    Generic chart that can be used for all chart parsers
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-// Implements a chart with from/to edges. Items are added by chart.add_item(i, j, item)
-// Items are identified by their id. Items that have the same id are not added to the same edge.
-
-// For deep comparing items including children
-var _ = require('underscore');
-var typeOf = require('typeof');
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('DEBUG');
-
-// Creates a chart for recognition of a sentence of length N
-function Chart(N) {
-  logger.debug("Chart: " + N);
-  this.N = N;
-  this.outgoing_edges = new Array(N+1);
-  this.incoming_edges = new Array(N+1);
-  
-  var i;
-  for (i = 0; i <= N; i++) {
-    this.outgoing_edges[i] = {};
-    this.incoming_edges[i] = {};
-  }
-}
-
-// Adds an item to the chart if it is not already there; returns 1 if the item was added, 0 otherwise
-// Items are compared using deep compare (so including children)
-Chart.prototype.add_item = function(item) {
-  var nr_items_added = 0;
-  var item_found = false;
-
-  logger.debug("Enter Chart.add_item: " + item.id);
-  if (this.outgoing_edges[item.data.from][item.id]) {
-    // item already exists -> deep compare
-    this.outgoing_edges[item.data.from][item.id].some(function(item2) {
-      if (_.isEqual(item, item2)) {
-        item_found = true;
-        return(true);
-      }
-    });
-    if (!item_found) {  
-      // if not found -> add  item to chart
-      logger.debug("Chart.add_item: " + this.outgoing_edges[item.data.from][item.id]);
-      this.outgoing_edges[item.data.from][item.id].push(item);
-      logger.debug("Chart.add_item: " + this.incoming_edges[item.data.to][item.id]);
-      if (!this.incoming_edges[item.data.to][item.id]) {
-        this.incoming_edges[item.data.to][item.id] = [];
-      }
-      this.incoming_edges[item.data.to][item.id].push(item);
-      nr_items_added = 1;
-    }
-  }
-  else {
-      // item does not exist -> add to the chart
-      this.outgoing_edges[item.data.from][item.id] = [item];
-      this.incoming_edges[item.data.to][item.id] = [item];
-      nr_items_added = 1;
-  }
-  
-  logger.debug("Exit Chart.add_item: number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-Chart.prototype.is_not_on_chart = function(item) {
-  var item_found = false;
-
-  logger.debug("Enter Chart.is_not_on_chart " + item.id);
-  if (this.outgoing_edges[item.data.from][item.id]) {
-    // item already exists -> deep compare
-    this.outgoing_edges[item.data.from][item.id].some(function(item2) {
-      if (_.isEqual(item, item2)) {
-        item_found = true;
-        return(true);
-      }
-    });
-  }
-  logger.debug("Exit Chart.is_not_on_chart: " + item_found);
-  return(!item_found);
-};
-
-// Returns all items that span i to j
-Chart.prototype.get_items_from_to = function(i, j) {
-  var res = [];
-  var that = this;
-  
-  logger.debug("Enter Chart.get_items_from_to(" + i + ", " + j + ")");
-  Object.keys(this.outgoing_edges[i]).forEach(function(item_id){
-    if (that.outgoing_edges[i][item_id].length > 0) {
-      if (that.outgoing_edges[i][item_id][0].data.to === j) {
-        res = res.concat(that.outgoing_edges[i][item_id]);
-      }
-    }
-  });
-  logger.debug("Exit Chart.get_items_from_to: " + JSON.stringify(res));
-  return(res);
-};
-
-Chart.prototype.get_items_from = function(i) {
-  var res = [];
-  var that = this;
-
-  logger.debug("Enter Chart.get_items_from(" + i + ")");
-  Object.keys(this.outgoing_edges[i]).forEach(function(item_id){
-    res = res.concat(that.outgoing_edges[i][item_id]);
-  });
-  logger.debug("Exit Chart.get_items_from: " + res);
-  return(res);
-};
-
-Chart.prototype.get_items_to = function(j) {
-  var res = [];
-  var that = this;
-  
-  logger.debug("Enter Chart.get_items_to(" + j + ")");
-  Object.keys(this.incoming_edges[j]).forEach(function(item_id){
-    res = res.concat(that.incoming_edges[j][item_id]);
-  });
-  logger.debug("Exit Chart.get_items_to:" + res);
-  return(res);
-};
-
-Chart.prototype.nr_items_to = function(j) {
-  var that = this;
-  var nr_items = 0;
-  
-  logger.debug("Enter Chart.nr_items_to(" + j + ")");
-  Object.keys(this.incoming_edges[j]).forEach(function(item_id){
-      nr_items += that.incoming_edges[j][item_id].length;
-  });
-  logger.debug("Exit Chart.nr_items_to: " + nr_items);
-  return(nr_items);
-};
-
-// Returns all complete items that span i to j
-Chart.prototype.get_complete_items_from_to = function(i, j) {
-  var res = [];
-  var that = this;
-  
-  logger.debug("Enter Chart.get_complete_items_from_to(" + i + ", " + j + ")");
-  this.get_items_from_to(i, j).forEach(function(item){
-    if (item.is_complete()) {
-      res.push(item);
-    }
-  });
-  logger.debug("Exit Chart.get_complete_items_from_to: " + res);
-  return(res);
-};
-
-// Returns all complete items span i to j AND start with nonterminal A
-Chart.prototype.full_parse_items = function(A) {
-  var that = this;
-  var items = [];
-  
-  logger.debug("Enter Chart.full_parse_items(" + A + ")");
-  this.get_complete_items_from_to(0, this.N).forEach(function(item) {
-    if (item.data.rule.lhs === A) {
-      items.push(item);
-    }
-  });
-  logger.debug("Exit Chart.full_parse_items: " + items);
-  return(items);
-};
-
-// Returns the parse trees in textual bracketed form
-// item_type selects the right type of item to create the parse tree from
-Chart.prototype.parse_trees = function(nonterminal, item_type) {
-  var that = this;
-  var parses = [];
-
-  logger.debug("Enter Chart.parse_items(" + nonterminal + ", " + item_type + ")");
-  this.get_items_from_to(0, this.N).forEach(function(item) {
-    if (typeOf(item) === item_type) {
-      if ((item.data.rule.lhs === nonterminal) && item.is_complete()) {
-        parses.push(item.create_parse_tree());
-      }
-    }
-  });
-  logger.debug("Exit Chart.parse_items:" + parses);
-  return(parses);
-};
-
-// Returns the total number of items on the chart
-Chart.prototype.nr_of_items = function() {
-  var nr_items = 0;
-  
-  logger.debug("Enter Chart.nr_of_items()");
-  for (i = 0; i <= this.N; i++) {
-    nr_items += this.nr_items_to(i);
-  }
-  logger.debug("Exit Chart.nr_of_items: " + nr_items);
-  return(nr_items);
-};
-
-module.exports = Chart;

package/lib/natural/parsers/ChartParser.js

@@ -1,60 +0,0 @@
-/*
-    Generic chart parser that can be used to create different types of chart parsers
-    Copyright (C) 2014 Hugo W.L. ter Doest
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-var log4js = require('log4js');
-var logger = log4js.getLogger();
-logger.setLevel('ERROR');
-
-var EarleyItem = require('./EarleyItem');
-var Chart = require('./Chart');
-
-ChartParser.prototype.initialise = function(tagged_sentence) {
-  var that = this;
-  var nr_items_added = 0;
-  
-  logger.debug('Enter ChartParser.initialise');
-  this.tagged_sentence = tagged_sentence;
-  this.N = this.tagged_sentence.length;
-  // Initialise chart
-  this.chart = new Chart(this.N);
-  // Add items for rules that have the start symbol as left-hand-side
-  var rules = this.grammar.rules_with_lhs(this.grammar.get_start_symbol());
-  rules.forEach(function(rule) {
-    var new_item = new EarleyItem(rule, 0, 0, 0);
-    nr_items_added += that.chart.add_item(new_item);
-  });
-
-  // Add items for the tagged sentence
-  for (var i = 0; i < this.N; i++) {
-    // Create terminal item
-    var term_item = new EarleyItem({'lhs': this.tagged_sentence[i][0], 'rhs': ''}, 1, i, i + 1);
-    // Add tag items
-    for (var j = 1; j < tagged_sentence[i].length; j++) {
-      var tag_item = new EarleyItem({'lhs': this.tagged_sentence[i][j], 'rhs': [this.tagged_sentence[i][0]]}, 1, i, i + 1);
-      tag_item.append_child(term_item);
-      nr_items_added += this.chart.add_item(tag_item);
-      logger.debug("ChartParser.initialise: |- " + tag_item.id);
-    }
-  }
-  logger.debug("Exit ChartParser.initialise; number of items added: " + nr_items_added);
-  return(nr_items_added);
-};
-
-function ChartParser() {}
-
-module.exports = ChartParser;