natural 0.5.6 → 0.6.3

package/lib/natural/brill_pos_tagger/lib/RuleTemplates.js

@@ -261,54 +261,40 @@ var ruleTemplates = {
 // ==================================
 // Predicates that start with words
 // ==================================
-function next_word_is_cap(tagged_sentence, i, parameter) {
-  if (i < tagged_sentence.length - 1) {
-    var next_word = tagged_sentence[i+1][0];
-    return(next_word[0] === next_word[0].toUpperCase());
-  }
-  else {
-    return(false);
-  }
-}
-
-/*
 function next_word_is_cap(sentence, i, parameter) {
-  if (i < sentence.length - 1) {
-    var next_word = tagged_sentence[i+1].token;
+  if (i < sentence.taggedWords.length - 1) {
+    var next_word = sentence.taggedWords[i+1].token;
     return(next_word[0] === next_word[0].toUpperCase());
   }
-  else {
-    return(false);
-  }
+  return(false);
 }
-*/
 
-function next_word_is(tagged_sentence, i, parameter) {
-  if (i < tagged_sentence.length - 1) {
-    return(tagged_sentence[i + 1][0] === parameter);
+function next_word_is(sentence, i, parameter) {
+  if (i < sentence.taggedWords.length - 1) {
+    return(sentence.taggedWords[i + 1].token === parameter);
   }
 }
 
 function nextWordParameterValues(sentence, i) {
-  if (i < sentence.length - 1) {
-    return [sentence[i + 1].token];
+  if (i < sentence.taggedWords.length - 1) {
+    return [sentence.taggedWords[i + 1].token];
   }
   else {
     return [];
   }
 }
 
-function prev_word_is_cap(tagged_sentence, i, parameter) {
+function prev_word_is_cap(sentence, i, parameter) {
   var prev_word = null;
   if (i > 0) {
-    prev_word = tagged_sentence[i-1][0];
+    prev_word = sentence.taggedWords[i-1].token;
     return(prev_word[0] === prev_word[0].toUpperCase());
   }
   return(false);
 }
 
-function current_word_is_cap(tagged_sentence, i, parameter) {
-  var current_word = tagged_sentence[i][0];
+function current_word_is_cap(sentence, i, parameter) {
+  var current_word = sentence.taggedWords[i].token;
   return(current_word[0] === current_word[0].toUpperCase());
 }
 
@@ -316,40 +302,40 @@ function currentWordParameterValues(sentence, i) {
   return [sentence[i].token];
 }
 
-function current_word_is(tagged_sentence, i, parameter) {
-  return(tagged_sentence[i][0] === parameter);
+function current_word_is(sentence, i, parameter) {
+  return(sentence.taggedWords[i].token === parameter);
 }
 
 function isNumeric(num) {
   return (!isNaN(num));
 }
 
-function current_word_is_number(tagged_sentence, i, parameter) {
-  var is_number = isNumeric(tagged_sentence[i][0]);
+function current_word_is_number(sentence, i, parameter) {
+  var is_number = isNumeric(sentence.taggedWords[i].token);
   // Attempt to parse it as a float
   if (!is_number) {
-    is_number = parseFloat(tagged_sentence[i][0]);
+    is_number = parseFloat(sentence.taggedWords[i].token);
   }
   return((parameter === "YES") ? is_number : !is_number);
 }
 
 // Checks if the current word is a url
 // Adapted from the original Javascript Brill tagger
-function current_word_is_url(tagged_sentence, i, parameter) {
+function current_word_is_url(sentence, i, parameter) {
   var is_url = false;
-  if (tagged_sentence[i][0].indexOf(".") > -1) {
+  if (sentence.taggedWords[i].token.indexOf(".") > -1) {
     // url if there are two contiguous alpha characters
-    if (/[a-zA-Z]{2}/.test(tagged_sentence[i][0])) {
+    if (/[a-zA-Z]{2}/.test(sentence.taggedWords[i].token)) {
       is_url = true;
     }
   }
   return((parameter === "YES") ? is_url : !is_url);
 }
 
-function current_word_and_2_tag_after_are(tagged_sentence, i, parameter1, parameter2) {
-  if (i < tagged_sentence.length - 2) {
-    if (tagged_sentence[i+2][1] === parameter2) {
-      return(tagged_sentence[i][0] === parameter1);
+function current_word_and_2_tag_after_are(sentence, i, parameter1, parameter2) {
+  if (i < sentence.taggedWords.length - 2) {
+    if (sentence.taggedWords[i + 2][1] === parameter2) {
+      return(sentence.taggedWords[i].token === parameter1);
     }
     else {
       return(false);
@@ -361,69 +347,69 @@ function current_word_and_2_tag_after_are(tagged_sentence, i, parameter1, parame
 }
 
 function twoTagAfterParameterValues(sentence, i) {
-  if (i < sentence.length - 2) {
-    return [sentence[i + 2].tag];
+  if (i < sentence.taggedWords.length - 2) {
+    return [sentence.taggedWords[i + 2].tag];
   }
   else {
     return [];
   }
 }
 
-function current_word_and_next_tag_are(tagged_sentence, i, parameter1, parameter2) {
+function current_word_and_next_tag_are(sentence, i, parameter1, parameter2) {
   var next_tag = false;
   // check current word
-  var current_word = (tagged_sentence[i][0] === parameter1);
+  var current_word = (sentence.taggedWords[i].token === parameter1);
   // check next tag
-  if (i < tagged_sentence.length - 1) {
-    next_tag = (tagged_sentence[i+1][1] === parameter2);
+  if (i < sentence.taggedWords.length - 1) {
+    next_tag = (sentence.taggedWords[i+1].tag === parameter2);
   }
   return(current_word && next_tag);
 }
 
-function current_word_and_prev_tag_are(tagged_sentence, i, parameter1, parameter2) {
+function current_word_and_prev_tag_are(sentence, i, parameter1, parameter2) {
   var prev_tag = false;
   // check current word
-  var current_word = (tagged_sentence[i][0] === parameter2);
+  var current_word = (sentence.taggedWords[i].token === parameter2);
   // check prev tag
   if (i > 0) {
-    prev_tag = (tagged_sentence[i-1][1] === parameter1);
+    prev_tag = (sentence.taggedWords[i-1].tag === parameter1);
   }
   return(current_word && prev_tag);
 }
 
-function current_word_and_2_tag_before_are(tagged_sentence, i, parameter1, parameter2) {
+function current_word_and_2_tag_before_are(sentence, i, parameter1, parameter2) {
   var two_tags_before = false;
   // check current word
-  var current_word = (tagged_sentence[i][0] === parameter2);
+  var current_word = (sentence.taggedWords[i].token === parameter2);
   if (i > 1) {
     // check two tags before
-    two_tags_before = (tagged_sentence[i-2][1] === parameter1);
+    two_tags_before = (sentence.taggedWords[i - 2].tag === parameter1);
   }
   return(current_word && two_tags_before);
 }
 
 function twoTagBeforeParameterValues(sentence, i) {
   if (i > 1) {
-    return [sentence[i - 2].tag];
+    return [sentence.taggedWords[i - 2].tag];
   }
   else {
     return [];
   }
 }
 
-function current_word_and_2_after_are(tagged_sentence, i, parameter1, parameter2) {
+function current_word_and_2_after_are(sentence, i, parameter1, parameter2) {
   var two_words_after = false;
   // check current word
-  var current_word = (tagged_sentence[i][0] === parameter1);
-  if (i < tagged_sentence - 2) {
-    two_words_after = (tagged_sentence[i+2][0] === parameter2);
+  var current_word = (sentence.taggedWords[i].token === parameter1);
+  if (i < sentence.taggedWords.length - 2) {
+    two_words_after = (sentence.taggedWords[i+2].token === parameter2);
   }
   return(current_word && two_words_after);
 }
 
-function prev_word_is(tagged_sentence, i, parameter) {
+function prev_word_is(sentence, i, parameter) {
   if (i > 0) {
-    return(tagged_sentence[i-1][0].toLowerCase() === parameter.toLowerCase());
+    return(sentence.taggedWords[i - 1].token.toLowerCase() === parameter.toLowerCase());
   }
   else {
     return(false);
@@ -433,21 +419,21 @@ function prev_word_is(tagged_sentence, i, parameter) {
 // Returns the right value for parameter 1 of prev_word_is
 function prevWordParameterValues(sentence, i) {
   if (i > 0) {
-    return [sentence[i - 1].token];
+    return [sentence.taggedWords[i - 1].token];
   }
   else {
     return [];
   }
 }
 
-function prev_1_or_2_word_is(tagged_sentence, i, parameter) {
+function prev_1_or_2_word_is(sentence, i, parameter) {
   var prev_1 = false;
   var prev_2 = false;
   if (i > 0) {
-    prev_1 = (tagged_sentence[i-1][0].toLowerCase() === parameter.toLowerCase());
+    prev_1 = (sentence.taggedWords[i-1].token.toLowerCase() === parameter.toLowerCase());
   }
   if (i > 1) {
-    prev_2 = (tagged_sentence[i-2][0].toLowerCase() === parameter.toLowerCase());
+    prev_2 = (sentence.taggedWords[i-2].token.toLowerCase() === parameter.toLowerCase());
   }
   return(prev_1 || prev_2);
 }
@@ -465,8 +451,8 @@ function prev1Or2WordParameterValues(sentence, i) {
 
 // Indicates whether or not this string ends with the specified string.
 // Adapted from the original Javascript Brill tagger
-function current_word_ends_with(tagged_sentence, i, parameter) {
-  var word = tagged_sentence[i][0];
+function current_word_ends_with(sentence, i, parameter) {
+  var word = sentence.taggedWords[i].token;
   if (!parameter || (parameter.length > word.length)) {
     return false;
   }
@@ -480,85 +466,85 @@ function currentWordEndsWithParameterValues(sentence, i) {
   return values;
 }
 
-function right_bigram_is(tagged_sentence, i, parameter1, parameter2) {
-  var word_1 = (tagged_sentence[i][0] === parameter1);
+function right_bigram_is(sentence, i, parameter1, parameter2) {
+  var word_1 = (sentence.taggedWords[i].token === parameter1);
   var word_2 = false;
-  if (i < tagged_sentence.length - 1) {
-    word_2 = (tagged_sentence[i+1][0] === parameter2);
+  if (i < sentence.taggedWords.length - 1) {
+    word_2 = (sentence.taggedWords[i+1].token === parameter2);
   }
   return(word_1 && word_2);
 }
 
-function left_bigram_is(tagged_sentence, i, parameter1, parameter2) {
+function left_bigram_is(sentence, i, parameter1, parameter2) {
   var word_1 = false;
-  var word_2 = (tagged_sentence[i][0] === parameter2);
+  var word_2 = (sentence.taggedWords[i].token === parameter2);
   if (i > 0) {
-    word_1 = (tagged_sentence[i-1][0] === parameter1);
+    word_1 = (sentence.taggedWords[i-1].token === parameter1);
   }
   return(word_1 && word_2);
 }
 
-function next_bigram_is(tagged_sentence, i, parameter1, parameter2) {
+function next_bigram_is(sentence, i, parameter1, parameter2) {
   var word_1 = false;
   var word_2 = false;
-  if (i < tagged_sentence.length - 1) {
-    word_1 = (tagged_sentence[i+1][0] === parameter1);
+  if (i < sentence.taggedWords.length - 1) {
+    word_1 = (sentence.taggedWords[i + 1].token === parameter1);
   }
-  if (i < tagged_sentence.length - 2) {
-    word_2 = (tagged_sentence[i+2][0] === parameter2);
+  if (i < sentence.taggedWords.length - 2) {
+    word_2 = (sentence.taggedWords[i + 2].token === parameter2);
   }
   return(word_1 && word_2);
 }
 
 function twoWordAfterParameterValues(sentence, i) {
-  if (i < sentence.length - 2) {
-    return [sentence[i + 2].token];
+  if (i < sentence.taggedWords.length - 2) {
+    return [sentence.taggedWords[i + 2].token];
   }
   else {
     return [];
   }
 }
 
-function prev_bigram_is(tagged_sentence, i, parameter1, parameter2) {
+function prev_bigram_is(sentence, i, parameter1, parameter2) {
   var word_1 = false;
   var word_2 = false;
   if (i >  1) {
-    word_1 = (tagged_sentence[i-2][0] === parameter1);
+    word_1 = (sentence.taggedWords[i-2].token === parameter1);
   }
   if (i > 0) {
-    word_2 = (tagged_sentence[i-1][0] === parameter2);
+    word_2 = (sentence.taggedWords[i-1].token === parameter2);
   }
   return(word_1 && word_2);
 }
 
 function twoWordBeforeParameterValues(sentence, i) {
   if (i >  1) {
-    return [sentence[i - 2].token];
+    return [sentence.taggedWords[i - 2].token];
   }
   else {
     return [];
   }
 }
 
-function next_1_or_2_word_is(tagged_sentence, i, parameter1, parameter2) {
+function next_1_or_2_word_is(sentence, i, parameter1, parameter2) {
   next_1 = false;
   next_2 = false;
-  if (i < tagged_sentence.length - 1) {
-    next_1 = (tagged_sentence[i+1][0] === parameter1);
+  if (i < sentence.taggedWords.length - 1) {
+    next_1 = (sentence.taggedWords[i+1].token === parameter1);
   }
-  if (i < tagged_sentence.length - 2) {
-    next_2 = (tagged_sentence[i+2][0] === parameter2);
+  if (i < sentence.taggedWords.length - 2) {
+    next_2 = (sentence.taggedWords[i+2].token === parameter2);
   }
   return(next_1 || next_2);
 }
 
 function next1Or2WordParameterValues(sentence, i) {
   var values = [];
-  if (i < sentence.length - 1) {
-    values.push(sentence[i + 1].token);
+  if (i < sentence.taggedWords.length - 1) {
+    values.push(sentence.taggedWords[i + 1].token);
   }
-  if (i < sentence.length - 2) {
-    values.push(sentence[i + 2].token);
+  if (i < sentence.taggedWords.length - 2) {
+    values.push(sentence.taggedWords[i + 2].token);
   }
   return values;
 }
@@ -566,9 +552,9 @@ function next1Or2WordParameterValues(sentence, i) {
 // ==================================
 // Predicates about tags
 // ==================================
-function next_tag_is(tagged_sentence, i, parameter) {
-  if (i < tagged_sentence.length - 1) {
-    return(tagged_sentence[i+1][1] === parameter);
+function next_tag_is(sentence, i, parameter) {
+  if (i < sentence.taggedWords.length - 1) {
+    return(sentence.taggedWords[i + 1].tag === parameter);
   }
   else {
     return(false);
@@ -576,17 +562,17 @@ function next_tag_is(tagged_sentence, i, parameter) {
 }
 
 function nextTagParameterValues(sentence, i) {
-  if (i < sentence.length - 1) {
-    return [sentence[i + 1].tag];
+  if (i < sentence.taggedWords.length - 1) {
+    return [sentence.taggedWords[i + 1].tag];
   }
   else {
     return [];
   }
 }
 
-function next_2_tag_is(tagged_sentence, i, parameter) {
-  if (i < tagged_sentence.length - 2) {
-    return(tagged_sentence[i+2][1] === parameter);
+function next_2_tag_is(sentence, i, parameter) {
+  if (i < sentence.taggedWords.length - 2) {
+    return(sentence.taggedWords[i+2].tag === parameter);
   }
   else {
     return(false);
@@ -594,74 +580,74 @@ function next_2_tag_is(tagged_sentence, i, parameter) {
 }
 
 function next2TagParameterValues(sentence, i) {
-  if (i < sentence.length - 2) {
-    return [sentence[i+2].tag];
+  if (i < sentence.taggedWords.length - 2) {
+    return [sentence.taggedWords[i+2].tag];
   }
   else {
     return [];
   }
 }
 
-function next_1_or_2_tag_is(tagged_sentence, i, parameter) {
+function next_1_or_2_tag_is(sentence, i, parameter) {
   var next_1 = false;
   var next_2 = false;
-  if (i < tagged_sentence.length - 1) {
-    next_1 = (tagged_sentence[i+1][1] === parameter);
+  if (i < sentence.taggedWords.length - 1) {
+    next_1 = (sentence.taggedWords[i+1].tag === parameter);
   }
-  if (i < tagged_sentence.length - 2) {
-    next_2 = (tagged_sentence[i+2][1] === parameter);
+  if (i < sentence.taggedWords.length - 2) {
+    next_2 = (sentence.taggedWords[i+2].tag === parameter);
   }
   return(next_1 || next_2);
 }
 
 function next1Or2TagIsParameterValues(sentence, i) {
   var values = [];
-  if (i < sentence.length - 1) {
-    values.push(sentence[i + 1].tag);
+  if (i < sentence.taggedWords.length - 1) {
+    values.push(sentence.taggedWords[i + 1].tag);
   }
-  if (i < sentence.length - 2) {
-    values.push(sentence[i + 2].tag);
+  if (i < sentence.taggedWords.length - 2) {
+    values.push(sentence.taggedWords[i + 2].tag);
   }
   return values;
 }
 
-function next_1_or_2_or_3_tag(tagged_sentence, i, parameter) {
+function next_1_or_2_or_3_tag(sentence, i, parameter) {
   var next_1 = false;
   var next_2 = false;
   var next_3 = false;
-  if (i < tagged_sentence.length - 1) {
-    next_1 = (tagged_sentence[i+1][1] === parameter);
+  if (i < sentence.taggedWords.length - 1) {
+    next_1 = (sentence.taggedWords[i+1].tag === parameter);
   }
-  if (i < tagged_sentence.length - 2) {
-    next_2 = (tagged_sentence[i+2][1] === parameter);
+  if (i < sentence.taggedWords.length - 2) {
+    next_2 = (sentence.taggedWords[i+2].tag === parameter);
   }
-  if (i < tagged_sentence.length - 3) {
-    next_3 = (tagged_sentence[i+3][1] === parameter);
+  if (i < sentence.taggedWords.length - 3) {
+    next_3 = (sentence.taggedWords[i+3].tag === parameter);
   }
   return(next_1 || next_2 || next_3);
 }
 
 function next1Or2Or3TagParameterValues(sentence, i) {
   var values = [];
-  if (i < sentence.length - 1) {
-    values.push(sentence[i + 1].tag);
+  if (i < sentence.taggedWords.length - 1) {
+    values.push(sentence.taggedWords[i + 1].tag);
   }
-  if (i < sentence.length - 2) {
-    values.push(sentence[i + 2].tag);
+  if (i < sentence.taggedWords.length - 2) {
+    values.push(sentence.taggedWords[i + 2].tag);
   }
-  if (i < sentence.length - 3) {
-    values.push(sentence[i + 3].tag);
+  if (i < sentence.taggedWords.length - 3) {
+    values.push(sentence.taggedWords[i + 3].tag);
   }
   return values;
 }
 
-function surrounded_by_tags(tagged_sentence, i, parameter1, parameter2) {
-  if (i < tagged_sentence.length - 1) {
+function surrounded_by_tags(sentence, i, parameter1, parameter2) {
+  if (i < sentence.taggedWords.length - 1) {
     // check next tag
-    if (tagged_sentence[i+1][1] === parameter2) {
+    if (sentence.taggedWords[i+1].tag === parameter2) {
       // check previous tag
       if (i > 0) {
-        return(tagged_sentence[i-1][1] === parameter1)
+        return(sentence.taggedWords[i-1].tag === parameter1)
       }
       else {
         return(false);
@@ -676,18 +662,18 @@ function surrounded_by_tags(tagged_sentence, i, parameter1, parameter2) {
   }
 }
 
-function prev_1_or_2_or_3_tag(tagged_sentence, i, parameter) {
+function prev_1_or_2_or_3_tag(sentence, i, parameter) {
   var prev_1 = null;
   if (i > 0) {
-    prev_1 = tagged_sentence[i-1][1];
+    prev_1 = sentence.taggedWords[i-1].tag;
   }
   var prev_2 = null;
   if (i > 1) {
-    prev_2 = tagged_sentence[i-2][1];
+    prev_2 = sentence.taggedWords[i-2].tag;
   }
   var prev_3 = null;
   if (i > 2) {
-    prev_3 = tagged_sentence[i-3][1];
+    prev_3 = sentence.taggedWords[i-3].tag;
   }
   return((prev_1 === parameter) || (prev_2 === parameter) || (prev_3 === parameter));
 }
@@ -695,25 +681,25 @@ function prev_1_or_2_or_3_tag(tagged_sentence, i, parameter) {
 function prev1Or2Or3TagParameterValues(sentence, i) {
   var values = [];
   if (i > 0) {
-    values.push(sentence[i - 1].tag);
+    values.push(sentence.taggedWords[i - 1].tag);
   }
   if (i > 1) {
-    values.push(sentence[i - 2].tag);
+    values.push(sentence.taggedWords[i - 2].tag);
   }
   if (i > 2) {
-    values.push(sentence[i - 3].tag);
+    values.push(sentence.taggedWords[i - 3].tag);
   }
   return values;
 }
 
-function prev_1_or_2_tag(tagged_sentence, i, parameter) {
+function prev_1_or_2_tag(sentence, i, parameter) {
   var prev_1 = null;
   if (i > 0) {
-    prev_1 = tagged_sentence[i-1][1];
+    prev_1 = sentence.taggedWords[i - 1].tag;
   }
   var prev_2 = null;
   if (i > 1) {
-    prev_2 = tagged_sentence[i-2][1];
+    prev_2 = sentence.taggedWords[i - 2].tag;
   }
   return((prev_1 === parameter) || (prev_2 === parameter));
 }
@@ -721,25 +707,25 @@ function prev_1_or_2_tag(tagged_sentence, i, parameter) {
 function prev1Or2TagParameterValues(sentence, i) {
   values = [];
   if (i > 0) {
-    values.push(sentence[i - 1].tag);
+    values.push(sentence.taggedWords[i - 1].tag);
   }
   if (i > 1) {
-    values.push(sentence[i - 2].tag);
+    values.push(sentence.taggedWords[i - 2].tag);
   }
   return values;
 }
 
-function prev_tag_is(tagged_sentence, i, parameter) {
+function prev_tag_is(sentence, i, parameter) {
   var prev = false;
   if (i > 0) {
-    prev = (tagged_sentence[i-1][1] === parameter);
+    prev = (sentence.taggedWords[i-1].tag === parameter);
   }
   return(prev);
 }
 
 function prevTagParameterValues(sentence, i) {
   if (i > 0) {
-    return [sentence[i - 1].tag];
+    return [sentence.taggedWords[i - 1].tag];
   }
   else {
     return [];
@@ -748,19 +734,19 @@ function prevTagParameterValues(sentence, i) {
 
 // Looks like a useless predicate because transformation already take the
 // current tag into account
-function current_word_is_tag(tagged_sentence, i, parameter) {
-  return(tagged_sentence[i][1] === parameter);
+function current_word_is_tag(sentence, i, parameter) {
+  return(sentence.taggedWords[i].tag === parameter);
 }
 
-function prev_2_tag_is(tagged_sentence, i, parameter) {
+function prev_2_tag_is(sentence, i, parameter) {
   var prev_2 = false;
   if (i > 1) {
-    prev_2 = (tagged_sentence[i-2][1] === parameter);
+    prev_2 = (sentence.taggedWords[i-2].tag === parameter);
   }
   return(prev_2);
 }
 
-function default_predicate(tagged_sentence, i, parameter) {
+function default_predicate(sentence, i, parameter) {
   return(false);
 }
 

package/lib/natural/brill_pos_tagger/lib/Sentence.js

@@ -0,0 +1,41 @@
+/*
+  Sentence class that generates sample elements from sentences
+  Copyright (C) 2018 Hugo W.L. ter Doest
+
+  This program is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+var Context = require('../../classifiers/maxent/Context');
+
+function Sentence() {
+  this.taggedWords = [];
+}
+
+Sentence.prototype.addTaggedWord = function(token, tag) {
+  this.taggedWords.push({
+    "token": token,
+    "tag": tag
+  });
+};
+
+Sentence.prototype.clone = function() {
+  var s = new Sentence();
+  this.taggedWords.forEach(function(wordObject) {
+    s.addTaggedWord(wordObject.token, wordObject.tag);
+  });
+  return s;
+};
+
+module.exports = Sentence;