natural 6.10.2 → 6.10.4

package/lib/natural/brill_pos_tagger/lib/Corpus.js

@@ -26,165 +26,167 @@ const JSON = 2
 // sentences: an array of annotated sentences
 // A sentence is an array of annotated tokens
 // A token is an object with (token, tag, testTag, ruleList)
-function Corpus (data, typeOfCorpus, SentenceClass) {
-  this.wordCount = 0
-  this.sentences = []
-  const that = this
-  if (data) {
-    // For other types of corpora add a case here and supply a parsing method
-    switch (typeOfCorpus) {
-      case BROWN:
-        this.parseBrownCorpus(data, SentenceClass)
-        break
-      case JSON:
-        // Assume it is a JSON object of a corpus
-        data.sentences.forEach(function (s) {
-          const taggedSentence = new SentenceClass(s.taggedWords)
-          that.sentences.push(taggedSentence)
-          that.wordCount += s.taggedWords.length
-        })
-        break
+class Corpus {
+  constructor (data, typeOfCorpus, SentenceClass) {
+    this.wordCount = 0
+    this.sentences = []
+    const that = this
+    if (data) {
+      // For other types of corpora add a case here and supply a parsing method
+      switch (typeOfCorpus) {
+        case BROWN:
+          this.parseBrownCorpus(data, SentenceClass)
+          break
+        case JSON:
+          // Assume it is a JSON object of a corpus
+          data.sentences.forEach(function (s) {
+            const taggedSentence = new SentenceClass(s.taggedWords)
+            that.sentences.push(taggedSentence)
+            that.wordCount += s.taggedWords.length
+          })
+          break
+      }
     }
   }
-}
-
-// data is raw text
-// A corpus parsing method should split the corpus in sentences each of which
-// consist of an array of tokens.
-Corpus.prototype.parseBrownCorpus = function (data, SentenceClass) {
-  const that = this
-
-  const lines = data.split('\n')
-  lines.forEach(function (line) {
-    const trimmedLine = line.trim()
-    // Only parse lines that contain characters
-    if (trimmedLine !== '') {
-      const taggedSentence = new SentenceClass()
-      const tokens = line.trim().split(/\s+/)
-      tokens.forEach(function (token) {
-        that.wordCount++
-        // Create a tagged sentences consisting of tokens
-        const wordPlusTag = token.split('_')
-        taggedSentence.addTaggedWord(wordPlusTag[0], wordPlusTag[1])
-      })
 
-      // Add the sentence to the corpus
-      that.sentences.push(taggedSentence)
-    }
-  })
-}
+  // data is raw text
+  // A corpus parsing method should split the corpus in sentences each of which
+  // consist of an array of tokens.
+  parseBrownCorpus (data, SentenceClass) {
+    const that = this
+
+    const lines = data.split('\n')
+    lines.forEach(function (line) {
+      const trimmedLine = line.trim()
+      // Only parse lines that contain characters
+      if (trimmedLine !== '') {
+        const taggedSentence = new SentenceClass()
+        const tokens = line.trim().split(/\s+/)
+        tokens.forEach(function (token) {
+          that.wordCount++
+          // Create a tagged sentences consisting of tokens
+          const wordPlusTag = token.split('_')
+          taggedSentence.addTaggedWord(wordPlusTag[0], wordPlusTag[1])
+        })
 
-// Returns an array of all POS tags used in the corpus
-Corpus.prototype.getTags = function () {
-  return Object.keys(this.posTags)
-}
+        // Add the sentence to the corpus
+        that.sentences.push(taggedSentence)
+      }
+    })
+  }
 
-// Splits the corpus in a training and testing set.
-// percentageTrain is the size of the training corpus in percent
-// Returns an array with two elements: training corpus, testing corpus
-Corpus.prototype.splitInTrainAndTest = function (percentageTrain) {
-  const corpusTrain = new Corpus()
-  const corpusTest = new Corpus()
-
-  const p = percentageTrain / 100
-  this.sentences.forEach(function (sentence, i) {
-    if (Math.random() < p) {
-      corpusTrain.sentences.push(sentence)
-    } else {
-      corpusTest.sentences.push(sentence)
-    }
-  })
-  return [corpusTrain, corpusTest]
-}
+  // Returns an array of all POS tags used in the corpus
+  getTags () {
+    return Object.keys(this.posTags)
+  }
 
-// Analyses the corpus:
-// - registers used POS tags
-// - records the frequency of POS tag for each word
-Corpus.prototype.analyse = function () {
-  this.tagFrequencies = {}
-  this.posTags = {}
-  this.wordCount = 0
-
-  const that = this
-  this.sentences.forEach(function (sentence) {
-    sentence.taggedWords.forEach(function (token) {
-      that.wordCount++
-
-      // Register the tags used in the corpus
-      that.posTags[token.tag] = true
-
-      // Register the frequency of the tag
-      if (!that.tagFrequencies[token.token]) {
-        that.tagFrequencies[token.token] = {}
-      }
-      if (!that.tagFrequencies[token.token][token.tag]) {
-        that.tagFrequencies[token.token][token.tag] = 0
+  // Splits the corpus in a training and testing set.
+  // percentageTrain is the size of the training corpus in percent
+  // Returns an array with two elements: training corpus, testing corpus
+  splitInTrainAndTest (percentageTrain) {
+    const corpusTrain = new Corpus()
+    const corpusTest = new Corpus()
+
+    const p = percentageTrain / 100
+    this.sentences.forEach(function (sentence, i) {
+      if (Math.random() < p) {
+        corpusTrain.sentences.push(sentence)
+      } else {
+        corpusTest.sentences.push(sentence)
       }
-      that.tagFrequencies[token.token][token.tag]++
     })
-  })
-}
+    return [corpusTrain, corpusTest]
+  }
 
-// Creates a lexicon by taking the most frequently occurring tag of a word
-// as the right tag
-Corpus.prototype.buildLexicon = function () {
-  const lexicon = new Lexicon()
-  const that = this
+  // Analyses the corpus:
+  // - registers used POS tags
+  // - records the frequency of POS tag for each word
+  analyse () {
+    this.tagFrequencies = {}
+    this.posTags = {}
+    this.wordCount = 0
+
+    const that = this
+    this.sentences.forEach(function (sentence) {
+      sentence.taggedWords.forEach(function (token) {
+        that.wordCount++
 
-  this.analyse()
-  Object.keys(this.tagFrequencies).forEach(function (token) {
-    const catToFreq = that.tagFrequencies[token]
-    const categories = Object.keys(catToFreq)
+        // Register the tags used in the corpus
+        that.posTags[token.tag] = true
 
-    function compareByFrequency (a, b) {
-      if (catToFreq[a] > catToFreq[b]) {
-        return -1
-      } else {
-        if (catToFreq[a] < catToFreq[b]) {
-          return 1
+        // Register the frequency of the tag
+        if (!that.tagFrequencies[token.token]) {
+          that.tagFrequencies[token.token] = {}
+        }
+        if (!that.tagFrequencies[token.token][token.tag]) {
+          that.tagFrequencies[token.token][token.tag] = 0
+        }
+        that.tagFrequencies[token.token][token.tag]++
+      })
+    })
+  }
+
+  // Creates a lexicon by taking the most frequently occurring tag of a word
+  // as the right tag
+  buildLexicon () {
+    const lexicon = new Lexicon()
+    const that = this
+
+    this.analyse()
+    Object.keys(this.tagFrequencies).forEach(function (token) {
+      const catToFreq = that.tagFrequencies[token]
+      const categories = Object.keys(catToFreq)
+
+      function compareByFrequency (a, b) {
+        if (catToFreq[a] > catToFreq[b]) {
+          return -1
         } else {
-          return 0
+          if (catToFreq[a] < catToFreq[b]) {
+            return 1
+          } else {
+            return 0
+          }
         }
       }
-    }
 
-    const sortedCategories = categories.sort(compareByFrequency)
-    lexicon.addWord(token, sortedCategories)
-  })
-  return lexicon
-}
+      const sortedCategories = categories.sort(compareByFrequency)
+      lexicon.addWord(token, sortedCategories)
+    })
+    return lexicon
+  }
 
-Corpus.prototype.tag = function (lexicon) {
-  this.sentences.forEach(function (sentence) {
-    sentence.taggedWords.forEach(function (token) {
-      // tagWord returns a list of categories, take the first category
-      token.testTag = lexicon.tagWord(token.token)[0]
+  tag (lexicon) {
+    this.sentences.forEach(function (sentence) {
+      sentence.taggedWords.forEach(function (token) {
+        // tagWord returns a list of categories, take the first category
+        token.testTag = lexicon.tagWord(token.token)[0]
+      })
     })
-  })
-}
+  }
 
-Corpus.prototype.nrSentences = function () {
-  return this.sentences.length
-}
+  nrSentences () {
+    return this.sentences.length
+  }
 
-Corpus.prototype.nrWords = function () {
-  return this.wordCount
-}
+  nrWords () {
+    return this.wordCount
+  }
 
-Corpus.prototype.generateFeatures = function () {
-  let features = []
-  this.sentences.forEach(function (sentence) {
-    features = sentence.generateFeatures(features)
-  })
-  // console.log(JSON.stringify(features));
-  return features
-}
+  generateFeatures () {
+    let features = []
+    this.sentences.forEach(function (sentence) {
+      features = sentence.generateFeatures(features)
+    })
+    // console.log(JSON.stringify(features));
+    return features
+  }
 
-Corpus.prototype.prettyPrint = function () {
-  this.sentences.forEach(function (sentence, index) {
-    // logger.debug("sentence no " + index + "\n" +
-    //  JSON.stringify(sentence, null, 2));
-  })
+  prettyPrint () {
+    this.sentences.forEach(function (sentence, index) {
+      // logger.debug("sentence no " + index + "\n" +
+      //  JSON.stringify(sentence, null, 2));
+    })
+  }
 }
 
 module.exports = Corpus

package/lib/natural/brill_pos_tagger/lib/Lexicon.js

@@ -23,97 +23,99 @@ const dutchLexicon = require('../data/Dutch/brill_Lexicon.json')
 
 const DEBUG = false
 
-// Constructor creates a Lexicon for language
-function Lexicon (language, defaultCategory, defaultCategoryCapitalised, extendedLexicon) {
-  const lexicon = (() => {
-    switch (language) {
-      case 'EN':
-        return englishLexicon
-      case 'DU':
-        return dutchLexicon
-      default:
-        return dutchLexicon
+class Lexicon {
+  // Constructor creates a Lexicon for language
+  constructor (language, defaultCategory, defaultCategoryCapitalised, extendedLexicon) {
+    const lexicon = (() => {
+      switch (language) {
+        case 'EN':
+          return englishLexicon
+        case 'DU':
+          return dutchLexicon
+        default:
+          return dutchLexicon
+      }
+    })()
+
+    this.lexicon = Object.assign(lexicon, extendedLexicon || {})
+
+    if (defaultCategory) {
+      this.defaultCategory = defaultCategory
+      if (defaultCategoryCapitalised) {
+        this.defaultCategoryCapitalised = defaultCategoryCapitalised
+      }
     }
-  })()
+  }
 
-  this.lexicon = Object.assign(lexicon, extendedLexicon || {})
+  // Parses a lexicon in text format: word cat1 cat2 ... catn
+  parseLexicon (data) {
+    // Split into an array of non-empty lines
+    const arrayOfLines = data.match(/[^\r\n]+/g)
+    this.lexicon = {} // Object.create(null);
+    const that = this
+    arrayOfLines.forEach(function (line) {
+      // Split line by whitespace
+      const elements = line.trim().split(/\s+/)
+      if (elements.length > 0) {
+        that.lexicon[elements[0]] = elements.slice(1)
+      }
+    })
+  }
 
-  if (defaultCategory) {
-    this.defaultCategory = defaultCategory
-    if (defaultCategoryCapitalised) {
-      this.defaultCategoryCapitalised = defaultCategoryCapitalised
+  tagWordWithDefaults (word) {
+    if (/[A-Z]/.test(word[0]) && this.defaultCategoryCapitalised) {
+      // Capitalised
+      return this.defaultCategoryCapitalised
+    } else {
+      // If not found assign default_category
+      return this.defaultCategory
     }
   }
-}
 
-// Parses a lexicon in text format: word cat1 cat2 ... catn
-Lexicon.prototype.parseLexicon = function (data) {
-  // Split into an array of non-empty lines
-  const arrayOfLines = data.match(/[^\r\n]+/g)
-  this.lexicon = {} // Object.create(null);
-  const that = this
-  arrayOfLines.forEach(function (line) {
-    // Split line by whitespace
-    const elements = line.trim().split(/\s+/)
-    if (elements.length > 0) {
-      that.lexicon[elements[0]] = elements.slice(1)
+  // Returns a list of categories for word
+  tagWord (word) {
+    let categories = this.lexicon[word]
+    DEBUG && console.log(categories)
+    if (!categories || (typeof categories === 'function')) {
+      categories = this.lexicon[word.toLowerCase()]
     }
-  })
-}
-
-Lexicon.prototype.tagWordWithDefaults = function (word) {
-  if (/[A-Z]/.test(word[0]) && this.defaultCategoryCapitalised) {
-    // Capitalised
-    return this.defaultCategoryCapitalised
-  } else {
-    // If not found assign default_category
-    return this.defaultCategory
+    if (!categories || (typeof categories === 'function')) {
+      categories = [this.tagWordWithDefaults(word)]
+    }
+    return (categories)
   }
-}
 
-// Returns a list of categories for word
-Lexicon.prototype.tagWord = function (word) {
-  let categories = this.lexicon[word]
-  DEBUG && console.log(categories)
-  if (!categories || (typeof categories === 'function')) {
-    categories = this.lexicon[word.toLowerCase()]
+  // Adds a word to the lexicon. NB simply replaces the entry
+  addWord (word, categories) {
+    this.lexicon[word] = categories
   }
-  if (!categories || (typeof categories === 'function')) {
-    categories = [this.tagWordWithDefaults(word)]
-  }
-  return (categories)
-}
 
-// Adds a word to the lexicon. NB simply replaces the entry
-Lexicon.prototype.addWord = function (word, categories) {
-  this.lexicon[word] = categories
-}
-
-Lexicon.prototype.prettyPrint = function () {
-  let result = ''
-  const that = this
-  Object.keys(this.lexicon).forEach(function (token) {
-    result += token + '\t'
-    that.lexicon[token].forEach(function (cat) {
-      result += cat + '\t'
+  prettyPrint () {
+    let result = ''
+    const that = this
+    Object.keys(this.lexicon).forEach(function (token) {
+      result += token + '\t'
+      that.lexicon[token].forEach(function (cat) {
+        result += cat + '\t'
+      })
+      result += '\n'
     })
-    result += '\n'
-  })
-  return result
-}
+    return result
+  }
 
-Lexicon.prototype.nrEntries = function () {
-  return Object.keys(this.lexicon).length
-}
+  nrEntries () {
+    return Object.keys(this.lexicon).length
+  }
 
-Lexicon.prototype.size = function () {
-  return this.nrEntries()
-}
+  size () {
+    return this.nrEntries()
+  }
 
-Lexicon.prototype.setDefaultCategories = function (category, categoryCapitalised) {
-  this.defaultCategory = category
-  if (categoryCapitalised) {
-    this.defaultCategoryCapitalised = categoryCapitalised
+  setDefaultCategories (category, categoryCapitalised) {
+    this.defaultCategory = category
+    if (categoryCapitalised) {
+      this.defaultCategoryCapitalised = categoryCapitalised
+    }
   }
 }
 

package/lib/natural/brill_pos_tagger/lib/Predicate.js

@@ -23,25 +23,27 @@ const DEBUG = false
 const predicates = require('./RuleTemplates')
 DEBUG && console.log('RuleSet loaded predicates: ' + predicates)
 
-function Predicate (name, parameter1, parameter2) {
-  this.name = name
-  this.meta = predicates[name]
-  if (!this.meta) {
-    this.meta = predicates.DEFAULT
+class Predicate {
+  constructor (name, parameter1, parameter2) {
+    this.name = name
+    this.meta = predicates[name]
+    if (!this.meta) {
+      this.meta = predicates.DEFAULT
+    }
+    // if (this.meta.nrParameters > 0) {
+    this.parameter1 = parameter1
+    // }
+    // if (this.meta.nrParameters > 1) {
+    this.parameter2 = parameter2
+    // }
+    DEBUG && console.log('Predicate\n' + JSON.toString(this.meta, null, 2))
   }
-  // if (this.meta.nrParameters > 0) {
-  this.parameter1 = parameter1
-  // }
-  // if (this.meta.nrParameters > 1) {
-  this.parameter2 = parameter2
-  // }
-  DEBUG && console.log('Predicate\n' + JSON.toString(this.meta, null, 2))
-}
 
-Predicate.prototype.evaluate = function (sentence, position) {
-  DEBUG && console.log('Predicate.evalute: ' + this.name)
-  const predicate = this.meta.function
-  return (predicate(sentence, position, this.parameter1, this.parameter2))
+  evaluate (sentence, position) {
+    DEBUG && console.log('Predicate.evalute: ' + this.name)
+    const predicate = this.meta.function
+    return (predicate(sentence, position, this.parameter1, this.parameter2))
+  }
 }
 
 module.exports = Predicate

package/lib/natural/brill_pos_tagger/lib/RuleSet.js

@@ -25,72 +25,74 @@ const englishRuleSet = require('../data/English/tr_from_posjs.json')
 
 const DEBUG = false
 
-// Constructor takes a language abbreviation and loads the right rule set
-function RuleSet (language) {
-  let data = englishRuleSet
-  DEBUG && console.log(data)
-  switch (language) {
-    case 'EN':
-      data = englishRuleSet
-      break
-    case 'DU':
-      data = dutchRuleSet
-      break
+class RuleSet {
+  // Constructor takes a language abbreviation and loads the right rule set
+  constructor (language) {
+    let data = englishRuleSet
+    DEBUG && console.log(data)
+    switch (language) {
+      case 'EN':
+        data = englishRuleSet
+        break
+      case 'DU':
+        data = dutchRuleSet
+        break
+    }
+    if (data.rules) {
+      this.rules = {}
+      const that = this
+      data.rules.forEach(function (ruleString) {
+        that.addRule(TFParser.parse(ruleString))
+      })
+    }
+    DEBUG && console.log(this.rules)
+    DEBUG && console.log('Brill_POS_Tagger.read_transformation_rules: number of transformation rules read: ' + Object.keys(this.rules).length)
   }
-  if (data.rules) {
-    this.rules = {}
-    const that = this
-    data.rules.forEach(function (ruleString) {
-      that.addRule(TFParser.parse(ruleString))
-    })
-  }
-  DEBUG && console.log(this.rules)
-  DEBUG && console.log('Brill_POS_Tagger.read_transformation_rules: number of transformation rules read: ' + Object.keys(this.rules).length)
-}
 
-RuleSet.prototype.addRule = function (rule) {
-  // this.rules.push(rule);
-  if (!this.rules[rule.key()]) {
-    this.rules[rule.key()] = rule
-    return true
-  } else {
-    return false
+  addRule (rule) {
+    // this.rules.push(rule);
+    if (!this.rules[rule.key()]) {
+      this.rules[rule.key()] = rule
+      return true
+    } else {
+      return false
+    }
   }
-}
 
-RuleSet.prototype.removeRule = function (rule) {
-  if (this.rules[rule.key()]) {
-    delete this.rules[rule.key()]
+  removeRule (rule) {
+    if (this.rules[rule.key()]) {
+      delete this.rules[rule.key()]
+    }
   }
-}
 
-RuleSet.prototype.getRules = function () {
-  const that = this
-  return Object.keys(this.rules).map(function (key) {
-    return that.rules[key]
-  })
-}
+  getRules () {
+    const that = this
+    return Object.keys(this.rules).map(function (key) {
+      return that.rules[key]
+    })
+  }
 
-RuleSet.prototype.nrRules = function () {
-  return Object.keys(this.rules).length
-}
+  nrRules () {
+    return Object.keys(this.rules).length
+  }
 
-RuleSet.prototype.hasRule = function (rule) {
-  if (this.rules[rule.key()]) {
-    return true
-  } else {
-    return false
+  hasRule (rule) {
+    if (this.rules[rule.key()]) {
+      return true
+    } else {
+      return false
+    }
   }
-}
 
-RuleSet.prototype.prettyPrint = function () {
-  let result = ''
-  const that = this
-  Object.keys(this.rules).forEach(function (key) {
-    const rule = that.rules[key]
-    result += rule.prettyPrint() + '\n'
-  })
-  return result
+  prettyPrint () {
+    let result = ''
+    const that = this
+    Object.keys(this.rules).forEach(function (key) {
+      const rule = that.rules[key]
+      result += rule.prettyPrint() + '\n'
+    })
+    return result
+  }
 }
 
 module.exports = RuleSet