natural 6.10.0 → 6.10.3

package/lib/natural/brill_pos_tagger/lib/RuleTemplate.js

@@ -16,16 +16,18 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-function RuleTemplate (templateName, metadata) {
-  this.predicateName = templateName
-  this.meta = metadata
-}
+class RuleTemplate {
+  constructor (templateName, metadata) {
+    this.predicateName = templateName
+    this.meta = metadata
+  }
 
-RuleTemplate.prototype.windowFitsSite = function (sentence, i) {
-  return ((i + this.meta.window[0] >= 0) &&
-    (i + this.meta.window[0] < sentence.taggedWords.length) &&
-    (i + this.meta.window[1] >= 0) &&
-    (i + this.meta.window[1] < sentence.taggedWords.length))
+  windowFitsSite (sentence, i) {
+    return ((i + this.meta.window[0] >= 0) &&
+      (i + this.meta.window[0] < sentence.taggedWords.length) &&
+      (i + this.meta.window[1] >= 0) &&
+      (i + this.meta.window[1] < sentence.taggedWords.length))
+  }
 }
 
 module.exports = RuleTemplate

package/lib/natural/brill_pos_tagger/lib/Sentence.js

@@ -16,28 +16,29 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-'úse strict'
-
-function Sentence (data) {
-  this.taggedWords = []
-  if (data) {
-    this.taggedWords = data
+'use strict'
+
+class Sentence {
+  constructor (data) {
+    this.taggedWords = []
+    if (data) {
+      this.taggedWords = data
+    }
   }
-}
 
-Sentence.prototype.addTaggedWord = function (token, tag) {
-  this.taggedWords.push({
-    token: token,
-    tag: tag
-  })
-}
+  addTaggedWord (token, tag) {
+    this.taggedWords.push({
+      token: token,
+      tag: tag
+    })
+  }
 
-Sentence.prototype.clone = function () {
-  const s = new Sentence()
-  this.taggedWords.forEach(function (wordObject) {
-    s.addTaggedWord(wordObject.token, wordObject.tag)
-  })
-  return s
+  clone () {
+    const s = new Sentence()
+    this.taggedWords.forEach(function (wordObject) {
+      s.addTaggedWord(wordObject.token, wordObject.tag)
+    })
+    return s
+  }
 }
-
 module.exports = Sentence

package/lib/natural/brill_pos_tagger/lib/TransformationRule.js

@@ -20,80 +20,80 @@
 
 const Predicate = require('./Predicate')
 
-// logger.setLevel('INFO');
-
 const categoryWildCard = '*'
 
-function TransformationRule (c1, c2, predicate, parameter1, parameter2) {
-  this.literal = [c1, c2, predicate, parameter1, parameter2]
-  this.predicate = new Predicate(predicate, parameter1, parameter2)
-  this.oldCategory = c1
-  this.newCategory = c2
-  // These members are for the learning algorithm
-  this.neutral = 0
-  this.negative = 0
-  this.positive = 0
-  this.hasBeenSelectedAsHighRuleBefore = false
-}
+class TransformationRule {
+  constructor (c1, c2, predicate, parameter1, parameter2) {
+    this.literal = [c1, c2, predicate, parameter1, parameter2]
+    this.predicate = new Predicate(predicate, parameter1, parameter2)
+    this.oldCategory = c1
+    this.newCategory = c2
+    // These members are for the learning algorithm
+    this.neutral = 0
+    this.negative = 0
+    this.positive = 0
+    this.hasBeenSelectedAsHighRuleBefore = false
+  }
 
-TransformationRule.prototype.key = function () {
-  return (this.literal.toString())
-}
+  key () {
+    return (this.literal.toString())
+  }
 
-TransformationRule.prototype.apply = function (sentence, position) {
-  if ((sentence.taggedWords[position].tag === this.oldCategory) ||
-      (this.oldCategory === categoryWildCard)) {
-    if (this.predicate.evaluate(sentence, position)) {
-      sentence.taggedWords[position].tag = this.newCategory
+  apply (sentence, position) {
+    if ((sentence.taggedWords[position].tag === this.oldCategory) ||
+        (this.oldCategory === categoryWildCard)) {
+      if (this.predicate.evaluate(sentence, position)) {
+        sentence.taggedWords[position].tag = this.newCategory
+      }
     }
   }
-}
 
-//
-// Methods for processing sentences from a corpus that consist of an array of tokens
-//
+  //
+  // Methods for processing sentences from a corpus that consist of an array of tokens
+  //
 
-// Returns true if the rule applies at site. As a side effect it assigns the new
-// category to newTag
-TransformationRule.prototype.isApplicableAt = function (sentence, taggedSentence, i) {
-  const applies = (taggedSentence.taggedWords[i].tag === this.oldCategory) &&
-    this.predicate.evaluate(taggedSentence, i)
+  // Returns true if the rule applies at site. As a side effect it assigns the new
+  // category to newTag
+  isApplicableAt (sentence, taggedSentence, i) {
+    const applies = (taggedSentence.taggedWords[i].tag === this.oldCategory) &&
+      this.predicate.evaluate(taggedSentence, i)
 
-  // Set newTag to let the trainer know what the new tag would become
-  if (applies) {
-    sentence.taggedWords[i].newTag = this.newCategory
+    // Set newTag to let the trainer know what the new tag would become
+    if (applies) {
+      sentence.taggedWords[i].newTag = this.newCategory
+    }
+    return (applies)
   }
-  return (applies)
-}
 
-TransformationRule.prototype.prettyPrint = function () {
-  let result = ''
-  // Old category and new category
-  result += this.oldCategory + ' ' + this.newCategory
-  // Predicate name
-  result += ' ' + this.predicate.name
-  // Parameter 1 and 2
-  if (this.predicate.parameter1) {
-    result += ' ' + this.predicate.parameter1
-    if (this.predicate.parameter2) {
-      result += ' ' + this.predicate.parameter2
+  prettyPrint () {
+    let result = ''
+    // Old category and new category
+    result += this.oldCategory + ' ' + this.newCategory
+    // Predicate name
+    result += ' ' + this.predicate.name
+    // Parameter 1 and 2
+    if (this.predicate.parameter1) {
+      result += ' ' + this.predicate.parameter1
+      if (this.predicate.parameter2) {
+        result += ' ' + this.predicate.parameter2
+      }
     }
+    return result
   }
-  return result
-}
 
-// Applies the rule the given location (if it applies)
-TransformationRule.prototype.applyAt = function (sentence, i) {
-  const taggedSentence = sentence.clone()
+  // Applies the rule the given location (if it applies)
+  applyAt (sentence, i) {
+    const taggedSentence = sentence.clone()
 
-  this.apply(sentence, i)
-  // Assign the new tag to the corpus site
-  sentence.taggedWords[i].testTag = taggedSentence.taggedWords[i].tag
-}
+    this.apply(sentence, i)
+    // Assign the new tag to the corpus site
+    sentence.taggedWords[i].testTag = taggedSentence.taggedWords[i].tag
+  }
 
-// Calculate the net score of this rule
-TransformationRule.prototype.score = function () {
-  return (this.positive - this.negative)
+  // Calculate the net score of this rule
+  score () {
+    return (this.positive - this.negative)
+  }
 }
 
 module.exports = TransformationRule

package/lib/natural/classifiers/bayes_classifier.js

@@ -22,41 +22,36 @@ THE SOFTWARE.
 
 'use strict'
 
-const util = require('util')
 const Classifier = require('./classifier')
 const ApparatusBayesClassifier = require('apparatus').BayesClassifier
 
-const BayesClassifier = function (stemmer, smoothing) {
-  let abc = new ApparatusBayesClassifier()
-  if (smoothing && isFinite(smoothing)) {
-    abc = new ApparatusBayesClassifier(smoothing)
+class BayesClassifier extends Classifier {
+  constructor (stemmer, smoothing) {
+    let abc = new ApparatusBayesClassifier()
+    if (smoothing && isFinite(smoothing)) {
+      abc = new ApparatusBayesClassifier(smoothing)
+    }
+    super(abc, stemmer)
   }
-  Classifier.call(this, abc, stemmer)
-}
-
-util.inherits(BayesClassifier, Classifier)
 
-function restore (classifier, stemmer) {
-  classifier = Classifier.restore(classifier, stemmer)
-  // __proto__ is deprecated
-  // classifier.__proto__ = BayesClassifier.prototype
-  Object.setPrototypeOf(classifier, BayesClassifier.prototype)
-  classifier.classifier = ApparatusBayesClassifier.restore(classifier.classifier)
-
-  return classifier
-}
+  static restore (classifier, stemmer) {
+    classifier = Classifier.restore(classifier, stemmer)
+    // __proto__ is deprecated
+    // classifier.__proto__ = BayesClassifier.prototype
+    Object.setPrototypeOf(classifier, BayesClassifier.prototype)
+    classifier.classifier = ApparatusBayesClassifier.restore(classifier.classifier)
+    return classifier
+  }
 
-function load (filename, stemmer, callback) {
-  Classifier.load(filename, function (err, classifier) {
-    if (err) {
-      return callback(err)
-    } else {
-      callback(err, restore(classifier, stemmer))
-    }
-  })
+  load (filename, stemmer, callback) {
+    Classifier.load(filename, function (err, classifier) {
+      if (err) {
+        return callback(err)
+      } else {
+        callback(err, this.restore(classifier, stemmer))
+      }
+    })
+  }
 }
 
-BayesClassifier.restore = restore
-BayesClassifier.load = load
-
 module.exports = BayesClassifier

package/lib/natural/classifiers/classifier.js

@@ -22,164 +22,179 @@ THE SOFTWARE.
 
 'use strict'
 
-const PorterStemmer = require('../stemmers/porter_stemmer')
-const events = require('events')
+const EventEmitter = require('events')
 
+const PorterStemmer = require('../stemmers/porter_stemmer')
 const parallelTrainer = require('./classifier_train_parallel')
 
-const Classifier = function (classifier, stemmer) {
-  this.classifier = classifier
-  this.docs = []
-  this.features = {}
-  this.stemmer = stemmer || PorterStemmer
-  this.lastAdded = 0
-  this.events = new events.EventEmitter()
-}
+class Classifier extends EventEmitter {
+  constructor (classifier, stemmer) {
+    super()
+    this.classifier = classifier
+    this.docs = []
+    this.features = {}
+    this.stemmer = stemmer || PorterStemmer
+    this.lastAdded = 0
+  }
 
-function addDocument (text, classification) {
-  // Ignore further processing if classification is undefined
-  if (typeof classification === 'undefined') return
+  addDocument (text, classification) {
+    // Ignore further processing if classification is undefined
+    if (typeof classification === 'undefined') return
 
-  // If classification is type of string then make sure it's dosen't have blank space at both end
-  if (typeof classification === 'string') {
-    classification = classification.trim()
-  }
+    // If classification is type of string then make sure it's dosen't have blank space at both end
+    if (typeof classification === 'string') {
+      classification = classification.trim()
+    }
 
-  if (typeof text === 'string') { text = this.stemmer.tokenizeAndStem(text, this.keepStops) }
+    if (typeof text === 'string') { text = this.stemmer.tokenizeAndStem(text, this.keepStops) }
 
-  if (text.length === 0) {
-    // ignore empty documents
-    return
-  }
+    if (text.length === 0) {
+      // ignore empty documents
+      return
+    }
 
-  this.docs.push({
-    label: classification,
-    text: text
-  })
+    this.docs.push({
+      label: classification,
+      text: text
+    })
 
-  for (let i = 0; i < text.length; i++) {
-    const token = text[i]
-    this.features[token] = (this.features[token] || 0) + 1
+    for (let i = 0; i < text.length; i++) {
+      const token = text[i]
+      this.features[token] = (this.features[token] || 0) + 1
+    }
   }
-}
 
-function removeDocument (text, classification) {
-  const docs = this.docs
-  let doc
-  let pos
+  removeDocument (text, classification) {
+    const docs = this.docs
+    let doc
+    let pos
 
-  if (typeof text === 'string') {
-    text = this.stemmer.tokenizeAndStem(text, this.keepStops)
-  }
+    if (typeof text === 'string') {
+      text = this.stemmer.tokenizeAndStem(text, this.keepStops)
+    }
 
-  for (let i = 0, ii = docs.length; i < ii; i++) {
-    doc = docs[i]
-    if (doc.text.join(' ') === text.join(' ') &&
-        doc.label === classification) {
-      pos = i
+    for (let i = 0, ii = docs.length; i < ii; i++) {
+      doc = docs[i]
+      if (doc.text.join(' ') === text.join(' ') &&
+          doc.label === classification) {
+        pos = i
+      }
     }
-  }
 
-  // Remove if there's a match
-  if (!isNaN(pos)) {
-    this.docs.splice(pos, 1)
+    // Remove if there's a match
+    if (!isNaN(pos)) {
+      this.docs.splice(pos, 1)
 
-    for (let i = 0, ii = text.length; i < ii; i++) {
-      delete this.features[text[i]]
+      for (let i = 0, ii = text.length; i < ii; i++) {
+        delete this.features[text[i]]
+      }
     }
   }
-}
 
-function textToFeatures (observation) {
-  const features = []
+  textToFeatures (observation) {
+    const features = []
 
-  if (typeof observation === 'string') { observation = this.stemmer.tokenizeAndStem(observation, this.keepStops) }
+    if (typeof observation === 'string') { observation = this.stemmer.tokenizeAndStem(observation, this.keepStops) }
+
+    for (const feature in this.features) {
+      if (observation.indexOf(feature) > -1) {
+        features.push(1)
+      } else {
+        features.push(0)
+      }
+    }
 
-  for (const feature in this.features) {
-    if (observation.indexOf(feature) > -1) { features.push(1) } else { features.push(0) }
+    return features
   }
 
-  return features
-}
+  train () {
+    const totalDocs = this.docs.length
+    for (let i = this.lastAdded; i < totalDocs; i++) {
+      const features = this.textToFeatures(this.docs[i].text)
+      this.classifier.addExample(features, this.docs[i].label)
+      this.emit('trainedWithDocument', { index: i, total: totalDocs, doc: this.docs[i] })
+      this.lastAdded++
+    }
+    this.emit('doneTraining', true)
+    this.classifier.train()
+  }
 
-function train () {
-  const totalDocs = this.docs.length
-  for (let i = this.lastAdded; i < totalDocs; i++) {
-    const features = this.textToFeatures(this.docs[i].text)
-    this.classifier.addExample(features, this.docs[i].label)
-    this.events.emit('trainedWithDocument', { index: i, total: totalDocs, doc: this.docs[i] })
-    this.lastAdded++
+  retrain () {
+    this.classifier = new (this.classifier.constructor)()
+    this.lastAdded = 0
+    this.train()
   }
-  this.events.emit('doneTraining', true)
-  this.classifier.train()
-}
 
-function retrain () {
-  this.classifier = new (this.classifier.constructor)()
-  this.lastAdded = 0
-  this.train()
-}
+  getClassifications (observation) {
+    return this.classifier.getClassifications(this.textToFeatures(observation))
+  }
 
-function getClassifications (observation) {
-  return this.classifier.getClassifications(this.textToFeatures(observation))
-}
+  classify (observation) {
+    return this.classifier.classify(this.textToFeatures(observation))
+  }
 
-function classify (observation) {
-  return this.classifier.classify(this.textToFeatures(observation))
-}
+  static restore (classifier, stemmer) {
+    classifier.stemmer = stemmer || PorterStemmer
+    return classifier
+  }
 
-function restore (classifier, stemmer) {
-  classifier.stemmer = stemmer || PorterStemmer
-  classifier.events = new events.EventEmitter()
-  return classifier
-}
+  save (filename, callback) {
+    const data = JSON.stringify(this)
+    const fs = require('fs')
+    const classifier = this
+    fs.writeFile(filename, data, 'utf8', function (err) {
+      if (callback) {
+        callback(err, err ? null : classifier)
+      }
+    })
+  }
 
-function save (filename, callback) {
-  const data = JSON.stringify(this)
-  const fs = require('fs')
-  const classifier = this
-  fs.writeFile(filename, data, 'utf8', function (err) {
-    if (callback) {
-      callback(err, err ? null : classifier)
-    }
-  })
-}
+  static load (filename, callback) {
+    const fs = require('fs')
 
-function load (filename, callback) {
-  const fs = require('fs')
+    if (!callback) {
+      return
+    }
+    fs.readFile(filename, 'utf8', function (err, data) {
+      if (err) {
+        callback(err, null)
+      } else {
+        const classifier = JSON.parse(data)
+        callback(err, classifier)
+      }
+    })
+  }
 
-  if (!callback) {
-    return
+  setOptions (options) {
+    this.keepStops = !!(options.keepStops)
   }
-  fs.readFile(filename, 'utf8', function (err, data) {
-    if (err) {
-      callback(err, null)
+
+  ClassifiertrainParallel (numThreads, callback) {
+    if (parallelTrainer.Threads) {
+      return parallelTrainer.trainParallel(numThreads, callback)
     } else {
-      const classifier = JSON.parse(data)
-      callback(err, classifier)
+      this.emit('No threads available')
+      return this.train()
     }
-  })
-}
+  }
 
-function setOptions (options) {
-  this.keepStops = !!(options.keepStops)
-}
+  trainParallelBatches (options) {
+    if (parallelTrainer.Threads) {
+      return parallelTrainer.trainParallelBatches(options)
+    } else {
+      this.emit('No threads available')
+      return this.train()
+    }
+  }
 
-Classifier.prototype.addDocument = addDocument
-Classifier.prototype.removeDocument = removeDocument
-Classifier.prototype.train = train
-if (parallelTrainer.Threads) {
-  Classifier.prototype.trainParallel = parallelTrainer.trainParallel
-  Classifier.prototype.trainParallelBatches = parallelTrainer.trainParallelBatches
-  Classifier.prototype.retrainParallel = parallelTrainer.retrainParallel
+  retrainParallel (numThreads, callback) {
+    if (parallelTrainer.Threads) {
+      return parallelTrainer.trainParallel(numThreads, callback)
+    } else {
+      this.emit('No threads available')
+      return this.retrain()
+    }
+  }
 }
-Classifier.prototype.retrain = retrain
-Classifier.prototype.classify = classify
-Classifier.prototype.textToFeatures = textToFeatures
-Classifier.prototype.save = save
-Classifier.prototype.getClassifications = getClassifications
-Classifier.prototype.setOptions = setOptions
-Classifier.restore = restore
-Classifier.load = load
 
 module.exports = Classifier

package/lib/natural/classifiers/logistic_regression_classifier.js

@@ -22,46 +22,42 @@ THE SOFTWARE.
 
 'use strict'
 
-const util = require('util')
 const Classifier = require('./classifier')
 const ApparatusLogisticRegressionClassifier = require('apparatus').LogisticRegressionClassifier
 
-const LogisticRegressionClassifier = function (stemmer) {
-  Classifier.call(this, new ApparatusLogisticRegressionClassifier(), stemmer)
+class LogisticRegressionClassifier extends Classifier {
+  constructor (stemmer) {
+    const abc = new ApparatusLogisticRegressionClassifier()
+    super(abc, stemmer)
+  }
+
+  static restore (classifier, stemmer) {
+    classifier = Classifier.restore(classifier, stemmer)
+    // Using ___proto__ is deprecated
+    // classifier.__proto__ = LogisticRegressionClassifier.prototype
+    Object.setPrototypeOf(classifier, LogisticRegressionClassifier.prototype)
+    classifier.classifier = ApparatusLogisticRegressionClassifier.restore(classifier.classifier)
+
+    return classifier
+  }
+
+  static load (filename, stemmer, callback) {
+    Classifier.load(filename, function (err, classifier) {
+      if (err) {
+        callback(err)
+      } else {
+        callback(err, this.restore(classifier, stemmer))
+      }
+    })
+  }
+
+  train () {
+    // we need to reset the training state because logistic regression
+    // needs its matricies to have their widths synced, etc.
+    this.lastAdded = 0
+    this.classifier = new ApparatusLogisticRegressionClassifier()
+    Classifier.prototype.train.call(this)
+  }
 }
 
-util.inherits(LogisticRegressionClassifier, Classifier)
-
-function restore (classifier, stemmer) {
-  classifier = Classifier.restore(classifier, stemmer)
-  // Using ___proto__ is deprecated
-  // classifier.__proto__ = LogisticRegressionClassifier.prototype
-  Object.setPrototypeOf(classifier, LogisticRegressionClassifier.prototype)
-  classifier.classifier = ApparatusLogisticRegressionClassifier.restore(classifier.classifier)
-
-  return classifier
-}
-
-function load (filename, stemmer, callback) {
-  Classifier.load(filename, function (err, classifier) {
-    if (err) {
-      callback(err)
-    } else {
-      callback(err, restore(classifier, stemmer))
-    }
-  })
-}
-
-function train () {
-  // we need to reset the traning state because logistic regression
-  // needs its matricies to have their widths synced, etc.
-  this.lastAdded = 0
-  this.classifier = new ApparatusLogisticRegressionClassifier()
-  Classifier.prototype.train.call(this)
-}
-
-LogisticRegressionClassifier.prototype.train = train
-LogisticRegressionClassifier.restore = restore
-LogisticRegressionClassifier.load = load
-
 module.exports = LogisticRegressionClassifier