natural 7.0.2 → 7.0.5

package/.github/workflows/npm-publish.yml

@@ -16,6 +16,8 @@ jobs:
         with:
           node-version: 20
       - run: npm ci
+      - name: Build
+        run: npm run build
       - run: npm test
 
   publish-npm:

package/.nycrc

@@ -0,0 +1,9 @@
+{
+  "include": [
+    "io_spec/**/*_spec.js",
+    "dist/**/*_spec.js",
+    "lib/natural/**/*.js",
+    "lib/natural/**/*.ts"
+  ],
+  "reporter": "lcov"
+}

package/lib/natural/analyzers/sentence_analyzer.js

@@ -25,7 +25,7 @@ THE SOFTWARE.
 const _ = require('underscore')
 
 /*
- Sentences Analizer Class
+ Sentences Analyzer Class
  From http://www.writingcentre.uottawa.ca/hypergrammar/sntpurps.html
 
  Take a POS input and analyse it for
@@ -44,148 +44,150 @@ const _ = require('underscore')
   - Show Preposition Phrases
 */
 
-const Sentences = function (pos, callbackFunction) {
-  this.posObj = pos
-  this.senType = null
-  callbackFunction(this)
-}
+class SentenceAnalyzer {
+  constructor (pos, callbackFunction) {
+    this.posObj = pos
+    this.senType = null
+    callbackFunction(this)
+  }
 
-Sentences.prototype.part = function (callbackFunction) {
-  const subject = []
-  const predicat = []
-  let verbFound = false
+  part (callbackFunction) {
+    const subject = []
+    const predicat = []
+    let verbFound = false
 
-  this.prepositionPhrases()
+    this.prepositionPhrases()
 
-  for (let i = 0; i < this.posObj.tags.length; i++) {
-    if (this.posObj.tags[i].pos === 'VB') {
-      if (i === 0) {
-        verbFound = true
-      } else {
-        // We need to Test for any EX before the VB
-        if (this.posObj.tags[i - 1].pos !== 'EX') {
+    for (let i = 0; i < this.posObj.tags.length; i++) {
+      if (this.posObj.tags[i].pos === 'VB') {
+        if (i === 0) {
           verbFound = true
         } else {
-          predicat.push(this.posObj.tags[i].token)
+          // We need to Test for any EX before the VB
+          if (this.posObj.tags[i - 1].pos !== 'EX') {
+            verbFound = true
+          } else {
+            predicat.push(this.posObj.tags[i].token)
+          }
         }
       }
-    }
 
-    // Add Pronoun Phrase (pp) Or Subject Phrase (sp)
-    if (!verbFound) {
-      if (this.posObj.tags[i].pp !== true) { this.posObj.tags[i].spos = 'SP' }
+      // Add Pronoun Phrase (pp) Or Subject Phrase (sp)
+      if (!verbFound) {
+        if (this.posObj.tags[i].pp !== true) { this.posObj.tags[i].spos = 'SP' }
 
-      subject.push(this.posObj.tags[i].token)
-    } else {
-      if (this.posObj.tags[i].pp !== true) { this.posObj.tags[i].spos = 'PP' }
+        subject.push(this.posObj.tags[i].token)
+      } else {
+        if (this.posObj.tags[i].pp !== true) { this.posObj.tags[i].spos = 'PP' }
 
-      predicat.push(this.posObj.tags[i].token)
+        predicat.push(this.posObj.tags[i].token)
+      }
     }
-  }
 
-  if (subject.length === 0) {
-    this.posObj.tags.push({ token: 'You', spos: 'SP', pos: 'PRP', added: true })
-  }
+    if (subject.length === 0) {
+      this.posObj.tags.push({ token: 'You', spos: 'SP', pos: 'PRP', added: true })
+    }
 
-  callbackFunction(this)
-}
+    callbackFunction(this)
+  }
 
-// Takes POS and removes IN to NN or NNS
-// Adds a PP for each prepositionPhrases
-Sentences.prototype.prepositionPhrases = function () {
-  let remove = false
+  // Takes POS and removes IN to NN or NNS
+  // Adds a PP for each prepositionPhrases
+  prepositionPhrases () {
+    let remove = false
 
-  for (let i = 0; i < this.posObj.tags.length; i++) {
-    if (this.posObj.tags[i].pos.match('IN')) {
-      remove = true
-    }
+    for (let i = 0; i < this.posObj.tags.length; i++) {
+      if (this.posObj.tags[i].pos.match('IN')) {
+        remove = true
+      }
 
-    if (remove) {
-      this.posObj.tags[i].pp = true
-    }
+      if (remove) {
+        this.posObj.tags[i].pp = true
+      }
 
-    if (this.posObj.tags[i].pos.match('NN')) {
-      remove = false
+      if (this.posObj.tags[i].pos.match('NN')) {
+        remove = false
+      }
     }
   }
-}
 
-Sentences.prototype.subjectToString = function () {
-  return this.posObj.tags.map(function (t) {
-    if (t.spos === 'SP' || t.spos === 'S') {
-      return t.token
-    } else return null
-  }).join(' ')
-}
+  subjectToString () {
+    return this.posObj.tags.map(function (t) {
+      if (t.spos === 'SP' || t.spos === 'S') {
+        return t.token
+      } else return null
+    }).join(' ')
+  }
 
-Sentences.prototype.predicateToString = function () {
-  return this.posObj.tags.map(function (t) {
-    if (t.spos === 'PP' || t.spos === 'P') {
-      return t.token
-    } else return null
-  }).join(' ')
-}
+  predicateToString () {
+    return this.posObj.tags.map(function (t) {
+      if (t.spos === 'PP' || t.spos === 'P') {
+        return t.token
+      } else return null
+    }).join(' ')
+  }
 
-Sentences.prototype.implicitYou = function () {
-  for (let i = 0; i < this.posObj.tags.length; i++) {
-    if (this.posObj.tags[i].added) {
-      return true
+  implicitYou () {
+    for (let i = 0; i < this.posObj.tags.length; i++) {
+      if (this.posObj.tags[i].added) {
+        return true
+      }
     }
+
+    return false
   }
 
-  return false
-}
+  toString () {
+    return this.posObj.tags.map(function (t) { return t.token }).join(' ')
+  }
 
-Sentences.prototype.toString = function () {
-  return this.posObj.tags.map(function (t) { return t.token }).join(' ')
-}
+  // This is quick and incomplete.
+  type (cbf) {
+    const callbackFunction = cbf || false
 
-// This is quick and incomplete.
-Sentences.prototype.type = function (cbf) {
-  const callbackFunction = cbf || false
-
-  // Check for implicit you before popping a tag.
-  const implicitYou = this.implicitYou()
-
-  // FIXME - punct seems useless
-  let lastElement = this.posObj.punct()
-  // console.log(lastElement)
-  lastElement = (lastElement.length !== 0) ? lastElement.pop() : this.posObj.tags.pop()
-  // console.log(lastElement)
-
-  if (lastElement.pos !== '.') {
-    if (implicitYou) {
-      this.senType = 'COMMAND'
-    } else if (_.contains(['WDT', 'WP', 'WP$', 'WRB'], this.posObj.tags[0].pos)) {
-      // Sentences that start with: who, what where when why and how, then they are questions
-      this.senType = 'INTERROGATIVE'
-    } else if (_.contains(['PRP'], lastElement.pos)) {
-      // Sentences that end in a Personal pronoun are most likely questions
-      // eg. We should run away, should we [?]
-      // eg. You want to see that again, do you [?]
-      this.senType = 'INTERROGATIVE'
-    } else {
-      this.senType = 'UNKNOWN'
-    }
-  } else {
-    switch (lastElement.token) {
-      case '?':
+    // Check for implicit you before popping a tag.
+    const implicitYou = this.implicitYou()
+
+    // FIXME - punct seems useless
+    let lastElement = this.posObj.punct()
+    // console.log(lastElement)
+    lastElement = (lastElement.length !== 0) ? lastElement.pop() : this.posObj.tags.pop()
+    // console.log(lastElement)
+
+    if (lastElement.pos !== '.') {
+      if (implicitYou) {
+        this.senType = 'COMMAND'
+      } else if (_.contains(['WDT', 'WP', 'WP$', 'WRB'], this.posObj.tags[0].pos)) {
+        // Sentences that start with: who, what where when why and how, then they are questions
+        this.senType = 'INTERROGATIVE'
+      } else if (_.contains(['PRP'], lastElement.pos)) {
+        // Sentences that end in a Personal pronoun are most likely questions
+        // eg. We should run away, should we [?]
+        // eg. You want to see that again, do you [?]
         this.senType = 'INTERROGATIVE'
-        break
-      case '!':
-        this.senType = (implicitYou) ? 'COMMAND' : 'EXCLAMATORY'
-        break
-      case '.':
-        this.senType = (implicitYou) ? 'COMMAND' : 'DECLARATIVE'
-        break
+      } else {
+        this.senType = 'UNKNOWN'
+      }
+    } else {
+      switch (lastElement.token) {
+        case '?':
+          this.senType = 'INTERROGATIVE'
+          break
+        case '!':
+          this.senType = (implicitYou) ? 'COMMAND' : 'EXCLAMATORY'
+          break
+        case '.':
+          this.senType = (implicitYou) ? 'COMMAND' : 'DECLARATIVE'
+          break
+      }
     }
-  }
 
-  if (callbackFunction && _.isFunction(callbackFunction)) {
-    callbackFunction(this)
-  } else {
-    return this.senType
+    if (callbackFunction && _.isFunction(callbackFunction)) {
+      callbackFunction(this)
+    } else {
+      return this.senType
+    }
   }
 }
 
-module.exports = Sentences
+module.exports = SentenceAnalyzer

package/lib/natural/spellcheck/spellcheck.js

@@ -27,79 +27,81 @@ const Trie = require('../trie/trie')
 // Words that are an edit distance of n away from the mispelled word are considered infinitely more probable than words that are of an edit distance >n
 
 // wordlist is a corpus (an array) from which word probabilities are calculated (so something like /usr/share/dict/words (on OSX) will work okay, but real world text will work better)
-function Spellcheck (wordlist) {
-  this.trie = new Trie()
-  this.trie.addStrings(wordlist)
-  this.word2frequency = {}
-  for (const i in wordlist) {
-    if (!this.word2frequency[wordlist[i]]) {
-      this.word2frequency[wordlist[i]] = 0
+class Spellcheck {
+  constructor (wordlist) {
+    this.trie = new Trie()
+    this.trie.addStrings(wordlist)
+    this.word2frequency = {}
+    for (const i in wordlist) {
+      if (!this.word2frequency[wordlist[i]]) {
+        this.word2frequency[wordlist[i]] = 0
+      }
+      this.word2frequency[wordlist[i]]++
     }
-    this.word2frequency[wordlist[i]]++
   }
-}
 
-Spellcheck.prototype.isCorrect = function (word) {
-  return this.trie.contains(word)
-}
+  isCorrect (word) {
+    return this.trie.contains(word)
+  }
 
-// Returns a list of suggested corrections, from highest to lowest probability
-// maxDistance is the maximum edit distance
-// According to Norvig, literature suggests that 80% to 95% of spelling errors are an edit distance of 1 away from the correct word. This is good, because there are roughly 54n+25 strings 1 edit distance away from any given string of length n. So after maxDistance = 2, this becomes very slow.
-Spellcheck.prototype.getCorrections = function (word, maxDistance) {
-  const self = this
-  if (!maxDistance) maxDistance = 1
-  let edits = this.editsWithMaxDistance(word, maxDistance)
-  edits = edits.slice(1, edits.length)
-  edits = edits.map(function (editList) {
-    return editList.filter(function (word) { return self.isCorrect(word) })
-      .map(function (word) { return [word, self.word2frequency[word]] })
-      .sort(function (a, b) { return a[1] > b[1] ? -1 : 1 })
-      .map(function (wordscore) { return wordscore[0] })
-  })
-  let flattened = []
-  for (const i in edits) {
-    if (edits[i].length) {
-      flattened = flattened.concat(edits[i])
+  // Returns a list of suggested corrections, from highest to lowest probability
+  // maxDistance is the maximum edit distance
+  // According to Norvig, literature suggests that 80% to 95% of spelling errors are an edit distance of 1 away from the correct word. This is good, because there are roughly 54n+25 strings 1 edit distance away from any given string of length n. So after maxDistance = 2, this becomes very slow.
+  getCorrections (word, maxDistance) {
+    const self = this
+    if (!maxDistance) maxDistance = 1
+    let edits = this.editsWithMaxDistance(word, maxDistance)
+    edits = edits.slice(1, edits.length)
+    edits = edits.map(function (editList) {
+      return editList.filter(function (word) { return self.isCorrect(word) })
+        .map(function (word) { return [word, self.word2frequency[word]] })
+        .sort(function (a, b) { return a[1] > b[1] ? -1 : 1 })
+        .map(function (wordscore) { return wordscore[0] })
+    })
+    let flattened = []
+    for (const i in edits) {
+      if (edits[i].length) {
+        flattened = flattened.concat(edits[i])
+      }
     }
+    return flattened.filter(function (v, i, a) {
+      return a.indexOf(v) === i
+    })
   }
-  return flattened.filter(function (v, i, a) {
-    return a.indexOf(v) === i
-  })
-}
 
-// Returns all edits that are 1 edit-distance away from the input word
-Spellcheck.prototype.edits = function (word) {
-  const alphabet = 'abcdefghijklmnopqrstuvwxyz'
-  let edits = []
-  for (let i = 0; i < word.length + 1; i++) {
-    if (i > 0) edits.push(word.slice(0, i - 1) + word.slice(i, word.length)) // deletes
-    if (i > 0 && i < word.length + 1) edits.push(word.slice(0, i - 1) + word.slice(i, i + 1) + word.slice(i - 1, i) + word.slice(i + 1, word.length)) // transposes
-    for (let k = 0; k < alphabet.length; k++) {
-      if (i > 0) edits.push(word.slice(0, i - 1) + alphabet[k] + word.slice(i, word.length)) // replaces
-      edits.push(word.slice(0, i) + alphabet[k] + word.slice(i, word.length)) // inserts
+  // Returns all edits that are 1 edit-distance away from the input word
+  edits (word) {
+    const alphabet = 'abcdefghijklmnopqrstuvwxyz'
+    let edits = []
+    for (let i = 0; i < word.length + 1; i++) {
+      if (i > 0) edits.push(word.slice(0, i - 1) + word.slice(i, word.length)) // deletes
+      if (i > 0 && i < word.length + 1) edits.push(word.slice(0, i - 1) + word.slice(i, i + 1) + word.slice(i - 1, i) + word.slice(i + 1, word.length)) // transposes
+      for (let k = 0; k < alphabet.length; k++) {
+        if (i > 0) edits.push(word.slice(0, i - 1) + alphabet[k] + word.slice(i, word.length)) // replaces
+        edits.push(word.slice(0, i) + alphabet[k] + word.slice(i, word.length)) // inserts
+      }
     }
+    // Deduplicate edits
+    edits = edits.filter(function (v, i, a) { return a.indexOf(v) === i })
+    return edits
   }
-  // Deduplicate edits
-  edits = edits.filter(function (v, i, a) { return a.indexOf(v) === i })
-  return edits
-}
 
-// Returns all edits that are up to "distance" edit distance away from the input word
-Spellcheck.prototype.editsWithMaxDistance = function (word, distance) {
-  return this.editsWithMaxDistanceHelper(distance, [[word]])
-}
+  // Returns all edits that are up to "distance" edit distance away from the input word
+  editsWithMaxDistance (word, distance) {
+    return this.editsWithMaxDistanceHelper(distance, [[word]])
+  }
 
-Spellcheck.prototype.editsWithMaxDistanceHelper = function (distanceCounter, distance2edits) {
-  if (distanceCounter === 0) return distance2edits
-  const currentDepth = distance2edits.length - 1
-  const words = distance2edits[currentDepth]
-  // const edits = this.edits(words[0])
-  distance2edits[currentDepth + 1] = []
-  for (const i in words) {
-    distance2edits[currentDepth + 1] = distance2edits[currentDepth + 1].concat(this.edits(words[i]))
+  editsWithMaxDistanceHelper (distanceCounter, distance2edits) {
+    if (distanceCounter === 0) return distance2edits
+    const currentDepth = distance2edits.length - 1
+    const words = distance2edits[currentDepth]
+    // const edits = this.edits(words[0])
+    distance2edits[currentDepth + 1] = []
+    for (const i in words) {
+      distance2edits[currentDepth + 1] = distance2edits[currentDepth + 1].concat(this.edits(words[i]))
+    }
+    return this.editsWithMaxDistanceHelper(distanceCounter - 1, distance2edits)
   }
-  return this.editsWithMaxDistanceHelper(distanceCounter - 1, distance2edits)
 }
 
 module.exports = Spellcheck

package/lib/natural/util/bag.js

@@ -22,31 +22,33 @@ THE SOFTWARE.
 
 'use strict'
 
-function Bag () {
-  this.dictionary = []
-  this.nElement = 0
-};
-
-Bag.prototype.add = function (element) {
-  this.dictionary.push(element)
-  this.nElement++
-  return this
-}
-
-Bag.prototype.isEmpty = function () {
-  return this.nElement > 0
-}
-
-Bag.prototype.contains = function (item) {
-  return this.dictionary.indexOf(item) >= 0
-}
-
-/**
- * unpack the bag , and get all items
- */
-Bag.prototype.unpack = function () {
-  // return a copy is better than original
-  return this.dictionary.slice()
+class Bag {
+  constructor () {
+    this.dictionary = []
+    this.nElement = 0
+  }
+
+  add (element) {
+    this.dictionary.push(element)
+    this.nElement++
+    return this
+  }
+
+  isEmpty () {
+    return this.nElement > 0
+  }
+
+  contains (item) {
+    return this.dictionary.indexOf(item) >= 0
+  }
+
+  /**
+   * unpack the bag , and get all items
+  */
+  unpack () {
+    // return a copy is better than original
+    return this.dictionary.slice()
+  }
 }
 
 module.exports = Bag

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "7.0.2",
+  "version": "7.0.5",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -77,10 +77,9 @@
     "benchmark": "node benchmarks",
     "build": "tsc && ncp spec/test_data dist",
     "clean": "rimraf *~ #* *classifier.json dist io_spec/tmp/*.json coverage",
-    "test": "cross-env NODE_PATH=. jasmine --random=false spec/*_spec.js",
     "test_io": "jasmine --random=false io_spec/*_spec.js",
-    "test_ts": "cross-env NODE_PATH=.:./dist jasmine --random=false dist/ts_spec/*_spec.js",
-    "coverage": "nyc --reporter=lcov npm run test && nyc --reporter=lcov npm run test_io && nyc --reporter=lcov npm run test_ts",
+    "test": "cross-env NODE_PATH=.:./dist jasmine --random=false dist/spec/*_spec.js",
+    "coverage": "nyc --no-clean npm run test && nyc --no-clean npm run test_io",
     "test_browser": "cross-env NODE_PATH=. node ./node_modules/gulp/bin/gulp.js",
     "lint": "eslint . --ext .ts"
   },

package/tsconfig.json

@@ -17,7 +17,7 @@
   },
   "include": [
     "lib/natural/**/*.ts",
-    "ts_spec/*.ts",
+    "spec/*.ts",
     "io_spec/*.ts"
 , "lib/natural/analyzers/SenType.ts"  ],
   "exclude": [

package/ts_spec/MaxEntClassifier_spec.ts

@@ -1,130 +0,0 @@
-/*
-Unit test of Classifier
-Copyright (C) 2018 Hugo W.L. ter Doest
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-'use strict'
-
-import {
-  Context,
-  FeatureSet,
-  Sample,
-  MaxEntClassifier as Classifier,
-  SEElement
-} from 'lib/natural'
-
-import type { Element } from 'lib/natural'
-
-const classifierFilename = 'classifier.json'
-const minImprovement = 0.01
-const nrIterations = 20
-
-let sample: Sample
-let featureSet: FeatureSet
-let classifier: Classifier
-
-const DEBUG = false
-
-describe('The MaxEnt module', function () {
-  it('The Sample class creates a sample', function () {
-    sample = new Sample()
-    sample.addElement(new SEElement('x', new Context('0')))
-    sample.addElement(new SEElement('x', new Context('0')))
-    sample.addElement(new SEElement('x', new Context('0')))
-    sample.addElement(new SEElement('y', new Context('0')))
-    sample.addElement(new SEElement('y', new Context('0')))
-    sample.addElement(new SEElement('y', new Context('0')))
-
-    sample.addElement(new SEElement('x', new Context('1')))
-    sample.addElement(new SEElement('y', new Context('1')))
-    sample.addElement(new SEElement('y', new Context('1')))
-    sample.addElement(new SEElement('y', new Context('1')))
-
-    expect(sample.size()).toBe(10)
-  })
-
-  it('The FeatureSet class creates a feature set', function () {
-    featureSet = new FeatureSet()
-    sample.generateFeatures(featureSet)
-
-    expect(featureSet.size()).toBe(2)
-  })
-
-  it('The Classifier class creates a classifier', function () {
-    // Create a classifier
-    classifier = new Classifier(featureSet, sample)
-
-    expect(classifier).toBeDefined()
-  })
-
-  it('Classifier does not need a correction feature', function () {
-
-  })
-
-  it('The classifier stops training after a specified number or iterations ' +
-    'or when the minimum improvement in likelihood is reached', function () {
-    classifier.train(nrIterations, minImprovement)
-
-    expect(classifier.scaler.iteration).toBeLessThan(nrIterations + 1)
-    if (classifier.scaler.iteration < nrIterations) {
-      expect(classifier.scaler.improvement).toBeLessThan(minImprovement)
-    }
-  })
-
-  it('Save classifer to a file', function (done) {
-    classifier.save(classifierFilename, function (err, c) {
-      if (err !== null) {
-        console.log(err)
-      } else {
-        DEBUG && console.log('Classifier saved to ' + classifierFilename)
-      }
-      done()
-    })
-  })
-
-  let newClassifier: Classifier
-  it('Load classifer', function (done) {
-    classifier.load(classifierFilename, SEElement as unknown as Element, function (err, c) {
-      if (err !== null) {
-        console.log(err)
-      } else if (c !== undefined) {
-        DEBUG && console.log('Classifier loaded from ' + classifierFilename)
-        newClassifier = c
-      }
-      done()
-    })
-    if (newClassifier !== undefined) {
-      classifier = newClassifier
-    }
-  })
-
-  it('The classifier classifies events', function () {
-    let context = new Context('0')
-    DEBUG && console.log('Classes plus scores ' + JSON.stringify(classifier.getClassifications(context)))
-    let classification = classifier.classify(context)
-    expect(classification).toBe('x')
-
-    context = new Context('1')
-    DEBUG && console.log('Classes plus scores ' + JSON.stringify(classifier.getClassifications(context)))
-    classification = classifier.classify(context)
-    expect(classification).toBe('y')
-  })
-})