natural 8.0.0 → 8.1.0

package/lib/natural/tfidf/index.d.ts

@@ -43,6 +43,7 @@ export class TfIdf {
   constructor (deserialized?: Record<string, unknown>)
   idf (term: string, force?: boolean): number
   addDocument (document: string | string[] | Record<string, string>, key?: Record<string, any> | any, restoreCache?: boolean): void
+  removeDocument (key: any): boolean
   addFileSync (path: string, encoding?: string, key?: string, restoreCache?: boolean): void
   tfidf (terms: string | string[], d: number): number
   tfidfs (terms: string | string[], callback?: TfIdfCallback): number[]

package/lib/natural/tokenizers/sentence_tokenizer.js

@@ -31,7 +31,9 @@ const ABBREV = 'ABBREV'
 const DEBUG = false
 
 function generateUniqueCode (base, index) {
-  return `${base}_${index}`
+  // Surround the placeholder with {{}} to prevent shorter numbers to be recognized
+  // in larger numbers
+  return `{{${base}_${index}}}`
 }
 
 function escapeRegExp (string) {
@@ -39,13 +41,17 @@ function escapeRegExp (string) {
 }
 
 class SentenceTokenizer extends Tokenizer {
-  constructor (abbreviations, sentenceDemarkers) {
+  constructor (abbreviations, trimSentences) {
     super()
-    this.abbreviations = abbreviations
-    if (sentenceDemarkers) {
-      this.sentenceDemarkers = sentenceDemarkers
+    if (abbreviations) {
+      this.abbreviations = abbreviations
     } else {
-      this.sentenceDemarkers = ['.', '!', '?']
+      this.abbreviations = []
+    }
+    if (trimSentences === undefined) {
+      this.trimSentences = true
+    } else {
+      this.trimSentences = trimSentences
     }
     this.replacementMap = null
     this.replacementCounter = 0
@@ -64,7 +70,10 @@ class SentenceTokenizer extends Tokenizer {
   }
 
   replaceAbbreviations (text) {
-    const pattern = new RegExp(`(${this.abbreviations.map(abbrev => escapeRegExp(abbrev)).join('|')})`, 'g')
+    if (this.abbreviations.length === 0) {
+      return text
+    }
+    const pattern = new RegExp(`(${this.abbreviations.map(abbrev => escapeRegExp(abbrev)).join('|')})`, 'gi')
     const replacedText = text.replace(pattern, match => {
       const code = generateUniqueCode(ABBREV, this.replacementCounter++)
       this.replacementMap.set(code, match)
@@ -77,12 +86,11 @@ class SentenceTokenizer extends Tokenizer {
   replaceDelimitersWithPlaceholders (text) {
     // Regular expression for sentence delimiters optionally followed by a bracket or quote
     // Multiple delimiters with spaces in between are allowed
-    // The look ahead makes sure that there is punctuation symbol as next symbol
-    const delimiterPattern = /(?=[.?!…])([.?!… ]+)(["'”’)}\]]?)/g
-
-    const modifiedText = text.replace(delimiterPattern, (match, p1, p2) => {
+    // The expression makes sure that the sentence delimiter group ends with a sentence delimiter
+    const delimiterPattern = /([.?!… ]*)([.?!…])(["'”’)}\]]?)/g
+    const modifiedText = text.replace(delimiterPattern, (match, p1, p2, p3) => {
       const placeholder = generateUniqueCode(DELIM, this.replacementCounter++)
-      this.delimiterMap.set(placeholder, p1 + p2)
+      this.delimiterMap.set(placeholder, p1 + p2 + p3)
       return placeholder
     })
 
@@ -178,7 +186,7 @@ class SentenceTokenizer extends Tokenizer {
     const trimmedSentences = this.trim(newSentences)
     DEBUG && console.log('Phase 7: trimming array of empty sentences: ' + JSON.stringify(trimmedSentences))
 
-    const trimmedSentences2 = trimmedSentences.map(sent => sent.trim())
+    const trimmedSentences2 = trimmedSentences.map(sent => this.trimSentences ? sent.trim() : sent)
     DEBUG && console.log('Phase 8: trimming sentences from surrounding whitespace: ' + JSON.stringify(trimmedSentences2))
     DEBUG && console.log('---End of sentence tokenization--------------------------')
     DEBUG && console.log('---Replacement map---------------------------------------')

package/lib/natural/util/abbreviations_en.js

@@ -7,6 +7,8 @@ const knownAbbreviations = [
   'c/o',
   'dept.',
   'D.I.Y.',
+  'Dr.',
+  'e.g.',
   'est.',
   'E.T.A.',
   'Inc.',

package/lib/natural/util/index.js

@@ -23,6 +23,7 @@ THE SOFTWARE.
 'use strict'
 
 exports.stopwords = require('./stopwords').words
+exports.abbreviations = require('./abbreviations_en').knownAbbreviations
 exports.ShortestPathTree = require('./shortest_path_tree')
 exports.LongestPathTree = require('./longest_path_tree')
 exports.DirectedEdge = require('./directed_edge')

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "8.0.0",
+  "version": "8.1.0",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",