natural 6.10.0 → 6.10.2

package/lib/natural/sentiment/SentimentAnalyzer.js

@@ -22,7 +22,7 @@
 
 'use strict'
 
-const DEBUG = true
+const DEBUG = false
 
 // Afinn
 const englishAfinnVoca = require('afinn-165')

package/lib/natural/tokenizers/aggressive_tokenizer.js

@@ -23,22 +23,18 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    // underscore is considered to be non word character
+    // Old splitter:
+    // return this.trim(text.split(/[\W|_]+/))
+    // New splitter:
+    // Explicitly mentions which characters make up words.
+    // So words may contain hyphen -, single quote ' and slash /
+    return this.trim(text.split(/[^a-zA-Z0-9'\-/]+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  // underscore is considered to be non word character
-  // Old splitter:
-  // return this.trim(text.split(/[\W|_]+/))
-  // New splitter:
-  // Explicitly mentions which characters make up words.
-  // So words may contain hyphen -, single quote ' and slash /
-  return this.trim(text.split(/[^a-zA-Z0-9'\-/]+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_es.js

@@ -23,16 +23,12 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.trim(text.split(/[^a-zA-Zá-úÁ-ÚñÑüÜ]+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.trim(text.split(/[^a-zA-Zá-úÁ-ÚñÑüÜ]+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_fa.js

@@ -24,27 +24,23 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  clearEmptyString (array) {
+    return array.filter(function (a) {
+      return a !== ''
+    })
+  }
+
+  clearText (text) {
+    return text.replace(/.:\+-=\(\)"'!\?،,؛;/g, ' ')
+  }
+
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    text = this.clearText(text)
+    return this.clearEmptyString(text.split(/\s+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.clearEmptyString = function (array) {
-  return array.filter(function (a) {
-    return a !== ''
-  })
-}
-
-AggressiveTokenizer.prototype.clearText = function (text) {
-  return text.replace(/.:\+-=\(\)"'!\?،,؛;/g, ' ')
-}
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  text = this.clearText(text)
-  return this.clearEmptyString(text.split(/\s+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_fr.js

@@ -23,16 +23,12 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.trim(text.split(/[^a-z0-9äâàéèëêïîöôùüûœç-]+/i))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.trim(text.split(/[^a-z0-9äâàéèëêïîöôùüûœç-]+/i))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_hi.js

@@ -28,13 +28,6 @@ DEALINGS IN THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
-
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
-}
-
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 /*
 To know more on hindi
@@ -44,9 +37,11 @@ Important links:
 3.https://www.unicode.org/charts/PDF/U0900.pdf
 */
 
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  const response = this.trim(text.replace(/[\u0964\u0965...?,]/g, '').split(/\s+|(?![\u0900-\u097F\u0020-\u007F])./u)).filter(Boolean)
-  return response
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    const response = this.trim(text.replace(/[\u0964\u0965...?,]/g, '').split(/\s+|(?![\u0900-\u097F\u0020-\u007F])./u)).filter(Boolean)
+    return response
+  }
 }
 
 module.exports = AggressiveTokenizer

package/lib/natural/tokenizers/aggressive_tokenizer_id.js

@@ -23,24 +23,20 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  // Remove all non alphanumeric characters except '-'
+  // Replace more than one space character to ' '
+  normalizeText (text) {
+    const result = text.replace(/[^a-z0-9 -]/g, ' ').replace(/( +)/g, ' ')
+    return result
+  }
+
+  tokenize (text) {
+    // break a string up into an array of tokens by space
+    text = this.normalizeText(text)
+    return this.trim(text.split(' '))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-// Remove all non alphanumeric characters except '-'
-// Replace more than one space character to ' '
-function normalizeText (text) {
-  const result = text.replace(/[^a-z0-9 -]/g, ' ').replace(/( +)/g, ' ')
-  return result
-}
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by space
-  text = normalizeText(text)
-  return this.trim(text.split(' '))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_it.js

@@ -23,16 +23,12 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.trim(text.split(/\W+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.trim(text.split(/\W+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_nl.js

@@ -23,16 +23,12 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.trim(text.split(/[^a-zA-Z0-9_'-]+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.trim(text.split(/[^a-zA-Z0-9_'-]+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_no.js

@@ -24,18 +24,14 @@ THE SOFTWARE.
 
 const Tokenizer = require('./tokenizer')
 const normalizer = require('../normalizers/normalizer_no')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    text = normalizer.removeDiacritics(text)
+
+    // break a string up into an array of tokens by anything non-word
+    return this.trim(text.split(/[^A-Za-z0-9_æøåÆØÅäÄöÖüÜ]+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  text = normalizer.removeDiacritics(text)
-
-  // break a string up into an array of tokens by anything non-word
-  return this.trim(text.split(/[^A-Za-z0-9_æøåÆØÅäÄöÖüÜ]+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_pl.js

@@ -23,25 +23,20 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
-}
-
-util.inherits(AggressiveTokenizer, Tokenizer)
+class AggressiveTokenizer extends Tokenizer {
+  withoutEmpty (array) {
+    return array.filter(function (a) { return a })
+  }
 
-module.exports = AggressiveTokenizer
+  clearText (text) {
+    return text.replace(/[^a-zążśźęćńół0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
+  }
 
-AggressiveTokenizer.prototype.withoutEmpty = function (array) {
-  return array.filter(function (a) { return a })
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.withoutEmpty(this.clearText(text).split(' '))
+  }
 }
 
-AggressiveTokenizer.prototype.clearText = function (text) {
-  return text.replace(/[^a-zążśźęćńół0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
-}
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.withoutEmpty(this.clearText(text).split(' '))
-}
+module.exports = AggressiveTokenizer

package/lib/natural/tokenizers/aggressive_tokenizer_pt.js

@@ -23,20 +23,16 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
-}
-util.inherits(AggressiveTokenizer, Tokenizer)
-
-module.exports = AggressiveTokenizer
+class AggressiveTokenizer extends Tokenizer {
+  withoutEmpty (array) {
+    return array.filter(function (a) { return a })
+  }
 
-AggressiveTokenizer.prototype.withoutEmpty = function (array) {
-  return array.filter(function (a) { return a })
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.withoutEmpty(this.trim(text.split(/[^a-zA-Zà-úÀ-Ú]/)))
+  }
 }
 
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.withoutEmpty(this.trim(text.split(/[^a-zA-Zà-úÀ-Ú]/)))
-}
+module.exports = AggressiveTokenizer

package/lib/natural/tokenizers/aggressive_tokenizer_ru.js

@@ -23,25 +23,20 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
-}
-
-util.inherits(AggressiveTokenizer, Tokenizer)
+class AggressiveTokenizer extends Tokenizer {
+  withoutEmpty (array) {
+    return array.filter(function (a) { return a })
+  }
 
-module.exports = AggressiveTokenizer
+  clearText (text) {
+    return text.replace(/[^a-zа-яё0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
+  }
 
-AggressiveTokenizer.prototype.withoutEmpty = function (array) {
-  return array.filter(function (a) { return a })
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.withoutEmpty(this.clearText(text).split(' '))
+  }
 }
 
-AggressiveTokenizer.prototype.clearText = function (text) {
-  return text.replace(/[^a-zа-яё0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
-}
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.withoutEmpty(this.clearText(text).split(' '))
-}
+module.exports = AggressiveTokenizer

package/lib/natural/tokenizers/aggressive_tokenizer_sv.js

@@ -24,19 +24,15 @@ THE SOFTWARE.
 
 const Tokenizer = require('./tokenizer')
 const normalizer = require('../normalizers/normalizer_sv')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
+class AggressiveTokenizer extends Tokenizer {
+  tokenize (text) {
+    text = normalizer.removeDiacritics(text)
+
+    // break a string up into an array of tokens by anything non-word
+    // Ü is not part of swedish alphabet but there are words using it like müsli and München
+    return this.trim(text.split(/[^A-Za-z0-9_åÅäÄöÖüÜ-]+/))
+  }
 }
-util.inherits(AggressiveTokenizer, Tokenizer)
 
 module.exports = AggressiveTokenizer
-
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  text = normalizer.removeDiacritics(text)
-
-  // break a string up into an array of tokens by anything non-word
-  // Ü is not part of swedish alphabet but there are words using it like müsli and München
-  return this.trim(text.split(/[^A-Za-z0-9_åÅäÄöÖüÜ-]+/))
-}

package/lib/natural/tokenizers/aggressive_tokenizer_uk.js

@@ -1,25 +1,43 @@
-'use strict'
+/*
+Copyright (c) 2023, Pluto Rotegott
+Tokenizer for Ukrainian
 
-const Tokenizer = require('./tokenizer')
-const util = require('util')
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
-}
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
-util.inherits(AggressiveTokenizer, Tokenizer)
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
 
-module.exports = AggressiveTokenizer
+'use strict'
 
-AggressiveTokenizer.prototype.withoutEmpty = function (array) {
-  return array.filter(function (a) { return a })
-}
+const Tokenizer = require('./tokenizer')
 
-AggressiveTokenizer.prototype.clearText = function (text) {
-  return text.replace(/[^a-zа-яґєії0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
-}
+class AggressiveTokenizer extends Tokenizer {
+  withoutEmpty (array) {
+    return array.filter(function (a) { return a })
+  }
+
+  clearText (text) {
+    return text.replace(/[^a-zа-яґєії0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
+  }
 
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  // break a string up into an array of tokens by anything non-word
-  return this.withoutEmpty(this.clearText(text).split(' '))
+  tokenize (text) {
+    // break a string up into an array of tokens by anything non-word
+    return this.withoutEmpty(this.clearText(text).split(' '))
+  }
 }
+
+module.exports = AggressiveTokenizer

package/lib/natural/tokenizers/aggressive_tokenizer_vi.js

@@ -23,17 +23,12 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const AggressiveTokenizer = function () {
-  Tokenizer.call(this)
-}
-
-util.inherits(AggressiveTokenizer, Tokenizer)
-
-// break a string up into an array of tokens by anything non-word
-AggressiveTokenizer.prototype.tokenize = function (text) {
-  return this.trim(text.split(/[^a-z0-9àáảãạăắằẳẵặâấầẩẫậéèẻẽẹêếềểễệíìỉĩịóòỏõọôốồổỗộơớờởỡợúùủũụưứừửữựýỳỷỹỵđ]+/i))
+class AggressiveTokenizer extends Tokenizer {
+  // break a string up into an array of tokens by anything non-word
+  tokenize (text) {
+    return this.trim(text.split(/[^a-z0-9àáảãạăắằẳẵặâấầẩẫậéèẻẽẹêếềểễệíìỉĩịóòỏõọôốồổỗộơớờởỡợúùủũụưứừửữựýỳỷỹỵđ]+/i))
+  }
 }
 
 module.exports = AggressiveTokenizer

package/lib/natural/tokenizers/regexp_tokenizer.js

@@ -22,34 +22,35 @@ THE SOFTWARE.
 
 'use strict'
 
+const DEBUG = false
+
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 const _ = require('underscore')
 
-// Base Class for RegExp Matching
-const RegexpTokenizer = function (opts) {
-  const options = opts || {}
-  this._pattern = options.pattern || this._pattern
-  this.discardEmpty = options.discardEmpty || true
+class RegexpTokenizer extends Tokenizer {
+  constructor (opts) {
+    super(opts)
+    const options = opts || {}
+    this._pattern = options.pattern || this._pattern
+    this.discardEmpty = options.discardEmpty || true
 
-  // Match and split on GAPS not the actual WORDS
-  this._gaps = options.gaps
+    // Match and split on GAPS not the actual WORDS
+    this._gaps = options.gaps
 
-  if (this._gaps === undefined) {
-    this._gaps = true
+    if (this._gaps === undefined) {
+      this._gaps = true
+    }
   }
-}
-
-util.inherits(RegexpTokenizer, Tokenizer)
 
-RegexpTokenizer.prototype.tokenize = function (s) {
-  let results
+  tokenize (s) {
+    let results
 
-  if (this._gaps) {
-    results = s.split(this._pattern)
-    return (this.discardEmpty) ? _.without(results, '', ' ') : results
-  } else {
-    return s.match(this._pattern)
+    if (this._gaps) {
+      results = s.split(this._pattern)
+      return (this.discardEmpty) ? _.without(results, '', ' ') : results
+    } else {
+      return s.match(this._pattern)
+    }
   }
 }
 
@@ -57,21 +58,24 @@ exports.RegexpTokenizer = RegexpTokenizer
 
 const orthographyMatchers = require('./orthography_matchers')
 
-/***
- * A tokenizer that accepts an alphabet definition.
- * @param {string} options.language ISO 639-1 for the language, e.g. 'en'
- */
-const OrthographyTokenizer = function (options) {
-  const pattern = orthographyMatchers[options.language]
-  if (!pattern) {
-    WordTokenizer.call(this, options)
-  } else {
-    this._pattern = pattern
-    RegexpTokenizer.call(this, options)
+class OrthographyTokenizer {
+  constructor (options) {
+    const pattern = orthographyMatchers[options.language]
+    DEBUG && console.log(pattern)
+
+    if (!pattern) {
+      this.tokenizer = new WordTokenizer()
+    } else {
+      this.tokenizer = new RegexpTokenizer(options)
+      this.tokenizer._pattern = pattern
+      DEBUG && console.log(this.tokenizer)
+    }
   }
-}
 
-util.inherits(OrthographyTokenizer, RegexpTokenizer)
+  tokenize (text) {
+    return this.tokenizer.tokenize(text)
+  }
+}
 
 exports.OrthographyTokenizer = OrthographyTokenizer
 
@@ -83,12 +87,13 @@ exports.OrthographyTokenizer = OrthographyTokenizer
  *      ['She', 'said', 'hello']
  *
  */
-const WordTokenizer = function (options) {
-  this._pattern = /[^A-Za-zА-Яа-я0-9_]+/
-  RegexpTokenizer.call(this, options)
+class WordTokenizer extends RegexpTokenizer {
+  constructor (options) {
+    super(options)
+    this._pattern = /[^A-Za-zА-Яа-я0-9_]+/
+  }
 }
 
-util.inherits(WordTokenizer, RegexpTokenizer)
 exports.WordTokenizer = WordTokenizer
 
 /***
@@ -99,10 +104,14 @@ exports.WordTokenizer = WordTokenizer
  *      ["She","said","'","hello","'","."]
  *
  */
-const WordPunctTokenizer = function (options) {
-  this._pattern = /([A-Za-zÀ-ÿ-]+|[0-9._]+|.|!|\?|'|"|:|;|,|-)/i
-  RegexpTokenizer.call(this, options)
+class WordPunctTokenizer extends RegexpTokenizer {
+  constructor (options) {
+    if (!options) {
+      options = {}
+    }
+    options.pattern = /([A-Za-zÀ-ÿ-]+|[0-9._]+|.|!|\?|'|"|:|;|,|-)/i
+    super(options)
+  }
 }
 
-util.inherits(WordPunctTokenizer, RegexpTokenizer)
 exports.WordPunctTokenizer = WordPunctTokenizer

package/lib/natural/tokenizers/sentence_tokenizer.js

@@ -23,32 +23,29 @@ THE SOFTWARE.
 'use strict'
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
 const DEBUG = false
 
-const SentenceTokenizer = function () {
-  Tokenizer.call(this)
-}
-util.inherits(SentenceTokenizer, Tokenizer)
-
-SentenceTokenizer.prototype.tokenize = function (text) {
-  // break string up in to sentences based on punctation and quotation marks
-  // let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?.*?[.?!…](\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
-  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!…]|.+)(\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
+class SentenceTokenizer extends Tokenizer {
+  tokenize (text) {
+    // Break string up in to sentences based on punctation and quotation marks
+    // See https://gist.github.com/Hugo-ter-Doest/4ed21fb7eb5077814d998fa61a726566
+    // for a breakdown of the regular expression
+    let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!…]|.+)(\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
 
-  DEBUG && console.log('SentenceTokenizer.tokenize: ' + tokens)
+    DEBUG && console.log('SentenceTokenizer.tokenize: ' + tokens)
 
-  if (!tokens) {
-    return [text]
-  }
+    if (!tokens) {
+      return [text]
+    }
 
-  // remove unecessary white space
-  tokens = tokens.map(Function.prototype.call, String.prototype.trim)
+    // remove unecessary white space
+    tokens = tokens.map(Function.prototype.call, String.prototype.trim)
 
-  DEBUG && console.log('SentenceTokenizer.tokenize: tokens after removing whitespace ' + tokens)
+    DEBUG && console.log('SentenceTokenizer.tokenize: tokens after removing whitespace ' + tokens)
 
-  return this.trim(tokens)
+    return this.trim(tokens)
+  }
 }
 
 module.exports = SentenceTokenizer

package/lib/natural/tokenizers/sentence_tokenizer_parser.js

@@ -22,16 +22,11 @@ THE SOFTWARE.
 const parser = require('./parser_sentence_tokenizer')
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
 
-const SentenceTokenizer = function () {
-  Tokenizer.call(this)
-}
-
-util.inherits(SentenceTokenizer, Tokenizer)
-
-SentenceTokenizer.prototype.tokenize = function (text) {
-  return (parser.parse(text))
+class SentenceTokenizer extends Tokenizer {
+  tokenize (text) {
+    return (parser.parse(text))
+  }
 }
 
 module.exports = SentenceTokenizer

package/lib/natural/tokenizers/tokenizer.js

@@ -20,35 +20,16 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-/**
- * \@todo Use .bind() in Tokenizer.prototype.attach().
- */
-
 'use strict'
 
-const Tokenizer = function () {
-}
-
-Tokenizer.prototype.trim = function (array) {
-  while (array[array.length - 1] === '') { array.pop() }
+class Tokenizer {
+  trim (array) {
+    while (array[array.length - 1] === '') { array.pop() }
 
-  while (array[0] === '') { array.shift() }
-
-  return array
-}
+    while (array[0] === '') { array.shift() }
 
-// Expose an attach function that will patch String with new methods.
-// Changing the prototype of a native type is bad practice
-/*
-Tokenizer.prototype.attach = function () {
-  const self = this
-
-  String.prototype.tokenize = function () {
-    return self.tokenize(this)
+    return array
   }
 }
-*/
-
-Tokenizer.prototype.tokenize = function () {}
 
 module.exports = Tokenizer

package/lib/natural/tokenizers/tokenizer_case.js

@@ -21,41 +21,26 @@
  */
 
 const Tokenizer = require('./tokenizer')
-const util = require('util')
-const CaseTokenizer = function () {
-  Tokenizer.call(this)
-}
-
-util.inherits(CaseTokenizer, Tokenizer)
-
-// Changing the prototype of a native type is bad practice
-/*
-CaseTokenizer.prototype.attach = function () {
-  const self = this
-
-  String.prototype.tokenize = function (preserveApostrophe) {
-    return self.tokenize(this, preserveApostrophe)
-  }
-}
-*/
 
 // Idea from Seagull: http://stackoverflow.com/a/26482650
-CaseTokenizer.prototype.tokenize = function (text, preserveApostrophe) {
-  const whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
-  const lower = text.toLowerCase()
-  const upper = text.toUpperCase()
-  let result = ''
-  let i
-
-  for (i = 0; i < lower.length; ++i) {
-    if (lower[i] !== upper[i] || whitelist.indexOf(lower[i]) > -1 || (text[i] === '\'' && preserveApostrophe)) {
-      result += text[i]
-    } else {
-      result += ' '
+class CaseTokenizer extends Tokenizer {
+  tokenize (text, preserveApostrophe) {
+    const whitelist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
+    const lower = text.toLowerCase()
+    const upper = text.toUpperCase()
+    let result = ''
+    let i
+
+    for (i = 0; i < lower.length; ++i) {
+      if (lower[i] !== upper[i] || whitelist.indexOf(lower[i]) > -1 || (text[i] === '\'' && preserveApostrophe)) {
+        result += text[i]
+      } else {
+        result += ' '
+      }
     }
-  }
 
-  return this.trim(result.replace(/\s+/g, ' ').split(' '))
+    return this.trim(result.replace(/\s+/g, ' ').split(' '))
+  }
 }
 
 module.exports = CaseTokenizer

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.10.0",
+  "version": "6.10.2",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -33,7 +33,7 @@
     "proxyquire": "^1.8.0",
     "rimraf": "^2.6.3",
     "sinon": "^1.12.2",
-    "standard": "^16.0.3",
+    "standard": "^16.0.4",
     "ts-standard": "^12.0.2",
     "typescript": "^4.9.3",
     "uubench": "^0.0.1",