natural 6.12.0 → 7.0.1

package/lib/natural/stemmers/token.js

@@ -22,141 +22,140 @@ THE SOFTWARE.
 
 'use strict'
 
-module.exports = (function () {
-  /**
-   * Stemmer token constructor.
-   *
-   * @param {String} string Token string.
-   */
-  const Token = function (string) {
-    this.vowels = ''
-    this.regions = {}
-    this.string = string
-    this.original = string
+/**
+ * Stemmer token constructor.
+ *
+ * @param {String} string Token string.
+ */
+function Token (string) {
+  this.vowels = ''
+  this.regions = {}
+  this.string = string
+  this.original = string
+}
+
+/**
+ * Set vowels.
+ *
+ * @param  {String|Array} vowels List of vowels.
+ * @return {Token}               Token instance.
+ */
+Token.prototype.usingVowels = function (vowels) {
+  this.vowels = vowels
+  return this
+}
+
+/**
+ * Marks a region by defining its starting index or providing a callback
+ * function that does.
+ *
+ * @param  {String}       region   Region name.
+ * @param  {Array|Number} args     Callback arguments or region start index.
+ * @param  {Function}     callback Function that determines the start index (optional).
+ * @param  {Object}       context  Callback context (optional, defaults to this).
+ * @return {Token}                 Token instance.
+ */
+Token.prototype.markRegion = function (region, args, callback, context) {
+  if (typeof callback === 'function') {
+    // this.regions[region] = callback.apply(context || this, [].concat(args))
+    this.regions[region] = callback.apply(context || this, [].concat(args))
+  } else if (!isNaN(args)) {
+    this.regions[region] = args
   }
 
-  /**
-   * Set vowels.
-   *
-   * @param  {String|Array} vowels List of vowels.
-   * @return {Token}               Token instance.
-   */
-  Token.prototype.usingVowels = function (vowels) {
-    this.vowels = vowels
-    return this
-  }
-
-  /**
-   * Marks a region by defining its starting index or providing a callback
-   * function that does.
-   *
-   * @param  {String}       region   Region name.
-   * @param  {Array|Number} args     Callback arguments or region start index.
-   * @param  {Function}     callback Function that determines the start index (optional).
-   * @param  {Object}       context  Callback context (optional, defaults to this).
-   * @return {Token}                 Token instance.
-   */
-  Token.prototype.markRegion = function (region, args, callback, context) {
-    if (typeof callback === 'function') {
-      this.regions[region] = callback.apply(context || this, [].concat(args))
-    } else if (!isNaN(args)) {
-      this.regions[region] = args
+  return this
+}
+
+/**
+ * Replaces all instances of a string with another.
+ *
+ * @param  {String} find    String to be replaced.
+ * @param  {String} replace Replacement string.
+ * @return {Token}          Token instance.
+ */
+Token.prototype.replaceAll = function (find, replace) {
+  this.string = this.string.split(find).join(replace)
+  return this
+}
+
+/**
+ * Replaces the token suffix if in a region.
+ *
+ * @param  {String} suffix  Suffix to replace.
+ * @param  {String} replace Replacement string.
+ * @param  {String} region  Region name.
+ * @return {Token}          Token instance.
+ */
+Token.prototype.replaceSuffixInRegion = function (suffix, replace, region) {
+  const suffixes = [].concat(suffix)
+  for (let i = 0; i < suffixes.length; i++) {
+    if (this.hasSuffixInRegion(suffixes[i], region)) {
+      this.string = this.string.slice(0, -suffixes[i].length) + replace
+      return this
     }
-
-    return this
-  }
-
-  /**
-   * Replaces all instances of a string with another.
-   *
-   * @param  {String} find    String to be replaced.
-   * @param  {String} replace Replacement string.
-   * @return {Token}          Token instance.
-   */
-  Token.prototype.replaceAll = function (find, replace) {
-    this.string = this.string.split(find).join(replace)
-    return this
   }
-
-  /**
-   * Replaces the token suffix if in a region.
-   *
-   * @param  {String} suffix  Suffix to replace.
-   * @param  {String} replace Replacement string.
-   * @param  {String} region  Region name.
-   * @return {Token}          Token instance.
-   */
-  Token.prototype.replaceSuffixInRegion = function (suffix, replace, region) {
-    const suffixes = [].concat(suffix)
-    for (let i = 0; i < suffixes.length; i++) {
-      if (this.hasSuffixInRegion(suffixes[i], region)) {
-        this.string = this.string.slice(0, -suffixes[i].length) + replace
-        return this
-      }
-    }
-    return this
+  return this
+}
+
+/**
+ * Determines whether the token has a vowel at the provided index.
+ *
+ * @param  {Integer} index Character index.
+ * @return {Boolean}       Whether the token has a vowel at the provided index.
+ */
+Token.prototype.hasVowelAtIndex = function (index) {
+  return this.vowels.indexOf(this.string[index]) !== -1
+}
+
+/**
+ * Finds the next vowel in the token.
+ *
+ * @param  {Integer} start Starting index offset.
+ * @return {Integer}       Vowel index, or the end of the string.
+ */
+Token.prototype.nextVowelIndex = function (start) {
+  let index = (start >= 0 && start < this.string.length) ? start : this.string.length
+  while (index < this.string.length && !this.hasVowelAtIndex(index)) {
+    index++
   }
-
-  /**
-   * Determines whether the token has a vowel at the provided index.
-   *
-   * @param  {Integer} index Character index.
-   * @return {Boolean}       Whether the token has a vowel at the provided index.
-   */
-  Token.prototype.hasVowelAtIndex = function (index) {
-    return this.vowels.indexOf(this.string[index]) !== -1
+  return index
+}
+
+/**
+ * Finds the next consonant in the token.
+ *
+ * @param  {Integer} start Starting index offset.
+ * @return {Integer}       Consonant index, or the end of the string.
+ */
+Token.prototype.nextConsonantIndex = function (start) {
+  let index = (start >= 0 && start < this.string.length) ? start : this.string.length
+  while (index < this.string.length && this.hasVowelAtIndex(index)) {
+    index++
   }
-
-  /**
-   * Finds the next vowel in the token.
-   *
-   * @param  {Integer} start Starting index offset.
-   * @return {Integer}       Vowel index, or the end of the string.
-   */
-  Token.prototype.nextVowelIndex = function (start) {
-    let index = (start >= 0 && start < this.string.length) ? start : this.string.length
-    while (index < this.string.length && !this.hasVowelAtIndex(index)) {
-      index++
-    }
-    return index
-  }
-
-  /**
-   * Finds the next consonant in the token.
-   *
-   * @param  {Integer} start Starting index offset.
-   * @return {Integer}       Consonant index, or the end of the string.
-   */
-  Token.prototype.nextConsonantIndex = function (start) {
-    let index = (start >= 0 && start < this.string.length) ? start : this.string.length
-    while (index < this.string.length && this.hasVowelAtIndex(index)) {
-      index++
-    }
-    return index
-  }
-
-  /**
-   * Determines whether the token has the provided suffix.
-   * @param  {String}  suffix Suffix to match.
-   * @return {Boolean}        Whether the token string ends in suffix.
-   */
-  Token.prototype.hasSuffix = function (suffix) {
-    return this.string.slice(-suffix.length) === suffix
-  }
-
-  /**
-   * Determines whether the token has the provided suffix within the specified
-   * region.
-   *
-   * @param {String} suffix Suffix to match.
-   * @param {String} region Region name.
-   * @return {Boolean} Whether the token string ends in suffix.
-   */
-  Token.prototype.hasSuffixInRegion = function (suffix, region) {
-    const regionStart = this.regions[region] || 0
-    const suffixStart = this.string.length - suffix.length
-    return this.hasSuffix(suffix) && suffixStart >= regionStart
-  }
-
-  return Token
-})()
+  return index
+}
+
+/**
+ * Determines whether the token has the provided suffix.
+ * @param  {String}  suffix Suffix to match.
+ * @return {Boolean}        Whether the token string ends in suffix.
+ */
+Token.prototype.hasSuffix = function (suffix) {
+  return this.string.slice(-suffix.length) === suffix
+}
+
+/**
+ * Determines whether the token has the provided suffix within the specified
+ * region.
+ *
+ * @param {String} suffix Suffix to match.
+ * @param {String} region Region name.
+ * @return {Boolean} Whether the token string ends in suffix.
+ */
+Token.prototype.hasSuffixInRegion = function (suffix, region) {
+  const regionStart = this.regions[region] || 0
+  const suffixStart = this.string.length - suffix.length
+  return this.hasSuffix(suffix) && suffixStart >= regionStart
+}
+
+module.exports = Token

package/lib/natural/tfidf/index.d.ts

@@ -23,9 +23,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-import { Tokenizer } from '../tokenizers'
+import type { Tokenizer } from '../tokenizers'
 
-declare type TfIdfCallback = (i: number, measure: number, key?: string) => void
+declare type TfIdfCallback = (i: number, measure: number, key?: string | Record<string, any>) => void
 
 declare interface TfIdfTerm {
   term: string
@@ -34,15 +34,15 @@ declare interface TfIdfTerm {
   tfidf: number
 }
 
-declare type TfIdfDocument = { __key: string | undefined } | { [key: string]: number | undefined }
+declare type TfIdfDocument = Record<string, number>
 
 export class TfIdf {
-  documents: TfIdfDocument | undefined
-  _idfCache: { [key: string]: number } | undefined
+  documents: TfIdfDocument[]
+  _idfCache: Record<string, number>
 
-  constructor (deserialized?: { documents: TfIdfDocument[] })
+  constructor (deserialized?: Record<string, unknown>)
   idf (term: string, force?: boolean): number
-  addDocument (document: string | string[], key?: string, restoreCache?: boolean): void
+  addDocument (document: string | string[] | Record<string, string>, key?: Record<string, any> | any, restoreCache?: boolean): void
   addFileSync (path: string, encoding?: string, key?: string, restoreCache?: boolean): void
   tfidf (terms: string | string[], d: number): number
   tfidfs (terms: string | string[], callback?: TfIdfCallback): number[]

package/lib/natural/tfidf/tfidf.js

@@ -26,6 +26,8 @@ let tokenizer = new Tokenizer()
 let stopwords = require('../util/stopwords').words
 const fs = require('fs')
 
+// Returns a frequency map of word to frequency
+// Key is the document key and stored in the map that is returned as __keys
 function buildDocument (text, key) {
   let stopOut
 
@@ -87,6 +89,8 @@ class TfIdf {
     return document[term] ? document[term] : 0
   }
 
+  // Returns the inverse document frequency of the term
+  // If force is true the cache will be invalidated and recomputed
   idf (term, force) {
     // Lookup the term in the New term-IDF caching,
     // this will cut search times down exponentially on large document sets.
@@ -95,10 +99,12 @@ class TfIdf {
       return this._idfCache[term]
     }
 
+    // Count the number of documents that contain the term
     const docsWithTerm = this.documents.reduce(function (count, document) {
       return count + (documentHasTerm(term, document) ? 1 : 0)
     }, 0)
 
+    // Compute the inverse document frequency
     const idf = 1 + Math.log((this.documents.length) / (1 + docsWithTerm))
 
     // Add the idf to the term cache and return it
@@ -169,7 +175,7 @@ class TfIdf {
       if (this.documents[d]) {
         if (term !== '__key') {
           terms.push({
-            term: term,
+            term,
             tf: TfIdf.tf(term, _this.documents[d]),
             idf: _this.idf(term),
             tfidf: _this.tfidf(term, d)

package/lib/natural/tokenizers/index.d.ts

@@ -87,12 +87,16 @@ export class AggressiveTokenizerHi extends Tokenizer {
   tokenize (text: string): string[]
 }
 
+export class AggressiveTokenizerUk extends Tokenizer {
+  tokenize (text: string): string[]
+}
+
 export class AggressiveTokenizer extends Tokenizer {
   tokenize (text: string): string[]
 }
 
 export class CaseTokenizer extends Tokenizer {
-  tokenize (text: string): string[]
+  tokenize (text: string, preserveApostrophe?: boolean): string[]
 }
 
 declare interface RegexTokenizerOptions {
@@ -103,7 +107,7 @@ declare interface RegexTokenizerOptions {
 
 export class RegexpTokenizer extends Tokenizer {
   constructor (opts?: RegexTokenizerOptions)
-  tokenize (text: string): string[] | null
+  tokenize (text: string): string[]
 }
 
 declare interface OrthographyTokenizerOptions extends RegexTokenizerOptions {

package/lib/natural/tokenizers/index.js

@@ -36,6 +36,7 @@ exports.AggressiveTokenizerSv = require('./aggressive_tokenizer_sv')
 exports.AggressiveTokenizerVi = require('./aggressive_tokenizer_vi')
 exports.AggressiveTokenizerId = require('./aggressive_tokenizer_id')
 exports.AggressiveTokenizerHi = require('./aggressive_tokenizer_hi')
+exports.AggressiveTokenizerUk = require('./aggressive_tokenizer_uk')
 exports.AggressiveTokenizer = require('./aggressive_tokenizer')
 exports.CaseTokenizer = require('./tokenizer_case')
 exports.RegexpTokenizer = require('./regexp_tokenizer').RegexpTokenizer

package/lib/natural/trie/index.d.ts

@@ -27,7 +27,7 @@ export class Trie {
   addString (string: string): boolean
   addStrings (list: string[]): void
   contains (string: string): boolean
-  findPrefix (search: string): string[]
+  findPrefix (search: string): Array<string | null>
   findMatchesOnPath (search: string): string[]
   keysWithPrefix (prefix: string): string[]
   getSize (): number

package/lib/natural/util/index.d.ts

@@ -70,3 +70,11 @@ export class LongestPathTree {
   hasPathTo (vertex: number): boolean
   pathTo (vertex: number): number[]
 }
+
+export class Topological {
+  constructor (digraph: EdgeWeightedDigraph)
+  isDAG (): boolean
+  order (): number[]
+}
+
+export * from './storage'

package/lib/natural/util/index.js

@@ -27,3 +27,5 @@ exports.ShortestPathTree = require('./shortest_path_tree')
 exports.LongestPathTree = require('./longest_path_tree')
 exports.DirectedEdge = require('./directed_edge')
 exports.EdgeWeightedDigraph = require('./edge_weighted_digraph')
+exports.Topological = require('./topological')
+exports.StorageBackend = require('./storage')

package/lib/natural/util/storage/index.d.ts

@@ -0,0 +1,39 @@
+/*
+Copyright (c) 2024, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+export declare enum STORAGE_TYPES {
+  POSTGRES = 'POSTGRES',
+  REDIS = 'REDIS',
+  MONGODB = 'MONGODB',
+  MEMCACHED = 'MEMCACHED',
+  FILE = 'FILE'
+}
+
+export class StorageBackend {
+  private readonly storageType: STORAGE_TYPES
+  private readonly client: any
+
+  constructor (type: STORAGE_TYPES)
+  setStorageType (storageType: STORAGE_TYPES): void
+  store (key: string, value: any): Promise<string>
+  retrieve (key: string): Promise<any>
+}

package/lib/natural/util/storage/index.js

@@ -0,0 +1,24 @@
+
+/*
+Copyright (c) 2024, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+exports.StorageBackend = require('./StorageBackend')

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.12.0",
+  "version": "7.0.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -11,6 +11,7 @@
     "node": ">=0.4.10"
   },
   "dependencies": {
+    "@types/jasmine": "^5.1.4",
     "afinn-165": "^1.0.2",
     "afinn-165-financialmarketnews": "^3.0.0",
     "apparatus": "^0.0.10",
@@ -28,20 +29,31 @@
   },
   "devDependencies": {
     "@types/node": "^18.11.18",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
     "browserfs": "^1.4.3",
+    "cross-env": "^7.0.3",
+    "eslint": "^8.57.0",
+    "eslint-config-standard-with-typescript": "^43.0.1",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-promise": "^6.1.1",
     "gulp": "^4.0.2",
     "gulp-jasmine": "^4.0.0",
     "gulp-jasmine-browser": "^4.1.0",
     "jasmine": "^3.3.1",
     "jasmine-core": "^3.3.0",
+    "lint": "^0.8.19",
+    "ncp": "^2.0.0",
     "nyc": "^15.1.0",
     "pegjs": "^0.10.0",
     "proxyquire": "^1.8.0",
     "rimraf": "^2.6.3",
     "sinon": "^1.12.2",
+    "source-map-support": "^0.5.21",
     "standard": "^16.0.4",
+    "ts-node": "^10.9.2",
     "ts-standard": "^12.0.2",
-    "typescript": "^4.9.3",
+    "typescript": "^5.4.2",
     "uubench": "^0.0.1",
     "webpack": "^4.29.0",
     "webpack-stream": "^5.2.1"
@@ -63,13 +75,14 @@
   },
   "scripts": {
     "benchmark": "node benchmarks",
-    "clean": "rimraf *~ #* *classifier.json",
-    "test": "NODE_PATH=. node ./node_modules/jasmine/bin/jasmine.js --random=false spec/*_spec.js",
-    "coverage": "nyc --reporter=lcov npm run test && nyc npm run test_io",
-    "test_io": "NODE_PATH=. node ./node_modules/jasmine/bin/jasmine.js --random=false io_spec/*_spec.js",
-    "test_browser": "NODE_PATH=. node ./node_modules/gulp/bin/gulp.js",
-    "lint": "eslint . --ext .ts",
-    "lint-ts": "ts-standard **/*.ts"
+    "build": "tsc && ncp spec/test_data dist",
+    "clean": "rimraf *~ #* *classifier.json dist io_spec/tmp/*.json coverage",
+    "test": "cross-env NODE_PATH=. jasmine --random=false spec/*_spec.js",
+    "test_io": "jasmine --random=false io_spec/*_spec.js",
+    "test_ts": "cross-env NODE_PATH=.:./dist jasmine --random=false dist/ts_spec/*_spec.js",
+    "coverage": "nyc --reporter=lcov npm run test && nyc --reporter=lcov npm run test_io && nyc --reporter=lcov npm run test_ts",
+    "test_browser": "cross-env NODE_PATH=. node ./node_modules/gulp/bin/gulp.js",
+    "lint": "eslint . --ext .ts"
   },
   "license": "MIT",
   "author": "Chris Umbel <chris@chrisumbel.com>",