natural 6.11.1 → 7.0.0

package/lib/natural/phonetics/index.d.ts

@@ -20,24 +20,54 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export interface Phonetic<T> {
-  compare: (stringA: string, stringB: string) => boolean
-  tokenizeAndPhoneticize: (str: string, phoneticsProcessor: Phonetic<T>, keepStops: boolean) => T
+export class Phonetic<T> {
+  compare (stringA: string, stringB: string): boolean
+  tokenizeAndPhoneticize (str: string, phoneticsProcessor: Phonetic<T>, keepStops: boolean): T
 }
 
-export let Metaphone: Phonetic<string> & {
-  process: (token: string, maxLength?: number) => string
+export class Metaphone extends Phonetic<string> {
+  dedup (token: string): string
+  dropInitialLetters (token: string): string
+  dropBafterMAtEnd (token: string): string
+  cTransform (token: string): string
+  dTransform (token: string): string
+  dropG (token: string): string
+  transformG (token: string): string
+  dropH (token: string): string
+  transformCK (token: string): string
+  transformPH (token: string): string
+  transformQ (token: string): string
+  transformS (token: string): string
+  transformT (token: string): string
+  dropT (token: string): string
+  transformV (token: string): string
+  transformWH (token: string): string
+  dropW (token: string): string
+  transformX (token: string): string
+  dropY (token: string): string
+  transformZ (token: string): string
+  dropVowels (token: string): string
+  process (token: string, maxLength?: number): string
 }
 
-export let SoundEx: Phonetic<string> & {
-  process: (token: string, maxLength?: number) => string
+export class SoundEx extends Phonetic<string> {
+  process (token: string, maxLength?: number): string
+  transformLipps (token: string): string
+  transformThroats (token: string): string
+  transformToungue (token: string): string
+  transformL (token: string): string
+  transformHum (token: string): string
+  transformR (token: string): string
+  condense (token: string): string
+  padRight0 (token: string): string
+  transform (token: string): string
 }
 
-export let DoubleMetaphone: Phonetic<string[]> & {
-  process: (token: string, maxLength?: number) => string[]
-  isVowel: (c: string) => string | RegExpMatchArray | null
+export class DoubleMetaphone extends Phonetic<string[]> {
+  process (token: string, maxLength?: number): string[]
+  isVowel (c: string): string | RegExpMatchArray | null
 }
 
-export let SoundExDM: Phonetic<string> & {
-  process: (word: string, codeLength?: number) => string
+export class SoundExDM extends Phonetic<string> {
+  process (word: string, codeLength?: number): string
 }

package/lib/natural/phonetics/metaphone.js

@@ -22,170 +22,147 @@ THE SOFTWARE.
 
 'use strict'
 
-'use strict'
-
 const Phonetic = require('./phonetic')
 
-function dedup (token) {
-  return token.replace(/([^c])\1/g, '$1')
-}
-
-function dropInitialLetters (token) {
-  if (token.match(/^(kn|gn|pn|ae|wr)/)) { return token.substr(1, token.length - 1) }
-
-  return token
-}
-
-function dropBafterMAtEnd (token) {
-  return token.replace(/mb$/, 'm')
-}
-
-function cTransform (token) {
-  token = token.replace(/([^s]|^)(c)(h)/g, '$1x$3').trim()
-
-  token = token.replace(/cia/g, 'xia')
-  token = token.replace(/c(i|e|y)/g, 's$1')
-  token = token.replace(/c/g, 'k')
-
-  return token
-}
-
-function dTransform (token) {
-  token = token.replace(/d(ge|gy|gi)/g, 'j$1')
-  token = token.replace(/d/g, 't')
-
-  return token
-}
-
-function dropG (token) {
-  token = token.replace(/gh(^$|[^aeiou])/g, 'h$1')
-  token = token.replace(/g(n|ned)$/g, '$1')
-
-  return token
-}
-
-function transformG (token) {
-  token = token.replace(/gh/g, 'f')
-  token = token.replace(/([^g]|^)(g)(i|e|y)/g, '$1j$3')
-  token = token.replace(/gg/g, 'g')
-  token = token.replace(/g/g, 'k')
-
-  return token
-}
-
-function dropH (token) {
-  return token.replace(/([aeiou])h([^aeiou]|$)/g, '$1$2')
-}
-
-function transformCK (token) {
-  return token.replace(/ck/g, 'k')
-}
-function transformPH (token) {
-  return token.replace(/ph/g, 'f')
-}
-
-function transformQ (token) {
-  return token.replace(/q/g, 'k')
-}
-
-function transformS (token) {
-  return token.replace(/s(h|io|ia)/g, 'x$1')
-}
-
-function transformT (token) {
-  token = token.replace(/t(ia|io)/g, 'x$1')
-  token = token.replace(/th/, '0')
+class Metaphone extends Phonetic {
+  dedup (token) {
+    return token.replace(/([^c])\1/g, '$1')
+  }
 
-  return token
-}
+  dropInitialLetters (token) {
+    if (token.match(/^(kn|gn|pn|ae|wr)/)) { return token.substr(1, token.length - 1) }
 
-function dropT (token) {
-  return token.replace(/tch/g, 'ch')
-}
+    return token
+  }
 
-function transformV (token) {
-  return token.replace(/v/g, 'f')
-}
+  dropBafterMAtEnd (token) {
+    return token.replace(/mb$/, 'm')
+  }
 
-function transformWH (token) {
-  return token.replace(/^wh/, 'w')
-}
+  cTransform (token) {
+    token = token.replace(/([^s]|^)(c)(h)/g, '$1x$3').trim()
 
-function dropW (token) {
-  return token.replace(/w([^aeiou]|$)/g, '$1')
+    token = token.replace(/cia/g, 'xia')
+    token = token.replace(/c(i|e|y)/g, 's$1')
+    token = token.replace(/c/g, 'k')
+
+    return token
+  }
+
+  dTransform (token) {
+    token = token.replace(/d(ge|gy|gi)/g, 'j$1')
+    token = token.replace(/d/g, 't')
+
+    return token
+  }
+
+  dropG (token) {
+    token = token.replace(/gh(^$|[^aeiou])/g, 'h$1')
+    token = token.replace(/g(n|ned)$/g, '$1')
+
+    return token
+  }
+
+  transformG (token) {
+    token = token.replace(/gh/g, 'f')
+    token = token.replace(/([^g]|^)(g)(i|e|y)/g, '$1j$3')
+    token = token.replace(/gg/g, 'g')
+    token = token.replace(/g/g, 'k')
+
+    return token
+  }
+
+  dropH (token) {
+    return token.replace(/([aeiou])h([^aeiou]|$)/g, '$1$2')
+  }
+
+  transformCK (token) {
+    return token.replace(/ck/g, 'k')
+  }
+
+  transformPH (token) {
+    return token.replace(/ph/g, 'f')
+  }
+
+  transformQ (token) {
+    return token.replace(/q/g, 'k')
+  }
+
+  transformS (token) {
+    return token.replace(/s(h|io|ia)/g, 'x$1')
+  }
+
+  transformT (token) {
+    token = token.replace(/t(ia|io)/g, 'x$1')
+    token = token.replace(/th/, '0')
+
+    return token
+  }
+
+  dropT (token) {
+    return token.replace(/tch/g, 'ch')
+  }
+
+  transformV (token) {
+    return token.replace(/v/g, 'f')
+  }
+
+  transformWH (token) {
+    return token.replace(/^wh/, 'w')
+  }
+
+  dropW (token) {
+    return token.replace(/w([^aeiou]|$)/g, '$1')
+  }
+
+  transformX (token) {
+    token = token.replace(/^x/, 's')
+    token = token.replace(/x/g, 'ks')
+    return token
+  }
+
+  dropY (token) {
+    return token.replace(/y([^aeiou]|$)/g, '$1')
+  }
+
+  transformZ (token) {
+    return token.replace(/z/, 's')
+  }
+
+  dropVowels (token) {
+    return token.charAt(0) + token.substr(1, token.length).replace(/[aeiou]/g, '')
+  }
+
+  process (token, maxLength) {
+    const maxLengthNew = maxLength || 32
+    token = token.toLowerCase()
+    token = this.dedup(token)
+    token = this.dropInitialLetters(token)
+    token = this.dropBafterMAtEnd(token)
+    token = this.transformCK(token)
+    token = this.cTransform(token)
+    token = this.dTransform(token)
+    token = this.dropG(token)
+    token = this.transformG(token)
+    token = this.dropH(token)
+    token = this.transformPH(token)
+    token = this.transformQ(token)
+    token = this.transformS(token)
+    token = this.transformX(token)
+    token = this.transformT(token)
+    token = this.dropT(token)
+    token = this.transformV(token)
+    token = this.transformWH(token)
+    token = this.dropW(token)
+    token = this.dropY(token)
+    token = this.transformZ(token)
+    token = this.dropVowels(token)
+
+    token.toUpperCase()
+    if (token.length >= maxLengthNew) { token = token.substring(0, maxLengthNew) }
+
+    return token.toUpperCase()
+  }
 }
 
-function transformX (token) {
-  token = token.replace(/^x/, 's')
-  token = token.replace(/x/g, 'ks')
-  return token
-}
-
-function dropY (token) {
-  return token.replace(/y([^aeiou]|$)/g, '$1')
-}
-
-function transformZ (token) {
-  return token.replace(/z/, 's')
-}
-
-function dropVowels (token) {
-  return token.charAt(0) + token.substr(1, token.length).replace(/[aeiou]/g, '')
-}
-
-const Metaphone = new Phonetic()
 module.exports = Metaphone
-
-Metaphone.process = function (token, maxLength) {
-  const maxLengthNew = maxLength || 32
-  token = token.toLowerCase()
-  token = dedup(token)
-  token = dropInitialLetters(token)
-  token = dropBafterMAtEnd(token)
-  token = transformCK(token)
-  token = cTransform(token)
-  token = dTransform(token)
-  token = dropG(token)
-  token = transformG(token)
-  token = dropH(token)
-  token = transformPH(token)
-  token = transformQ(token)
-  token = transformS(token)
-  token = transformX(token)
-  token = transformT(token)
-  token = dropT(token)
-  token = transformV(token)
-  token = transformWH(token)
-  token = dropW(token)
-  token = dropY(token)
-  token = transformZ(token)
-  token = dropVowels(token)
-
-  token.toUpperCase()
-  if (token.length >= maxLengthNew) { token = token.substring(0, maxLengthNew) }
-
-  return token.toUpperCase()
-}
-
-// expose functions for testing
-Metaphone.dedup = dedup
-Metaphone.dropInitialLetters = dropInitialLetters
-Metaphone.dropBafterMAtEnd = dropBafterMAtEnd
-Metaphone.cTransform = cTransform
-Metaphone.dTransform = dTransform
-Metaphone.dropG = dropG
-Metaphone.transformG = transformG
-Metaphone.dropH = dropH
-Metaphone.transformCK = transformCK
-Metaphone.transformPH = transformPH
-Metaphone.transformQ = transformQ
-Metaphone.transformS = transformS
-Metaphone.transformT = transformT
-Metaphone.dropT = dropT
-Metaphone.transformV = transformV
-Metaphone.transformWH = transformWH
-Metaphone.dropW = dropW
-Metaphone.transformX = transformX
-Metaphone.dropY = dropY
-Metaphone.transformZ = transformZ
-Metaphone.dropVowels = dropVowels

package/lib/natural/phonetics/phonetic.js

@@ -26,12 +26,12 @@ const stopwords = require('../util/stopwords')
 const Tokenizer = require('../tokenizers/aggressive_tokenizer')
 const tokenizer = new Tokenizer()
 
-module.exports = function () {
-  this.compare = function (stringA, stringB) {
+class Phonetic {
+  compare (stringA, stringB) {
     return this.process(stringA) === this.process(stringB)
   }
 
-  this.tokenizeAndPhoneticize = function (str, phoneticsProcessor, keepStops) {
+  tokenizeAndPhoneticize (str, phoneticsProcessor, keepStops) {
     const phoneticizedTokens = []
 
     tokenizer.tokenize(str).forEach(function (token) {
@@ -42,30 +42,6 @@ module.exports = function () {
 
     return phoneticizedTokens
   }
-
-/*
-  this.attach = function () {
-    const phonetic = this
-
-    String.prototype.soundsLike = function (compareTo) {
-      return phonetic.compare(this, compareTo)
-    }
-
-    String.prototype.phonetics = function () {
-      return phonetic.process(this)
-    }
-
-    String.prototype.tokenizeAndPhoneticize = function (keepStops) {
-      const phoneticizedTokens = []
-
-      tokenizer.tokenize(this).forEach(function (token) {
-        if (keepStops || stopwords.words.indexOf(token) < 0) {
-          phoneticizedTokens.push(token.phonetics())
-        }
-      })
-
-      return phoneticizedTokens
-    }
-  }
-  */
 }
+
+module.exports = Phonetic

package/lib/natural/phonetics/soundex.js

@@ -24,60 +24,51 @@ THE SOFTWARE.
 
 const Phonetic = require('./phonetic')
 
-function transformLipps (token) {
-  return token.replace(/[bfpv]/g, '1')
+class SoundEx extends Phonetic {
+  process (token, maxLength) {
+    token = token.toLowerCase()
+    let transformed = this.condense(this.transform(token.substr(1, token.length - 1))) // anything that isn't a digit goes
+    // deal with duplicate INITIAL consonant SOUNDS
+    transformed = transformed.replace(new RegExp('^' + this.transform(token.charAt(0))), '')
+    return token.charAt(0).toUpperCase() + this.padRight0(transformed.replace(/\D/g, '')).substr(0, (maxLength && maxLength - 1) || 3)
+  }
+
+  transformLipps (token) {
+    return token.replace(/[bfpv]/g, '1')
+  }
+
+  transformThroats (token) {
+    return token.replace(/[cgjkqsxz]/g, '2')
+  }
+
+  transformToungue (token) {
+    return token.replace(/[dt]/g, '3')
+  }
+
+  transformL (token) {
+    return token.replace(/l/g, '4')
+  }
+
+  transformHum (token) {
+    return token.replace(/[mn]/g, '5')
+  }
+
+  transformR (token) {
+    return token.replace(/r/g, '6')
+  }
+
+  condense (token) {
+    return token.replace(/(\d)?\1+/g, '$1')
+  }
+
+  padRight0 (token) {
+    if (token.length < 4) { return token + Array(4 - token.length).join('0') } else { return token }
+  }
+
+  transform (token) {
+    return this.transformLipps(this.transformThroats(
+      this.transformToungue(this.transformL(this.transformHum(this.transformR(token))))))
+  }
 }
 
-function transformThroats (token) {
-  return token.replace(/[cgjkqsxz]/g, '2')
-}
-
-function transformToungue (token) {
-  return token.replace(/[dt]/g, '3')
-}
-
-function transformL (token) {
-  return token.replace(/l/g, '4')
-}
-
-function transformHum (token) {
-  return token.replace(/[mn]/g, '5')
-}
-
-function transformR (token) {
-  return token.replace(/r/g, '6')
-}
-
-function condense (token) {
-  return token.replace(/(\d)?\1+/g, '$1')
-}
-
-function padRight0 (token) {
-  if (token.length < 4) { return token + Array(4 - token.length).join('0') } else { return token }
-}
-
-function transform (token) {
-  return transformLipps(transformThroats(
-    transformToungue(transformL(transformHum(transformR(token))))))
-}
-
-const SoundEx = new Phonetic()
 module.exports = SoundEx
-
-SoundEx.process = function (token, maxLength) {
-  token = token.toLowerCase()
-  let transformed = condense(transform(token.substr(1, token.length - 1))) // anything that isn't a digit goes
-  // deal with duplicate INITIAL consonant SOUNDS
-  transformed = transformed.replace(new RegExp('^' + transform(token.charAt(0))), '')
-  return token.charAt(0).toUpperCase() + padRight0(transformed.replace(/\D/g, '')).substr(0, (maxLength && maxLength - 1) || 3)
-}
-
-// export for tests;
-SoundEx.transformLipps = transformLipps
-SoundEx.transformThroats = transformThroats
-SoundEx.transformToungue = transformToungue
-SoundEx.transformL = transformL
-SoundEx.transformHum = transformHum
-SoundEx.transformR = transformR
-SoundEx.condense = condense
-SoundEx.padRight0 = padRight0

package/lib/natural/sentiment/index.d.ts

@@ -1,6 +1,8 @@
-
 /*
-Copyright (c) 2022, Dylan R. E. Moonfire <https://github.com/dmoonfire>, Emily Marigold Klassen <https://github.com/forivall>, Hugo W.L. ter Doest
+Copyright (c) 2022,
+  Dylan R. E. Moonfire <https://github.com/dmoonfire>,
+  Emily Marigold Klassen <https://github.com/forivall>,
+  Hugo W.L. ter Doest
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -21,7 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-import { Stemmer } from '../stemmers'
+import type { Stemmer } from '../stemmers'
 
 export type AfinnLanguageType = 'afinn'
 export type AfinnLanguage = 'English' | 'Spanish' | 'Portuguese'
@@ -33,8 +35,6 @@ export type PatternLanguageType = 'pattern'
 export type PatternLanguage = 'Dutch' | 'Italian' | 'English' | 'French' | 'German'
 
 export declare class SentimentAnalyzer {
-  constructor (language: AfinnLanguage, stemmer: Stemmer, type: AfinnLanguageType)
-  constructor (language: SenticonLanguage, stemmer: Stemmer, type: SenticonLanguageType)
-  constructor (language: PatternLanguage, stemmer: Stemmer, type: PatternLanguageType)
+  constructor (language: string, stemmer: Stemmer | undefined, type: string)
   getSentiment (words: string[]): number
 }

package/lib/natural/spellcheck/index.d.ts

@@ -27,4 +27,6 @@ export declare class Spellcheck {
   constructor (wordlist: string[])
   isCorrect (word: string): boolean
   getCorrections (word: string, maxDistance?: number): string[]
+  edits (word: string): string[]
+  editsWithMaxDistance (word: string, maxDistance: number): string[]
 }

package/lib/natural/stemmers/index.d.ts

@@ -20,13 +20,35 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-declare interface Stemmer {
+declare interface RegionsType {
+  r1: number
+  r2: number
+  rv: number
+}
+
+export declare interface Stemmer {
   stem: (token: string) => string
   addStopWord: (stopWord: string) => void
   addStopWords: (moreStopWords: string[]) => void
   removeStopWord: (stopWord: string) => void
   removeStopWords: (moreStopWords: string[]) => void
   tokenizeAndStem: (text: string, keepStops?: boolean) => string[]
+  regions: (word: string) => RegionsType
+  prelude: (token: string) => string
+  endsinArr: (token: string, suffixes: string[]) => string
+  categorizeGroups: (token: string) => string
+  measure: (token: string) => number
+
+  // Exposed for testing purposes:
+  step1: (token: string) => string
+  step1a: (token: string) => string
+  step1b: (token: string) => string
+  step1c: (token: string) => string
+  step2: (token: string) => string
+  step3: (token: string) => string
+  step4: (token: string) => string
+  step5a: (token: string) => string
+  step5b: (token: string) => string
 }
 
 export let CarryStemmerFr: Stemmer
@@ -45,3 +67,23 @@ export let PorterStemmerRu: Stemmer
 export let PorterStemmerSv: Stemmer
 export let StemmerId: Stemmer
 export let StemmerJa: Stemmer
+
+export declare type TokenCallback = (...args: number[]) => number[] | number
+
+export declare class Token {
+  vowels: string[] | string
+  regions: Record<string, number>
+  string: string
+  original: string
+
+  constructor (s: string)
+  usingVowels (vowels: string | string[]): Token
+  markRegion (region: string, args: number[] | number | null, callback?: TokenCallback, context?: unknown): Token
+  replaceAll (find: string, replace: string): Token
+  replaceSuffixInRegion (suffix: string, replace: string, region: string): Token
+  hasVowelAtIndex (index: number): boolean
+  nextVowelIndex (index: number): number
+  nextConsonantIndex (index: number): number
+  hasSuffix (suffix: string): number
+  hasSuffixInRegion (suffix: string, region: string): boolean
+}

package/lib/natural/stemmers/index.js

@@ -38,3 +38,4 @@ exports.PorterStemmerNl = require('./porter_stemmer_nl')
 exports.LancasterStemmer = require('./lancaster_stemmer')
 exports.StemmerJa = require('./stemmer_ja')
 exports.StemmerId = require('./indonesian/stemmer_id')
+exports.Token = require('./token')