natural 6.2.0 → 6.3.0

package/lib/natural/analyzers/index.d.ts

@@ -21,34 +21,40 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export declare interface TaggedWord {
-  [key: string]: string
+declare interface TaggedWord {
   token: string
   pos: string
+  pp?: boolean
   spos?: string
   added?: boolean
 }
 
-export declare type PunctuationFunction = () => string[] | ''
+declare type PunctuationFunction = () => string[] | ''
 
-export declare interface TaggedSentence {
+declare interface TaggedSentence {
   tags: TaggedWord[]
-  punct?: PunctuationFunction
+  punct: PunctuationFunction
 }
 
-export declare type CallbackFunction = (obj: SentenceAnalyzer) => void
+declare type CallbackFunction = (obj: SentenceAnalyzer) => void
 
-export declare type SenType = string
+declare enum SenType {
+  Unknown = 'UNKNOWN',
+  Command = 'COMMAND',
+  Interrogative = 'INTERROGATIVE',
+  Exclamatory = 'EXCLAMATORY',
+  Declarative = 'DECLARATIVE',
+}
 
-export declare class SentenceAnalyzer {
+export class SentenceAnalyzer {
   posObj: TaggedSentence
-  senType: string | null
+  senType: SenType | null
 
   constructor (pos: TaggedSentence, cbf: CallbackFunction)
   part (cbf: CallbackFunction): void
   prepositionPhrases (): void
-  subjectToString (): string | null
-  predicateToString (): string | null
+  subjectToString (): string
+  predicateToString (): string
   implicitYou (): boolean
   toString (): string
   type (cbf: CallbackFunction): SenType

package/lib/natural/brill_pos_tagger/index.d.ts

@@ -23,51 +23,140 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export declare class Predicate {
+declare interface RuleTemplatesItem {
+  function: (sentence: Sentence, i: number, parameter1?: string, parameter2?: string) => boolean
+  window: number[]
+  nrParameters: number
+  parameter1Values?: (sentence: Sentence, i: number) => string[]
+  parameter2Values?: (sentence: Sentence, i: number) => string[]
+}
+
+export interface RuleTemplates {
+  [key: string]: RuleTemplatesItem | undefined
+}
+
+export class RuleTemplate {
+  constructor (templateName: string, metadata: RuleTemplatesItem)
+  windowFitsSite (sentence: Sentence, i: number): void
+}
+
+declare class Predicate {
   constructor (name: string, parameter1: string, parameter2?: string)
   name: string
   parameter1: string
-  parameter2?: string | undefined
-  function?: ((tagged_sentence: string[][], i: number, parameter: string) => boolean) | undefined
-  evaluate (taggedSentence: string[][], position: number): boolean
+  parameter2: string | undefined
+  meta: RuleTemplatesItem
+  evaluate (sentence: Sentence, position: number): boolean
 }
 
-export declare class TransformationRule {
+declare class TransformationRule {
   constructor (c1: string, c2: string, predicate: string, parameter1: string, parameter2?: string)
   literal: string[]
   predicate: Predicate
   old_category: string
   new_category: string
-  apply (taggedSentence: string[][], position: number): void
+  neutral: number
+  positive: number
+  negative: number
+  hasBeenSelectedAsHighRuleBefore: boolean
+  key (): string
+  apply (sentence: Sentence, position: number): void
+  isApplicableAt (sentence: Sentence, taggedSentence: Sentence, i: number): boolean
+  prettyPrint (): string
+  applyAt (sentence: Sentence, position: number): void
+  score (): number
 }
 
-export declare class RuleSet {
-  constructor (filename: string)
+export class RuleSet {
+  constructor (language: string)
   rules: TransformationRule[]
+  addRule (rule: TransformationRule): boolean
+  removeRule (rule: TransformationRule): void
+  getRules (): TransformationRule[]
+  nrRules (): number
+  hasRule (rule: TransformationRule): boolean
+  prettyPrint (): string
 }
 
-export declare class Lexicon {
-  constructor (filename: string, defaultCategory: string)
+export class Lexicon {
+  constructor (language: string, defaultCategory: string, defaultCategoryCapitalised?: string)
+  lexicon: { [key: string]: string[] | undefined }
   defaultCategory: string
+  defaultCategoryCapitalised: string | undefined
   parseLexicon (data: string): void
+  tagWordWithDefaults (word: string): string
   tagWord (word: string): string[]
+  addWord (word: string, categories: string[]): void
+  prettyPrint (): string
+  nrEntries (): number
+  size (): number
+  setDefaultCategories (category: string, categoryCapitalised: string): void
+}
+
+export class Corpus {
+  constructor (data: string | Corpus, typeOfCorpus: number, SentenceClass: typeof Sentence)
+  parseBrownCorpus (data: string, SentenceClass: typeof Sentence): void
+  getTags (): string[]
+  splitInTrainAndTest (percentageTrain: number): [Corpus, Corpus]
+  analyse (): void
+  buildLexicon (): Lexicon
+  tag (lexicon: Lexicon): void
+  nrSentences (): number
+  nrWords (): number
+  // Incomplete methods
+  // generateFeatures
+  // prettyPrint
 }
 
-export interface TaggedWord {
+declare interface BrillPOSTaggedWord {
   token: string
   tag: string
 }
 
-export declare class Sentence {
+export class Sentence {
   constructor (data?: string[])
-  taggedWords: TaggedWord[]
+  taggedWords: BrillPOSTaggedWord[]
   addTaggedWord (token: string, tag: string): void
   clone (): Sentence
 }
 
-export declare class BrillPOSTagger {
+export class BrillPOSTagger {
   constructor (lexicon: Lexicon, ruleSet: RuleSet)
   lexicon: Lexicon
   ruleSet: RuleSet
   tag (sentence: string[]): Sentence
+  tagWithLexicon (sentence: string[]): Sentence
+  applyRules (sentence: Sentence): Sentence
+}
+
+export class BrillPOSTester {
+  constructor (lexicon: Lexicon, ruleSet: RuleSet)
+  lexicon: Lexicon
+  ruleSet: RuleSet
+  test (corpus: Corpus, tagger: BrillPOSTagger): [number, number]
+}
+
+export class BrillPOSTrainer {
+  constructor (ruleScoreThreshold: number)
+  ruleScoreThreshold: number
+  corpus: Corpus
+  templates: RuleTemplates
+  positiveRules: RuleSet
+  mapRuleToSites: { [key: string]: { [key: number ]: { [key: number ]: boolean | undefined } | undefined } | undefined }
+  mapSiteToRules: { [key: number]: { [key: number ]: { [key: string ]: TransformationRule | undefined } | undefined } | undefined }
+  selectHighRule (): TransformationRule
+  mapRuleToSite (rule: TransformationRule, i: number, j: number): void
+  mapSiteToRule (i: number, j: number, rule: TransformationRule): void
+  associateSiteWithRule (i: number, j: number, rule: TransformationRule): void
+  siteIsAssociatedWithRule (i: number, j: number, rule: TransformationRule): boolean
+  getSites (rule: TransformationRule): Array<[number, number]>
+  getRules (i: number, j: number): TransformationRule[]
+  disconnectSiteFromRule (i: number, j: number, rule: TransformationRule): void
+  scoreRule (rule: TransformationRule, i: number, j: number): void
+  generatePositiveRules (i: number, j: number): RuleSet
+  scanForPositiveRules (): void
+  scanForSites (): void
+  neighbourhood (i: number, j: number): Array<[number, number]>
+  train (corpus: Corpus, templates: RuleTemplates, lexicon: Lexicon): RuleSet
+  printRulesWithScores (): string
 }

package/lib/natural/classifiers/index.d.ts

@@ -1,135 +1,234 @@
-
-export interface BayesClassifierClassification {
+import events from 'events'
+import { Corpus, Sentence } from '../brill_pos_tagger'
+import { Stemmer } from '../stemmers'
+
+// Start apparatus declarations
+
+// TODO: Once TS declarations are available in NaturalNode/apparatus,
+// replace local definitions with imports:
+// import {
+//   BayesClassifier as ApparatusBayesClassifier,
+//   Classification as ApparatusClassification,
+//   Classifier as ApparatusClassifier,
+//   LogisticRegressionClassifier as ApparatusLogisticRegressionClassifier,
+//   Observation as ApparatusObservation,
+// } from 'apparatus'
+
+declare interface ApparatusClassification {
   label: string
   value: number
 }
 
-export type BayesClassifierCallback = (err: any, classifier: BayesClassifier) => void
-
-declare class BayesClassifier {
-  events: events.EventEmitter
+declare type ApparatusObservation = number[] | { [key: string]: number | string | boolean | undefined }
 
-  constructor (stem?: Stemmer)
-  addDocument (text: string, stem: string): void
-  addDocument (text: string[], stem: string): void
+declare class ApparatusClassifier {
+  addExample (observation: ApparatusObservation, label: string): void
+  classify (observation: ApparatusObservation): string
   train (): void
-  classify (observation: string): string
-  getClassifications (observation: string): BayesClassifierClassification[]
-  save (filename: string, callback: BayesClassifierCallback): void
-  static load (filename: string, stemmer: Stemmer, callback: BayesClassifierCallback): void
-  static restore (classifier: any, stemmer?: Stemmer): BayesClassifier
+  static restore (classifier: string | ApparatusClassifier): ApparatusClassifier
+}
+
+// TODO: Not needed for natural repository, but could be moved to
+// aparatus repository. Temporarily leaving here for copypasta.
+declare class ApparatusBayesClassifier extends ApparatusClassifier {
+  classFeatures: { [key: string]: { [key: number]: number | undefined } | undefined }
+  classTotals: { [key: string]: number | undefined }
+  totalExamples: number
+  smoothing: number
+
+  constructor (smoothing?: number)
+  getClassifications (observation: ApparatusObservation): ApparatusClassification[]
+  probabilityOfClass (observation: ApparatusObservation, label: string): number
+  static restore (classifier: string | ApparatusBayesClassifier): ApparatusBayesClassifier
+}
+
+// TODO: Not needed for natural repository, but could be moved to
+// aparatus repository. Temporarily leaving here for copypasta.
+declare class ApparatusLogisticRegressionClassifier extends ApparatusClassifier {
+  examples: { [key: string]: number[] | undefined }
+  // TODO: These appear used
+  // features: any[]
+  // featurePositions: { [key: string]: any | undefined }
+  maxFeaturePosition: number
+  classifications: string[]
+  exampleCount: number
+
+  createClassifications (): number[][]
+  addExample (data: number[], classification: string): void
+  getClassifications (observation: number[]): ApparatusClassification[]
+  static restore (classifier: string | ApparatusLogisticRegressionClassifier): ApparatusLogisticRegressionClassifier
 }
 
-interface LogisticRegressionClassifierClassification {
+// End apparatus declarations
+
+declare interface ClassifierDoc {
   label: string
-  value: number
+  text: string
+}
+
+declare interface ClassifierOptions {
+  keepStops?: boolean
 }
 
-export type LogisticRegressionCallback = (err: any, result: LogisticRegressionClassifier) => void
+declare type ClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: ClassifierBase) => void
 
-declare class LogisticRegressionClassifier {
+declare class ClassifierBase {
+  classifier: ApparatusClassifier
+  docs: ClassifierDoc[]
+  features: { [key: string]: number | undefined }
+  stemmer: Stemmer
+  lastAdded: number
   events: events.EventEmitter
 
-  addDocument (text: string, stem: string): void
-  addDocument (text: string[], stem: string): void
+  constructor (classifier: ApparatusClassifier, stemmer?: Stemmer)
+  addDocument (text: string | string[], classification: string): void
+  removeDocument (text: string | string[], classification: string): void
+  textToFeatures (observation: string | string[]): number[]
   train (): void
-  classify (observation: string): string
-  getClassifications (observation: string): LogisticRegressionClassifierClassification[]
-  save (filename: string, callback: LogisticRegressionCallback): void
-  static load (filename: string, stemmer: Stemmer, callback: LogisticRegressionCallback): void
-  static restore (classifier: any, stemmer?: Stemmer): LogisticRegressionClassifier
+  retrain (): void
+  getClassifications (observation: string | string[]): ApparatusClassification[]
+  classify (observation: string | string[]): string
+  setOptions (options: ClassifierOptions): void
+  save (filename: string, callback: ClassifierCallback): void
 }
 
-export type MaxEntClassifierCallback = (err: any, result: MaxEntClassifier) => void
+declare type BayesClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: BayesClassifier) => void
+
+export class BayesClassifier extends ClassifierBase {
+  constructor (stemmer?: Stemmer, smoothing?: number)
+  static load (filename: string, stemmer: Stemmer | null | undefined, callback: BayesClassifierCallback): void
+  static restore (classifier: BayesClassifier, stemmer?: Stemmer): BayesClassifier
+}
 
-declare class MaxEntClassifier {
+declare type LogisticRegressionClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: LogisticRegressionClassifier) => void
+
+export class LogisticRegressionClassifier extends ClassifierBase {
+  constructor (stemmer?: Stemmer)
+  static load (filename: string, stemmer: Stemmer | null | undefined, callback: LogisticRegressionClassifierCallback): void
+  static restore (classifier: LogisticRegressionClassifier, stemmer?: Stemmer): LogisticRegressionClassifier
+}
+
+declare type MaxEntClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: MaxEntClassifier | null) => void
+
+export class MaxEntClassifier {
   sample: Sample
   features: FeatureSet
 
   constructor (features: FeatureSet, sample: Sample)
   addElement (x: Element): void
   addDocument (context: Context, classification: string, elementClass: Element): void
-  train (maxIterations: number, minImprovement: number, approxExpectation: object): void
-  getClassifications (b: Context): array
+  train (maxIterations: number, minImprovement: number, unused: any): void
+  getClassifications (b: Context): ApparatusClassification[]
   classify (b: Context): string
+  // These are not static like in other Classifier classes
   save (filename: string, callback: MaxEntClassifierCallback): void
-  static load (filename: string, elementClass: Element, callback: MaxEntClassifierCallback): void
+  load (filename: string, elementClass: Element, callback: MaxEntClassifierCallback): void
+}
+
+declare class Distribution {
+  alpha: number[]
+  featureSet: FeatureSet
+  sample: Sample
+
+  constructor (alpha: number[], featureSet: FeatureSet, sample: Sample)
+  toString (): string
+  weight (x: Element): number
+  calculateAPriori (x: Element): number
+  prepareWeights (): void
+  calculateAPosteriori (x: Element): number
+  aPosterioriNormalisation (b: Context): number
+  prepareAPosterioris (): void
+  prepare (): void
+  KullbackLieblerDistance (): number
+  logLikelihood (): number
+  entropy (): number
+  checkSum (): number
 }
 
-export type FeatureFunction = (x: Element) => number
+declare type FeatureFunction = (x: Element) => number
 
-declare class Feature {
+export class Feature {
   evaluate: FeatureFunction
   name: string
-  parameters: array
+  parameters: string[]
 
-  constructor (f: FeatureFunction, name: string, parameters: array)
+  constructor (f: FeatureFunction, name: string, parameters: string[])
   apply (x: Element): number
   expectationApprox (p: Distribution, sample: Sample): number
-  expectation (p: Distribution, A: array, B: array): number
+  expectation (p: Distribution, A: string[], B: Context[]): number
   observedExpectation (sample: Sample): number
 }
 
-declare class FeatureSet {
-  features: array
-  map: object
+export class FeatureSet {
+  features: Feature[]
+  map: { [key: string]: boolean | undefined }
 
-  addFeature (feature: FeatureFunction): boolean
-  featureExists (feature: FeatureFunction): boolean
-  getFeatures (): array
+  addFeature (feature: Feature): boolean
+  featureExists (feature: Feature): boolean
+  getFeatures (): Feature[]
   size (): number
   prettyPrint (): string
 }
 
-export type SampleCallback = (err: any, result: Sample) => void
+declare type SampleCallback = (err: NodeJS.ErrnoException | null, sample?: Sample | null) => void
 
-declare class Sample {
-  frequencyOfContext: object
-  frequency: object
-  classes: array
+export class Sample {
+  frequencyOfContext: { [key: string]: number | undefined }
+  frequency: { [key: string]: number | undefined }
+  classes: string[]
 
-  constructor (elements)
+  constructor (elements?: Element[])
   analyse (): void
   addElement (x: Element): void
   observedProbabilityOfContext (context: Context): number
-  observedProbability (x): number
+  observedProbability (x: Element): number
   size (): number
-  getClasses (): array
+  getClasses (): string[]
   generateFeatures (featureSet: FeatureSet): void
   save (filename: string, callback: SampleCallback): void
-  load (filename: string, elementClass: Element, callback: SampleCallback): void
+  load (filename: string, elementClass: typeof Element, callback: SampleCallback): void
 }
 
-declare class Context {
+export class Context {
+  key: string | undefined
+
+  constructor (data: any)
   toString (): string
 }
 
-declare class Element {
-  constructor (a: any, b: Context)
-  key (): string
+export class Element {
+  a: string
+  b: Context
+  key: string | undefined
+
+  constructor (a: string, b: Context)
+  toString (): string
 }
 
-declare class SEElement extends Element {
+export class SEElement extends Element {
+  constructor (a: string, b: Context)
   generateFeatures (featureSet: FeatureSet): void
 }
 
-declare class POSElement extends Element {
+export class POSElement extends Element {
+  constructor (a: string, b: Context)
   generateFeatures (featureSet: FeatureSet): void
 }
 
-declare class GISScaler {
+export class GISScaler {
   constructor (featureSet: FeatureSet, sample: Sample)
-  calculateMaxSumOfFeatures (): void
+  calculateMaxSumOfFeatures (): boolean
   addCorrectionFeature (): void
   run (maxIterations: number, minImprovement: number): Distribution
 }
 
-declare class MESentence {
-  constructor (data: array): void
+export class MESentence {
+  constructor (data?: string[])
   generateSampleElements (sample: Sample): void
 }
 
-declare class MECorpus {
+export class MECorpus {
+  constructor (data: string | Corpus, BROWN: number, SentenceClass: typeof Sentence)
   generateSample (): Sample
-  splitInTrainAndTest (percentageTrain: number): array
+  splitInTrainAndTest (percentageTrain: number): [MECorpus, MECorpus]
 }

package/lib/natural/distance/index.d.ts

@@ -23,33 +23,42 @@ THE SOFTWARE.
 
 // Based on type definitions on Definitely Typed
 
-export interface JaroWinklerOptions {
-  dj?: number | undefined
-  // default: false
-  ignoreCase?: boolean | undefined
+declare interface JaroWinklerOptions {
+  dj?: number
+  /** @default false */
+  ignoreCase?: boolean
 }
-export declare function JaroWinklerDistance (s1: string, s2: string, options: JaroWinklerOptions): number
 
-export interface DamerauLevenshteinDistanceOptions {
+export function JaroWinklerDistance (s1: string, s2: string, options: JaroWinklerOptions): number
+
+declare interface DamerauLevenshteinDistanceOptions {
   /** @default 1 */
-  insertion_cost?: number | undefined
+  insertion_cost?: number
   /** @default 1 */
-  deletion_cost?: number | undefined
+  deletion_cost?: number
   /** @default 1 */
-  substitution_cost?: number | undefined
+  substitution_cost?: number
   /** @default 1 */
-  transposition_cost?: number | undefined
+  transposition_cost?: number
   /** @default false */
-  search?: boolean | undefined
+  search?: boolean
   /** @default false */
-  restricted?: boolean | undefined
-  damerau?: boolean | undefined
+  restricted?: boolean
+  damerau?: boolean
 }
-interface SubstringDistanceResult {
+
+declare interface SubstringDistanceResult {
   substring: string
   distance: number
+  offset: number
 }
-export declare function LevenshteinDistance (source: string, target: string, options?: DamerauLevenshteinDistanceOptions): number
+
+export function LevenshteinDistance (
+  source: string,
+  target: string,
+  options?: DamerauLevenshteinDistanceOptions
+): number
+
 /**
  * Returns the Damerau-Levenshtein distance between strings. Counts the distance
  * between two strings by returning the number of edit operations required to
@@ -63,14 +72,19 @@ export function DamerauLevenshteinDistance (
   target: string,
   options: DamerauLevenshteinDistanceOptions & { search: true }
 ): SubstringDistanceResult
+
 export function DamerauLevenshteinDistance (
   source: string,
   target: string,
-  options?: DamerauLevenshteinDistanceOptions & { search?: false | undefined }
+  options?: DamerauLevenshteinDistanceOptions & { search?: false }
 ): number
+
 export function DamerauLevenshteinDistance (
   source: string,
   target: string,
   options: DamerauLevenshteinDistanceOptions & { search: boolean }
 ): number | SubstringDistanceResult
-export declare function DiceCoefficient (str1: string, str2: string): number
+
+export function DiceCoefficient (str1: string, str2: string): number
+
+export function HammingDistance (str1: string, str2: string, ignoreCase: boolean): number

package/lib/natural/inflectors/index.d.ts

@@ -20,16 +20,19 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export declare class NounInflector {
+export class NounInflector {
   pluralize (token: string): string
   singularize (token: string): string
 }
 
-export declare let CountInflector: {
+export class NounInflectorFr extends NounInflector {}
+export class NounInflectorJa extends NounInflector {}
+
+export let CountInflector: {
   nth: (i: number) => string
 }
 
-export declare class PresentVerbInflector {
+export class PresentVerbInflector {
   pluralize (token: string): string
   singularize (token: string): string
 }

package/lib/natural/ngrams/index.d.ts

@@ -22,14 +22,30 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-declare interface ngramsFunctions {
-  bigrams: (sequence: string | string[], startSymbol?: string, endSymbol?: string) => string[][]
-  // bigrams: (sequence: string[], startSymbol?: string, endSymbol?: string) => string[][]
-  trigrams: (sequence: string | string[], startSymbol?: string, endSymbol?: string) => string[][]
-  // trigrams: (sequence: string[], startSymbol?: string, endSymbol?: string) => string[][]
-  ngrams: (sequence: string | string[], n: number, startSymbol?: string, endSymbol?: string) => string[][]
-  // ngrams: (sequence: string[], n: number, startSymbol?: string, endSymbol?: string) => string[][]
+import Tokenizer from '../tokenizers/tokenizer'
+
+declare interface NGramsStats {
+  ngrams: string[][]
+  frequencies: { [key: string]: number | undefined }
+  Nr: { [key: number]: number | undefined }
+  numberOfNgrams: number
+}
+
+declare interface NGramsFunctions {
+  // To support function overloads where the return type can differ depending on a parameter
+  // value, method signatures are necessary. Key collisions occur with property signatures.
+  /* eslint-disable @typescript-eslint/method-signature-style */
+  setTokenizer (t: Tokenizer): void
+  ngrams (sequence: string | string[], n: number, startSymbol: string | undefined, endSymbol: string | undefined, stats: true): NGramsStats
+  ngrams (sequence: string | string[], n: number, startSymbol?: string, endSymbol?: string, stats?: boolean): string[][]
+  bigrams (sequence: string | string[], startSymbol: string | undefined, endSymbol: string | undefined, stats: true): NGramsStats
+  bigrams (sequence: string | string[], startSymbol?: string, endSymbol?: string, stats?: boolean): string[][]
+  trigrams (sequence: string | string[], startSymbol: string | undefined, endSymbol: string | undefined, stats: true): NGramsStats
+  trigrams (sequence: string | string[], startSymbol?: string, endSymbol?: string, stats?: boolean): string[][]
+  multrigrams (sequence: string | string[], n: number, startSymbol: string | undefined, endSymbol: string | undefined, stats: true): NGramsStats
+  multrigrams (sequence: string | string[], n: number, startSymbol?: string, endSymbol?: string, stats?: boolean): string[][]
+  /* eslint-enable @typescript-eslint/method-signature-style */
 }
 
-export declare let NGrams: ngramsFunctions
-export declare let NGramsZH: ngramsFunctions
+export let NGrams: NGramsFunctions
+export let NGramsZH: NGramsFunctions

package/lib/natural/normalizers/index.d.ts

@@ -0,0 +1,28 @@
+/*
+Copyright (c) 2022, Dylan R. E. Moonfire <https://github.com/dmoonfire>,
+Emily Marigold Klassen <https://github.com/forivall>,
+Hugo W.L. ter Doest <https://github.com/Hugo-ter-Doest>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+export function normalize (tokens: string | string[]): string[]
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export function normalize_ja (str: string): string
+export function removeDiacritics (str: string): string

package/lib/natural/phonetics/index.d.ts

@@ -20,22 +20,24 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export interface Phonetic {
+export interface Phonetic<T> {
   compare: (stringA: string, stringB: string) => boolean
-  process: (token: string, maxLength?: number) => string
+  tokenizeAndPhoneticize: (str: string, phoneticsProcessor: Phonetic<T>, keepStops: boolean) => T
 }
 
-declare let Metaphone: {
-  compare: (stringA: string, stringB: string) => boolean
+export let Metaphone: Phonetic<string> & {
   process: (token: string, maxLength?: number) => string
 }
 
-declare let SoundEx: {
-  compare: (stringA: string, stringB: string) => boolean
+export let SoundEx: Phonetic<string> & {
   process: (token: string, maxLength?: number) => string
 }
 
-declare let DoubleMetaphone: {
-  compare: (stringA: string, stringB: string) => boolean
+export let DoubleMetaphone: Phonetic<string[]> & {
   process: (token: string, maxLength?: number) => string[]
+  isVowel: (c: string) => string | RegExpMatchArray | null
+}
+
+export let SoundExDM: Phonetic<string> & {
+  process: (word: string, codeLength?: number) => string
 }

package/lib/natural/sentiment/index.d.ts

@@ -21,7 +21,20 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
+import { Stemmer } from '../stemmers'
+
+export type AfinnLanguageType = 'afinn'
+export type AfinnLanguage = 'English' | 'Spanish' | 'Portuguese'
+
+export type SenticonLanguageType = 'senticon'
+export type SenticonLanguage = 'Spanish' | 'English' | 'Galician' | 'Catalan' | 'Basque'
+
+export type PatternLanguageType = 'pattern'
+export type PatternLanguage = 'Dutch' | 'Italian' | 'English' | 'French'
+
 export declare class SentimentAnalyzer {
-  constructor (language: string, stemmer: Stemmer, vocabulary: string)
+  constructor (language: AfinnLanguage, stemmer: Stemmer, type: AfinnLanguageType)
+  constructor (language: SenticonLanguage, stemmer: Stemmer, type: SenticonLanguageType)
+  constructor (language: PatternLanguage, stemmer: Stemmer, type: PatternLanguageType)
   getSentiment (words: string[]): number
 }

package/lib/natural/stemmers/index.d.ts

@@ -20,26 +20,27 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-// Based on the type definitions on definitely typed
-
-export interface Stemmer {
+declare interface Stemmer {
   stem: (token: string) => string
-  tokenizeAndStem: (text: string) => string[]
-  attach: () => void
+  addStopWord: (stopWord: string) => void
+  addStopWords: (moreStopWords: string[]) => void
+  removeStopWord: (stopWord: string) => void
+  removeStopWords: (moreStopWords: string[]) => void
+  tokenizeAndStem: (text: string, keepStops?: boolean) => string[]
 }
 
-declare const CarryStemmerFr: Stemmer
-declare const LancasterStemmer: Stemmer
-declare const PorterStemmer: Stemmer
-declare const PorterStemmerEs: Stemmer
-declare const PorterStemmerFa: Stemmer
-declare const PorterStemmerFr: Stemmer
-declare const PorterStemmerDe: Stemmer
-declare const PorterStemmerIt: Stemmer
-declare const PorterStemmerNl: Stemmer
-declare const PorterStemmerNo: Stemmer
-declare const PorterStemmerPt: Stemmer
-declare const PorterStemmerRu: Stemmer
-declare const PorterStemmerSv: Stemmer
-declare const StemmerId: Stemmer
-declare const StemmerJa: Stemmer
+export let CarryStemmerFr: Stemmer
+export let LancasterStemmer: Stemmer
+export let PorterStemmer: Stemmer
+export let PorterStemmerDe: Stemmer
+export let PorterStemmerEs: Stemmer
+export let PorterStemmerFa: Stemmer
+export let PorterStemmerFr: Stemmer
+export let PorterStemmerIt: Stemmer
+export let PorterStemmerNl: Stemmer
+export let PorterStemmerNo: Stemmer
+export let PorterStemmerPt: Stemmer
+export let PorterStemmerRu: Stemmer
+export let PorterStemmerSv: Stemmer
+export let StemmerId: Stemmer
+export let StemmerJa: Stemmer

package/lib/natural/tfidf/index.d.ts

@@ -23,20 +23,31 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export type TfIdfCallback = (i: number, measure: number) => void
+import Tokenizer from '../tokenizers/tokenizer'
 
-export interface TfIdfTerm {
+declare type TfIdfCallback = (i: number, measure: number, key?: string) => void
+
+declare interface TfIdfTerm {
   term: string
+  tf: number
+  idf: number
   tfidf: number
 }
 
-export declare class TfIdf {
-  constructor (deserialized?: any)
-  addDocument (document: string, key?: string, restoreCache?: boolean): void
-  addDocument (document: string[], key?: string, restoreCache?: boolean): void
+declare type TfIdfDocument = { __key: string | undefined } | { [key: string]: number | undefined }
+
+export class TfIdf {
+  documents: TfIdfDocument | undefined
+  _idfCache: { [key: string]: number } | undefined
+
+  constructor (deserialized?: { documents: TfIdfDocument[] })
+  idf (term: string, force?: boolean): number
+  addDocument (document: string | string[], key?: string, restoreCache?: boolean): void
   addFileSync (path: string, encoding?: string, key?: string, restoreCache?: boolean): void
-  tfidf (terms: string, d: number): void
-  tfidfs (terms: string, callback: TfIdfCallback): void
-  tfidfs (terms: string[], callback: TfIdfCallback): void
+  tfidf (terms: string | string[], d: number): number
+  tfidfs (terms: string | string[], callback?: TfIdfCallback): number[]
   listTerms (d: number): TfIdfTerm[]
+  setTokenizer (t: Tokenizer): void
+  setStopwords (customStopwords: string[]): boolean
+  static tf (term: string, document: TfIdfDocument): number
 }

package/lib/natural/tokenizers/index.d.ts

@@ -20,83 +20,117 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-// Based on type definitions on Definitely Typed
-
-export interface Tokenizer {
-  tokenize: (text: string) => string[]
+declare class Tokenizer {
+  trim: (array: string[]) => string[]
 }
-export declare class WordTokenizer implements Tokenizer {
+
+export class AggressiveTokenizerNl extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class TreebankWordTokenizer implements Tokenizer {
+
+export class AggressiveTokenizerFa extends Tokenizer {
+  clearEmptyString (array: string[]): string[]
+  clearText (text: string): string
   tokenize (text: string): string[]
 }
-export interface RegexTokenizerOptions {
-  pattern?: RegExp | undefined
-  discardEmpty?: boolean | undefined
-}
-export declare class RegexpTokenizer implements Tokenizer {
-  constructor (options: RegexTokenizerOptions)
+
+export class AggressiveTokenizerFr extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class OrthographyTokenizer implements Tokenizer {
-  constructor (options: RegexTokenizerOptions & { language: string })
+
+export class AggressiveTokenizerDe extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class WordPunctTokenizer implements Tokenizer {
+
+export class AggressiveTokenizerRu extends Tokenizer {
+  withoutEmpty (array: string[]): string[]
+  clearText (text: string): string
   tokenize (text: string): string[]
 }
-export declare class SentenceTokenizer implements Tokenizer {
+
+export class AggressiveTokenizerEs extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class SentenceTokenizerNew implements Tokenizer {
+
+export class AggressiveTokenizerIt extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class CaseTokenizer implements Tokenizer {
-  tokenize (text: string, preserveApostrophy?: boolean): string[]
-}
-export declare class AggressiveTokenizer implements Tokenizer {
+
+export class AggressiveTokenizerPl extends Tokenizer {
+  withoutEmpty (array: string[]): string[]
+  clearText (text: string): string
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerEs implements Tokenizer {
+
+export class AggressiveTokenizerPt extends Tokenizer {
+  withoutEmpty (array: string[]): string[]
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerFa implements Tokenizer {
+
+export class AggressiveTokenizerNo extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerFr implements Tokenizer {
+
+export class AggressiveTokenizerSv extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerDe implements Tokenizer {
+
+export class AggressiveTokenizerVi extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerId implements Tokenizer {
+
+export class AggressiveTokenizerId extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerIt implements Tokenizer {
+
+export class AggressiveTokenizer extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerNl implements Tokenizer {
+
+export class CaseTokenizer extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerNo implements Tokenizer {
-  tokenize (text: string): string[]
+
+declare interface RegexTokenizerOptions {
+  pattern?: RegExp
+  discardEmpty?: boolean
+  gaps?: boolean
 }
-export declare class AggressiveTokenizerPl implements Tokenizer {
-  tokenize (text: string): string[]
+
+export class RegexpTokenizer extends Tokenizer {
+  constructor (opts?: RegexTokenizerOptions)
+  tokenize (text: string): string[] | null
 }
-export declare class AggressiveTokenizerPt implements Tokenizer {
-  tokenize (text: string): string[]
+
+declare interface OrthographyTokenizerOptions extends RegexTokenizerOptions {
+  language: string
+}
+
+export class OrthographyTokenizer extends RegexpTokenizer {
+  constructor (options: OrthographyTokenizerOptions)
 }
-export declare class AggressiveTokenizerRu implements Tokenizer {
+
+export class WordTokenizer extends RegexpTokenizer {
+  constructor (options?: RegexTokenizerOptions)
+}
+
+export class WordPunctTokenizer extends RegexpTokenizer {
+  constructor (options?: RegexTokenizerOptions)
+}
+
+export class TreebankWordTokenizer extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerSv implements Tokenizer {
+
+export class TokenizerJa extends Tokenizer {
+  removePuncTokens (tokens: string[]): string[]
   tokenize (text: string): string[]
 }
-export declare class AggressiveTokenizerVi implements Tokenizer {
+
+export class SentenceTokenizer extends Tokenizer {
   tokenize (text: string): string[]
 }
-export declare class TokenizerJa implements Tokenizer {
+
+export class SentenceTokenizerNew extends Tokenizer {
   tokenize (text: string): string[]
 }

package/lib/natural/transliterators/index.d.ts

@@ -21,7 +21,4 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-// First define a function type for transliterators ...
-export declare type TransliteratorFunction = (s: string) => string
-// and then export the Japanese transliterator as a function of that type
-export declare let TransliterateJa: TransliteratorFunction
+export let TransliterateJa: (s: string) => string

package/lib/natural/trie/index.d.ts

@@ -22,12 +22,13 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export declare class Trie {
+export class Trie {
   constructor (caseSensitive?: boolean)
-  addString (text: string): boolean
-  addStrings (strings: string[]): void
-  contains (token: string): boolean
-  findPrefix (text: string): string[]
-  findMatchesOnPath (text: string): string[]
-  keysWithPrefix (text: string): string[]
+  addString (string: string): boolean
+  addStrings (list: string[]): void
+  contains (string: string): boolean
+  findPrefix (search: string): string[]
+  findMatchesOnPath (search: string): string[]
+  keysWithPrefix (prefix: string): string[]
+  getSize (): number
 }

package/lib/natural/util/index.d.ts

@@ -23,21 +23,49 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export declare class EdgeWeightedDigraph {
+export let stopwords: string[]
+
+declare class Bag<T> {
+  add (element: T): Bag<T>
+  isEmpty (): boolean
+  contains (item: T): boolean
+  unpack (): T[]
+}
+
+declare class DirectedEdge {
+  constructor (start: number, end: number, weight: number)
+  weight (): number
+  from (): number
+  to (): number
+  toString (): string
+}
+
+export class EdgeWeightedDigraph {
+  edgesNum: number
+  adj: Array<Bag<DirectedEdge>>
+
   add (start: number, end: number, weight: number): void
+  addEdge (e: DirectedEdge): void
   v (): number
   e (): number
+  getAdj (v: number): DirectedEdge[]
+  edges (): DirectedEdge[]
+  toString (): string
 }
 
-export declare class ShortestPathTree {
-  constructor (diagraph: EdgeWeightedDigraph, startVertex: number)
+export class ShortestPathTree {
+  constructor (diagraph: EdgeWeightedDigraph, start: number)
+  relaxEdge (e: DirectedEdge): void
+  relaxVertex (digraph: EdgeWeightedDigraph, vertex: number, tree: ShortestPathTree): void
   getDistTo (vertex: number): number
   hasPathTo (vertex: number): boolean
   pathTo (vertex: number): number[]
 }
 
-export declare class LongestPathTree {
-  constructor (diagraph: EdgeWeightedDigraph, startVertex: number)
+export class LongestPathTree {
+  constructor (diagraph: EdgeWeightedDigraph, start: number)
+  relaxEdge (e: DirectedEdge): void
+  relaxVertex (digraph: EdgeWeightedDigraph, vertex: number, tree: LongestPathTree): void
   getDistTo (vertex: number): number
   hasPathTo (vertex: number): boolean
   pathTo (vertex: number): number[]

package/lib/natural/wordnet/index.d.ts

@@ -23,20 +23,76 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-export interface WordNetLookupResults {
+declare interface DataPtr {
+  pointerSymbol: string
   synsetOffset: number
   pos: string
+  sourceTarget: string
+}
+
+declare interface DataRecord {
+  synsetOffset: number
+  lexFilenum: number
+  pos: string
+  wCnt: number
   lemma: string
   synonyms: string[]
+  lexId: string
+  ptrs: DataPtr[]
   gloss: string
+  def: string
+  exp: string[]
 }
 
-export type WordNetLookupCallback = (results: WordNetLookupResults[]) => void
+declare type DataRecordCallback = (results: DataRecord) => void
+declare type DataRecordsCallback = (results: DataRecord[]) => void
 
-export type WordNetGetCallback = (results: WordNetLookupResults) => void
+declare class DataFile {
+  constructor (dataDir: string, name: string)
+  get (location: number, callback: DataRecordCallback): void
+}
+
+declare interface IndexRecord {
+  lemma: string
+  pos: string
+  ptrSymbol: string[]
+  senseCnt: number
+  tagsenseCnt: number
+  synsetOffset: number[]
+}
+
+declare type IndexRecordCallback = (results: IndexRecord) => void
+
+declare interface IndexFind {
+  status: string
+  key: string
+  line: string
+  tokens: string[]
+}
+
+declare type IndexFindCallback = (results: IndexFind) => void
+
+declare class IndexFile {
+  constructor (dataDir: string, name: string)
+  lookup (word: string, callback: IndexRecordCallback): void
+  lookupFromFile (word: string, callback: IndexRecordCallback): void
+  find (word: string, callback: IndexFindCallback): void
+}
+
+declare interface WordNetLookupFile {
+  index: IndexFile
+  data: DataFile
+}
 
-export declare class WordNet {
-  constructor (filename?: string)
-  lookup (word: string, callback: WordNetLookupCallback): void
-  get (synsetOffset: number, pos: string, callback: WordNetGetCallback): void
+export class WordNet {
+  constructor (dataDir?: string)
+  get (synsetOffset: number, pos: string, callback: DataRecordCallback): void
+  lookup (word: string, callback: DataRecordsCallback): void
+  lookupFromFiles (files: WordNetLookupFile[], results: DataRecord[], word: string, callback: DataRecordsCallback): void
+  pushResults (data: DataFile, results: DataRecord[], offsets: number[], callback: DataRecordsCallback): void
+  loadResultSynonyms (synonyms: DataRecord[], results: DataRecord[], callback: DataRecordsCallback): void
+  loadSynonyms (synonyms: DataRecord[], results: DataRecord[], ptrs: DataPtr[], callback: DataRecordsCallback): void
+  lookupSynonyms (word: string, callback: DataRecordsCallback): void
+  getSynonyms (record: DataRecord, callback: DataRecordsCallback): void
+  getDataFile (pos: string): DataFile
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.2.0",
+  "version": "6.3.0",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",

package/tsconfig.json

@@ -12,8 +12,7 @@
     "outDir": "./dist",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
-    "strict": true,
-    "skipLibCheck": true
+    "strict": true
   },
   "include": [
     "lib/natural/**/*",