natural 6.11.1 → 7.0.0

package/ts_spec/brill_pos_tagger_spec.ts

@@ -0,0 +1,79 @@
+/*
+Unit test for Brill's POS Tagger: test against the pos module
+Copyright (C) 2019 Hugo W.L. ter Doest
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict'
+
+import { WordTokenizer, Lexicon, RuleSet, BrillPOSTagger } from 'lib/natural'
+import type { Sentence, TagResults } from 'lib/natural'
+
+import * as englishSentences from '../spec/test_data/NYT-20150205-picassos-granddaughter-plans-to-sell-art-worrying-the-market.json'
+
+import jsonData from '../spec/test_data/NYT-20150205-picassos-granddaughter-plans_expected_tag_results.json'
+
+import * as dutchSentences from '../spec/test_data/Volkskrant-20150205-Knot-geldpers-aanzetten-is-paardenmiddel-voor-half-procent-inflatie.json'
+
+const englishTagResults: TagResults = jsonData as TagResults
+const DEBUG = false
+
+// Compares two tagged sentences. First one is in the old POSJS format, i.e.
+// an array of two position arrays. The second one is a Sentence object
+// that holds an array of objects for each position {token: "string", tag: "string"}
+function compareTaggedSentences (sentenceInOldFormat: string[][], sentenceInNewFormat: Sentence): boolean {
+  let equal = true
+  sentenceInOldFormat.forEach(function (wordPlusTag: string[], index: number) {
+    equal = equal &&
+      (wordPlusTag[1] === sentenceInNewFormat.taggedWords[index].tag)
+    DEBUG && console.log(wordPlusTag[1] + ' ' + sentenceInNewFormat.taggedWords[index].tag)
+  })
+  return equal
+}
+
+describe('Brill\'s POS Tagger', function () {
+  let brillPOSTagger: BrillPOSTagger
+  let lexicon: Lexicon
+  let ruleSet: RuleSet
+  const tokenizer = new WordTokenizer()
+
+  it('should process an English newspaper article just like the dariusk/pos-js module', function () {
+    lexicon = new Lexicon('EN', 'NN')
+    expect(lexicon.nrEntries()).toBeGreaterThan(0)
+    ruleSet = new RuleSet('EN')
+    expect(ruleSet.nrRules()).toBeGreaterThan(0)
+    brillPOSTagger = new BrillPOSTagger(lexicon, ruleSet)
+
+    englishSentences.sentences.forEach(function (sentence: string, index: number) {
+      const tokenizedSentence = tokenizer.tokenize(sentence)
+      const taggedSentence = brillPOSTagger.tag(tokenizedSentence)
+      expect(compareTaggedSentences(englishTagResults.results[index], taggedSentence)).toBe(true)
+    })
+  })
+
+  it('should tag a Dutch news paper article', function () {
+    lexicon = new Lexicon('DU', 'N')
+    expect(lexicon.nrEntries()).toBeGreaterThan(0)
+    const ruleSet = new RuleSet('DU')
+    expect(ruleSet.nrRules()).toBeGreaterThan(0)
+    brillPOSTagger = new BrillPOSTagger(lexicon, ruleSet)
+
+    dutchSentences.sentences.forEach(function (sentence, index) {
+      const tokenizedSentence = tokenizer.tokenize(sentence)
+      const taggedSentence = brillPOSTagger.tag(tokenizedSentence)
+      expect(tokenizedSentence.length).toEqual(taggedSentence.taggedWords.length)
+    })
+  })
+})

package/ts_spec/brill_pos_trainer_spec.ts

@@ -0,0 +1,107 @@
+/*
+Unit test for Brill's POS Trainer
+Copyright (C) 2019 Hugo W.L. ter Doest
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict'
+
+import {
+  Corpus,
+  ruleTemplates,
+  RuleTemplate,
+  Sentence,
+  BrillPOSTrainer,
+  BrillPOSTagger,
+  BrillPOSTester
+} from 'lib/natural'
+
+import type {
+  Lexicon,
+  RuleSet
+} from 'lib/natural'
+
+// Import a JSON file with the brown corpus
+import brownCorpus from '../spec/test_data/browntag_nolines_excerpt.json'
+
+const DEBUG = false
+const JSON_FLAG = 2
+
+function selectRuleTemplates (templateNames: string[]): RuleTemplate[] {
+  const templates: RuleTemplate[] = []
+  templateNames.forEach(function (name: string) {
+    if (ruleTemplates[name] !== undefined) {
+      const template = new RuleTemplate(name, ruleTemplates[name])
+      templates.push(template)
+    }
+  })
+  return templates
+}
+
+describe('Brill\'s POS Trainer', function () {
+  let corpus = null
+  let corpora: Corpus[]
+  const percentageTrain = 60
+  let trainLexicon: Lexicon
+  // Templates consider only tags, no words
+  const templateNames = [
+    'NEXT-TAG',
+    'PREV-TAG',
+    'PREV-1-OR-2-OR-3-TAG',
+    'PREV-1-OR-2-TAG',
+    'NEXT1OR2TAG',
+    'NEXT1OR2OR3TAG',
+    'SURROUNDTAG',
+    'PREV2TAG',
+    'NEXT2TAG'
+  ]
+  let templates: RuleTemplate[]
+  let trainer = null
+  let ruleSet: RuleSet
+
+  it('should split the corpus in a training and testing corpus', function () {
+    corpus = new Corpus(brownCorpus, JSON_FLAG, Sentence)
+    DEBUG && console.log('Corpus: ' + JSON.stringify(corpus, null, 2))
+    corpora = corpus.splitInTrainAndTest(percentageTrain)
+    expect(corpora[0].nrSentences() + corpora[1].nrSentences()).toEqual(corpus.nrSentences())
+  })
+
+  it('should build a lexicon from the training corpus', function () {
+    trainLexicon = corpora[0].buildLexicon()
+    // Set default category to noun (NN)
+    // and default category for capitalised words to proper noun (NP)
+    trainLexicon.setDefaultCategories('NN', 'NP')
+    expect(trainLexicon.nrEntries()).not.toEqual(0)
+  })
+
+  it('should set up the rule templates', function () {
+    templates = selectRuleTemplates(templateNames)
+    expect(templates.length).toEqual(templateNames.length)
+  })
+
+  it('should train on the training corpus to derive transformation rules', function () {
+    trainer = new BrillPOSTrainer(1)
+    ruleSet = trainer.train(corpora[0], templates, trainLexicon)
+    expect(ruleSet.nrRules()).toBeGreaterThan(0)
+    expect(trainer.printRulesWithScores()).not.toEqual('')
+  })
+
+  it('should test the derived transformation rules on the test corpus', function () {
+    const tagger = new BrillPOSTagger(trainLexicon, ruleSet)
+    const tester = new BrillPOSTester()
+    const scores = tester.test(corpora[1], tagger)
+    expect(scores[1]).toBeGreaterThan(0)
+  })
+})

package/ts_spec/carry_stemmer_fr_spec.ts

@@ -0,0 +1,97 @@
+/*
+Copyright (c) 2020, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CarryStemmerFr as stemmer } from 'lib/natural'
+
+// const snowBallDict = require('spec/test_data/snowball_fr.json')
+
+const tests = [
+  {
+    input: 'action',
+    output: 'ac'
+  },
+  {
+    input: 'acteur',
+    output: 'ac'
+  },
+  {
+    input: 'actrices',
+    output: 'ac'
+  },
+  {
+    input: 'Dleyton',
+    output: 'Dleyton'
+  },
+  {
+    input: 'volera',
+    output: 'vol'
+  }
+]
+
+const testSentences = [
+  {
+    input: 'Le petit cheval de manège',
+    output: ['pet', 'cheval', 'manèg']
+  }
+]
+
+describe('Carry stemmer', function () {
+  tests.forEach(test => {
+    it('stems ' + test.input + ' correctly', function () {
+      expect(stemmer.stem(test.input)).toBe(test.output)
+    })
+  })
+
+  testSentences.forEach(test => {
+    it('tokenizes and stems ' + test.input + ' correctly', function () {
+      expect(stemmer.tokenizeAndStem(test.input)).toEqual(test.output)
+    })
+  })
+
+  /*
+  it('should perform stemming on a lot of words', function() {
+    var ok = [];
+    var ko = [];
+
+    Object.keys(snowBallDict).forEach(word => {
+
+      var stemmed = stemmer.stem(word);
+      var expectedStem = snowBallDict[word];
+
+      if (stemmed === expectedStem) {
+        ok.push(word);
+      }
+      else {
+        ko.push({
+          word: word,
+          expected: expectedStem,
+          actual: stemmed
+        });
+      }
+    });
+
+    expect(ko.length).toBe(0);
+  });
+*/
+})

package/ts_spec/classifier_spec.ts

@@ -0,0 +1,104 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { BayesClassifier } from 'lib/natural'
+
+describe('classifier', function () {
+  describe('addDocument', function () {
+    it('should ignore an empty document', function () {
+      const classifier = new BayesClassifier()
+      classifier.addDocument('', 'philosophy')
+      expect(classifier.docs.length).toBe(0)
+    })
+
+    it('should increment features', function () {
+      const classifier = new BayesClassifier()
+      classifier.addDocument('foo', '')
+      classifier.addDocument('foo', '')
+      classifier.addDocument('bar', '')
+      classifier.addDocument('bar', '')
+      classifier.addDocument('bar', '')
+      classifier.addDocument('baz', '')
+      expect(classifier.docs.length).toBe(6)
+      expect(classifier.features.foo).toBe(2)
+      expect(classifier.features.bar).toBe(3)
+      expect(classifier.features.baz).toBe(1)
+    })
+  })
+
+  describe('Classifier', function () {
+    const pushedEvents: any[] = []
+    function eventRegister (obj: any): void {
+      pushedEvents.push(obj)
+    }
+
+    function assertEventResults (): void {
+      expect(pushedEvents[0].index).toBe(0)
+      expect(pushedEvents[0].total).toBe(6)
+      expect(pushedEvents.length).toBe(6)
+    }
+
+    it('should emit events when documents are added or training is finished', function () {
+      const classifier = new BayesClassifier()
+      classifier.on('trainedWithDocument', eventRegister)
+      classifier.on('doneTraining', assertEventResults)
+
+      classifier.addDocument('i fixed the box', 'computing')
+      classifier.addDocument('i write code', 'computing')
+      classifier.addDocument('nasty script code', 'computing')
+      classifier.addDocument('write a book', 'literature')
+      classifier.addDocument('read a book', 'literature')
+      classifier.addDocument('study the books', 'literature')
+
+      classifier.train()
+      classifier.removeListener('trainedWithDocument', eventRegister)
+      classifier.removeListener('doneTraining', assertEventResults)
+    })
+  })
+
+  describe('Classifier', function () {
+    let classifier: BayesClassifier
+    beforeEach(function () {
+      classifier = new BayesClassifier()
+      classifier.addDocument('i fixed the box', 'computing')
+      classifier.addDocument('i write code', 'computing')
+      classifier.addDocument('write a book', 'literature')
+      classifier.addDocument('study the books', 'literature')
+    })
+
+    it('should do nothing if text is not a string', function () {
+      expect(classifier.docs.length).toBe(4)
+      classifier.removeDocument(['write a book'], 'literature')
+      classifier.removeDocument(['study the books'], 'literature')
+      expect(classifier.docs.length).toBe(4)
+    })
+
+    it('should do nothing if text is not a match', function () {
+      expect(classifier.docs.length).toBe(4)
+      classifier.removeDocument('something else', 'literature')
+      classifier.removeDocument('another thing', 'literature')
+      expect(classifier.docs.length).toBe(4)
+    })
+  })
+})

package/ts_spec/count_inflector_fr_spec.ts

@@ -0,0 +1,47 @@
+/*
+Copyright (c) 2012, Guillaume Marty
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CountInflectorFr } from 'lib/natural'
+const inflector = new CountInflectorFr()
+
+describe('count_inflector', function () {
+  it('should handle 1er cases', function () {
+    expect(inflector.nth(1)).toBe('1er')
+  })
+
+  it('should handle the 2e cases', function () {
+    expect(inflector.nth(0)).toBe('0e')
+    expect(inflector.nth(2)).toBe('2e')
+    expect(inflector.nth(3)).toBe('3e')
+    expect(inflector.nth(5)).toBe('5e')
+    expect(inflector.nth(11)).toBe('11e')
+    expect(inflector.nth(100)).toBe('100e')
+    expect(inflector.nth(999)).toBe('999e')
+  })
+
+  it('should handle roman numerals', function () {
+    expect(inflector.nth('I')).toBe('Ier')
+    expect(inflector.nth('XX')).toBe('XXe')
+  })
+})

package/ts_spec/count_inflector_spec.ts

@@ -0,0 +1,76 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CountInflector } from 'lib/natural'
+const inflector = new CountInflector()
+
+describe('count_inflector', function () {
+  it('should handle 1st cases', function () {
+    expect(inflector.nth(1)).toBe('1st')
+    expect(inflector.nth(101)).toBe('101st')
+    expect(inflector.nth(11)).not.toBe('11st')
+    expect(inflector.nth(111)).not.toBe('111st')
+  })
+
+  it('should handle the 12th cases', function () {
+    expect(inflector.nth(12)).toBe('12th')
+    expect(inflector.nth(112)).toBe('112th')
+    expect(inflector.nth(1112)).toBe('1112th')
+  })
+
+  it('should handle the 11th cases', function () {
+    expect(inflector.nth(11)).toBe('11th')
+    expect(inflector.nth(111)).toBe('111th')
+    expect(inflector.nth(1111)).toBe('1111th')
+  })
+
+  it('should handle the 13th cases', function () {
+    expect(inflector.nth(13)).toBe('13th')
+    expect(inflector.nth(113)).toBe('113th')
+    expect(inflector.nth(1113)).toBe('1113th')
+  })
+
+  it('should handle the th cases', function () {
+    expect(inflector.nth(10)).toBe('10th')
+    expect(inflector.nth(4)).toBe('4th')
+    expect(inflector.nth(400)).toBe('400th')
+    expect(inflector.nth(404)).toBe('404th')
+    expect(inflector.nth(5)).toBe('5th')
+    expect(inflector.nth(5000)).toBe('5000th')
+    expect(inflector.nth(5005)).toBe('5005th')
+    expect(inflector.nth(9)).toBe('9th')
+    expect(inflector.nth(90009)).toBe('90009th')
+    expect(inflector.nth(90000)).toBe('90000th')
+  })
+
+  it('should handle 2nd cases', function () {
+    expect(inflector.nth(2)).toBe('2nd')
+    expect(inflector.nth(12)).not.toBe('12nd')
+  })
+
+  it('should handle 3rd cases', function () {
+    expect(inflector.nth(3)).toBe('3rd')
+    expect(inflector.nth(13)).not.toBe('13rd')
+  })
+})

package/ts_spec/damerau_levenshtein_spec.ts

@@ -0,0 +1,53 @@
+'use strict'
+
+import { DamerauLevenshteinDistance, DamerauLevenshteinDistanceSearch } from 'lib/natural'
+
+describe('DamerauLevenshtein', function () {
+  describe('default', function () {
+    it('should be 0 when given equal strings', function () {
+      expect(DamerauLevenshteinDistance('test', 'test')).toBe(0)
+    })
+
+    it('should calculate 1 for adjacent transposition', function () {
+      expect(DamerauLevenshteinDistance('za', 'az')).toBe(1)
+      expect(DamerauLevenshteinDistance('Tomato', 'oTmato')).toBe(1)
+    })
+
+    it('should handle custom transposition_cost', function () {
+      expect(DamerauLevenshteinDistance('za', 'az', { transposition_cost: 0 })).toBe(0)
+    })
+
+    it('should calculate 2 when there are 2 transpositions', function () {
+      expect(DamerauLevenshteinDistance('tomato', 'otmaot')).toBe(2)
+    })
+
+    it('should calculate 2 for 1 transposition and 1 insertion', function () {
+      expect(DamerauLevenshteinDistance('CA', 'ABC')).toBe(2)
+      expect(DamerauLevenshteinDistance('a cat', 'a abct')).toBe(2)
+    })
+  })
+
+  describe('options.restricted = true', function () {
+    const restricted = { restricted: true }
+    it('should calculate 0 for equal strings', function () {
+      expect(DamerauLevenshteinDistance('identity', 'identity', restricted)).toBe(0)
+    })
+    it('should calculate 1 for an adjacent transposition', function () {
+      expect(DamerauLevenshteinDistance('za', 'az', restricted)).toBe(1)
+    })
+    it('should not count transposition more than 1 char away', function () {
+      expect(DamerauLevenshteinDistance('CA', 'ABC', restricted)).toBe(3)
+    })
+  })
+
+  it('should combine search with Damerau', function () {
+    const source = 'The RainCoat BookStore'
+    const target = 'All the best books are here at the Rain Coats Book Store'
+    const result = DamerauLevenshteinDistanceSearch(source, target)
+    expect(result).toEqual({
+      substring: 'the Rain Coats Book Store',
+      distance: 4,
+      offset: 31
+    })
+  })
+})

package/ts_spec/dice_coefficient_spec.ts

@@ -0,0 +1,37 @@
+'use strict'
+
+import { DiceCoefficient as dice } from 'lib/natural'
+import text1 from '../spec/test_data/Wikipedia_EN_FrenchRevolution.json'
+import text2 from '../spec/test_data/Wikipedia_EN_InfluenceOfTheFrenchRevolution.json'
+
+describe('dice', function () {
+  it('should handle exact matches', function () {
+    expect(dice('john', 'john')).toBe(1)
+  })
+
+  it('should match single character words', function () {
+    expect(dice('a', 'a')).toBe(1)
+    expect(dice('a', 'b')).toBe(0)
+  })
+
+  it('should handle total mis-matches', function () {
+    expect(dice('john', 'matt')).toBe(0)
+  })
+
+  // Example from http://en.wikipedia.org/wiki/Dice's_coefficient
+  it('should handle a typical case', function () {
+    expect(dice('night', 'nacht')).toBe(0.25)
+  })
+
+  it('should sanitize case', function () {
+    expect(dice('night', 'NIGHT')).toBe(1)
+  })
+
+  it('should sanitize spacing', function () {
+    expect(dice('the   space', 'the space')).toBe(1)
+  })
+
+  it('should compare complete texts', function () {
+    expect(dice(text1.text, text2.text)).toBe(0.7939374395356337)
+  })
+})