natural 6.12.0 → 7.0.1

package/ts_spec/carry_stemmer_fr_spec.ts

@@ -0,0 +1,97 @@
+/*
+Copyright (c) 2020, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CarryStemmerFr as stemmer } from 'lib/natural'
+
+// const snowBallDict = require('spec/test_data/snowball_fr.json')
+
+const tests = [
+  {
+    input: 'action',
+    output: 'ac'
+  },
+  {
+    input: 'acteur',
+    output: 'ac'
+  },
+  {
+    input: 'actrices',
+    output: 'ac'
+  },
+  {
+    input: 'Dleyton',
+    output: 'Dleyton'
+  },
+  {
+    input: 'volera',
+    output: 'vol'
+  }
+]
+
+const testSentences = [
+  {
+    input: 'Le petit cheval de manège',
+    output: ['pet', 'cheval', 'manèg']
+  }
+]
+
+describe('Carry stemmer', function () {
+  tests.forEach(test => {
+    it('stems ' + test.input + ' correctly', function () {
+      expect(stemmer.stem(test.input)).toBe(test.output)
+    })
+  })
+
+  testSentences.forEach(test => {
+    it('tokenizes and stems ' + test.input + ' correctly', function () {
+      expect(stemmer.tokenizeAndStem(test.input)).toEqual(test.output)
+    })
+  })
+
+  /*
+  it('should perform stemming on a lot of words', function() {
+    var ok = [];
+    var ko = [];
+
+    Object.keys(snowBallDict).forEach(word => {
+
+      var stemmed = stemmer.stem(word);
+      var expectedStem = snowBallDict[word];
+
+      if (stemmed === expectedStem) {
+        ok.push(word);
+      }
+      else {
+        ko.push({
+          word: word,
+          expected: expectedStem,
+          actual: stemmed
+        });
+      }
+    });
+
+    expect(ko.length).toBe(0);
+  });
+*/
+})

package/ts_spec/classifier_spec.ts

@@ -0,0 +1,104 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { BayesClassifier } from 'lib/natural'
+
+describe('classifier', function () {
+  describe('addDocument', function () {
+    it('should ignore an empty document', function () {
+      const classifier = new BayesClassifier()
+      classifier.addDocument('', 'philosophy')
+      expect(classifier.docs.length).toBe(0)
+    })
+
+    it('should increment features', function () {
+      const classifier = new BayesClassifier()
+      classifier.addDocument('foo', '')
+      classifier.addDocument('foo', '')
+      classifier.addDocument('bar', '')
+      classifier.addDocument('bar', '')
+      classifier.addDocument('bar', '')
+      classifier.addDocument('baz', '')
+      expect(classifier.docs.length).toBe(6)
+      expect(classifier.features.foo).toBe(2)
+      expect(classifier.features.bar).toBe(3)
+      expect(classifier.features.baz).toBe(1)
+    })
+  })
+
+  describe('Classifier', function () {
+    const pushedEvents: any[] = []
+    function eventRegister (obj: any): void {
+      pushedEvents.push(obj)
+    }
+
+    function assertEventResults (): void {
+      expect(pushedEvents[0].index).toBe(0)
+      expect(pushedEvents[0].total).toBe(6)
+      expect(pushedEvents.length).toBe(6)
+    }
+
+    it('should emit events when documents are added or training is finished', function () {
+      const classifier = new BayesClassifier()
+      classifier.on('trainedWithDocument', eventRegister)
+      classifier.on('doneTraining', assertEventResults)
+
+      classifier.addDocument('i fixed the box', 'computing')
+      classifier.addDocument('i write code', 'computing')
+      classifier.addDocument('nasty script code', 'computing')
+      classifier.addDocument('write a book', 'literature')
+      classifier.addDocument('read a book', 'literature')
+      classifier.addDocument('study the books', 'literature')
+
+      classifier.train()
+      classifier.removeListener('trainedWithDocument', eventRegister)
+      classifier.removeListener('doneTraining', assertEventResults)
+    })
+  })
+
+  describe('Classifier', function () {
+    let classifier: BayesClassifier
+    beforeEach(function () {
+      classifier = new BayesClassifier()
+      classifier.addDocument('i fixed the box', 'computing')
+      classifier.addDocument('i write code', 'computing')
+      classifier.addDocument('write a book', 'literature')
+      classifier.addDocument('study the books', 'literature')
+    })
+
+    it('should do nothing if text is not a string', function () {
+      expect(classifier.docs.length).toBe(4)
+      classifier.removeDocument(['write a book'], 'literature')
+      classifier.removeDocument(['study the books'], 'literature')
+      expect(classifier.docs.length).toBe(4)
+    })
+
+    it('should do nothing if text is not a match', function () {
+      expect(classifier.docs.length).toBe(4)
+      classifier.removeDocument('something else', 'literature')
+      classifier.removeDocument('another thing', 'literature')
+      expect(classifier.docs.length).toBe(4)
+    })
+  })
+})

package/ts_spec/count_inflector_fr_spec.ts

@@ -0,0 +1,47 @@
+/*
+Copyright (c) 2012, Guillaume Marty
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CountInflectorFr } from 'lib/natural'
+const inflector = new CountInflectorFr()
+
+describe('count_inflector', function () {
+  it('should handle 1er cases', function () {
+    expect(inflector.nth(1)).toBe('1er')
+  })
+
+  it('should handle the 2e cases', function () {
+    expect(inflector.nth(0)).toBe('0e')
+    expect(inflector.nth(2)).toBe('2e')
+    expect(inflector.nth(3)).toBe('3e')
+    expect(inflector.nth(5)).toBe('5e')
+    expect(inflector.nth(11)).toBe('11e')
+    expect(inflector.nth(100)).toBe('100e')
+    expect(inflector.nth(999)).toBe('999e')
+  })
+
+  it('should handle roman numerals', function () {
+    expect(inflector.nth('I')).toBe('Ier')
+    expect(inflector.nth('XX')).toBe('XXe')
+  })
+})

package/ts_spec/count_inflector_spec.ts

@@ -0,0 +1,76 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CountInflector } from 'lib/natural'
+const inflector = new CountInflector()
+
+describe('count_inflector', function () {
+  it('should handle 1st cases', function () {
+    expect(inflector.nth(1)).toBe('1st')
+    expect(inflector.nth(101)).toBe('101st')
+    expect(inflector.nth(11)).not.toBe('11st')
+    expect(inflector.nth(111)).not.toBe('111st')
+  })
+
+  it('should handle the 12th cases', function () {
+    expect(inflector.nth(12)).toBe('12th')
+    expect(inflector.nth(112)).toBe('112th')
+    expect(inflector.nth(1112)).toBe('1112th')
+  })
+
+  it('should handle the 11th cases', function () {
+    expect(inflector.nth(11)).toBe('11th')
+    expect(inflector.nth(111)).toBe('111th')
+    expect(inflector.nth(1111)).toBe('1111th')
+  })
+
+  it('should handle the 13th cases', function () {
+    expect(inflector.nth(13)).toBe('13th')
+    expect(inflector.nth(113)).toBe('113th')
+    expect(inflector.nth(1113)).toBe('1113th')
+  })
+
+  it('should handle the th cases', function () {
+    expect(inflector.nth(10)).toBe('10th')
+    expect(inflector.nth(4)).toBe('4th')
+    expect(inflector.nth(400)).toBe('400th')
+    expect(inflector.nth(404)).toBe('404th')
+    expect(inflector.nth(5)).toBe('5th')
+    expect(inflector.nth(5000)).toBe('5000th')
+    expect(inflector.nth(5005)).toBe('5005th')
+    expect(inflector.nth(9)).toBe('9th')
+    expect(inflector.nth(90009)).toBe('90009th')
+    expect(inflector.nth(90000)).toBe('90000th')
+  })
+
+  it('should handle 2nd cases', function () {
+    expect(inflector.nth(2)).toBe('2nd')
+    expect(inflector.nth(12)).not.toBe('12nd')
+  })
+
+  it('should handle 3rd cases', function () {
+    expect(inflector.nth(3)).toBe('3rd')
+    expect(inflector.nth(13)).not.toBe('13rd')
+  })
+})

package/ts_spec/damerau_levenshtein_spec.ts

@@ -0,0 +1,53 @@
+'use strict'
+
+import { DamerauLevenshteinDistance, DamerauLevenshteinDistanceSearch } from 'lib/natural'
+
+describe('DamerauLevenshtein', function () {
+  describe('default', function () {
+    it('should be 0 when given equal strings', function () {
+      expect(DamerauLevenshteinDistance('test', 'test')).toBe(0)
+    })
+
+    it('should calculate 1 for adjacent transposition', function () {
+      expect(DamerauLevenshteinDistance('za', 'az')).toBe(1)
+      expect(DamerauLevenshteinDistance('Tomato', 'oTmato')).toBe(1)
+    })
+
+    it('should handle custom transposition_cost', function () {
+      expect(DamerauLevenshteinDistance('za', 'az', { transposition_cost: 0 })).toBe(0)
+    })
+
+    it('should calculate 2 when there are 2 transpositions', function () {
+      expect(DamerauLevenshteinDistance('tomato', 'otmaot')).toBe(2)
+    })
+
+    it('should calculate 2 for 1 transposition and 1 insertion', function () {
+      expect(DamerauLevenshteinDistance('CA', 'ABC')).toBe(2)
+      expect(DamerauLevenshteinDistance('a cat', 'a abct')).toBe(2)
+    })
+  })
+
+  describe('options.restricted = true', function () {
+    const restricted = { restricted: true }
+    it('should calculate 0 for equal strings', function () {
+      expect(DamerauLevenshteinDistance('identity', 'identity', restricted)).toBe(0)
+    })
+    it('should calculate 1 for an adjacent transposition', function () {
+      expect(DamerauLevenshteinDistance('za', 'az', restricted)).toBe(1)
+    })
+    it('should not count transposition more than 1 char away', function () {
+      expect(DamerauLevenshteinDistance('CA', 'ABC', restricted)).toBe(3)
+    })
+  })
+
+  it('should combine search with Damerau', function () {
+    const source = 'The RainCoat BookStore'
+    const target = 'All the best books are here at the Rain Coats Book Store'
+    const result = DamerauLevenshteinDistanceSearch(source, target)
+    expect(result).toEqual({
+      substring: 'the Rain Coats Book Store',
+      distance: 4,
+      offset: 31
+    })
+  })
+})

package/ts_spec/dice_coefficient_spec.ts

@@ -0,0 +1,37 @@
+'use strict'
+
+import { DiceCoefficient as dice } from 'lib/natural'
+import text1 from '../spec/test_data/Wikipedia_EN_FrenchRevolution.json'
+import text2 from '../spec/test_data/Wikipedia_EN_InfluenceOfTheFrenchRevolution.json'
+
+describe('dice', function () {
+  it('should handle exact matches', function () {
+    expect(dice('john', 'john')).toBe(1)
+  })
+
+  it('should match single character words', function () {
+    expect(dice('a', 'a')).toBe(1)
+    expect(dice('a', 'b')).toBe(0)
+  })
+
+  it('should handle total mis-matches', function () {
+    expect(dice('john', 'matt')).toBe(0)
+  })
+
+  // Example from http://en.wikipedia.org/wiki/Dice's_coefficient
+  it('should handle a typical case', function () {
+    expect(dice('night', 'nacht')).toBe(0.25)
+  })
+
+  it('should sanitize case', function () {
+    expect(dice('night', 'NIGHT')).toBe(1)
+  })
+
+  it('should sanitize spacing', function () {
+    expect(dice('the   space', 'the space')).toBe(1)
+  })
+
+  it('should compare complete texts', function () {
+    expect(dice(text1.text, text2.text)).toBe(0.7939374395356337)
+  })
+})