natural 6.11.1 → 7.0.0

package/ts_spec/aggressive_tokenizer_es_spec.ts

@@ -0,0 +1,39 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWé.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerEs } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerEs()
+
+describe('aggressive_tokenizer_es', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('hola yo me llamo eduardo y esudié ingeniería')).toEqual(['hola', 'yo', 'me', 'llamo', 'eduardo', 'y', 'esudié', 'ingeniería'])
+  })
+
+  /*
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect('hola yo me llamo eduardo y esudié ingeniería'.tokenize()).toEqual(['hola', 'yo', 'me', 'llamo', 'eduardo', 'y', 'esudié', 'ingeniería']);
+  });
+  */
+})

package/ts_spec/aggressive_tokenizer_fr_spec.ts

@@ -0,0 +1,111 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE
+*/
+
+'use strict'
+
+import { AggressiveTokenizerFr } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerFr()
+
+const text = "Affectueusement surnommé « Gabo » dans toute l'Amérique latine, le Colombien Gabriel Garcia Marquez, prix Nobel de littérature 1982, l'un des plus grands écrivains du XXe siècle, est mort À son domicile de Mexico jeudi 17 avril. Il était âgé de 87 ans. Son Œuvre a été traduite dans toutes les langues ou presque, et vendue à quelque 50 millions d'exemplaires."
+
+const tokenized = ['Affectueusement',
+  'surnommé',
+  'Gabo',
+  'dans',
+  'toute',
+  'l',
+  'Amérique',
+  'latine',
+  'le',
+  'Colombien',
+  'Gabriel',
+  'Garcia',
+  'Marquez',
+  'prix',
+  'Nobel',
+  'de',
+  'littérature',
+  '1982',
+  'l',
+  'un',
+  'des',
+  'plus',
+  'grands',
+  'écrivains',
+  'du',
+  'XXe',
+  'siècle',
+  'est',
+  'mort',
+  'À',
+  'son',
+  'domicile',
+  'de',
+  'Mexico',
+  'jeudi',
+  '17',
+  'avril',
+  'Il',
+  'était',
+  'âgé',
+  'de',
+  '87',
+  'ans',
+  'Son',
+  'Œuvre',
+  'a',
+  'été',
+  'traduite',
+  'dans',
+  'toutes',
+  'les',
+  'langues',
+  'ou',
+  'presque',
+  'et',
+  'vendue',
+  'à',
+  'quelque',
+  '50',
+  'millions',
+  'd',
+  'exemplaires']
+
+describe('aggressive_tokenizer_fr', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize(text)).toEqual(tokenized)
+  })
+
+  it('should handle hyphens in words correctly', function () {
+    const sentence = 'Des sous-pages dans le sous-bois de la ville de Paris'
+    const res = tokenizer.tokenize(sentence)
+    const expectedRes = ['Des', 'sous-pages', 'dans', 'le', 'sous-bois', 'de', 'la', 'ville', 'de', 'Paris']
+    expect(res).toEqual(expectedRes)
+  })
+
+  /*
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect(text.tokenize()).toEqual(tokenized);
+  });
+  */
+})

package/ts_spec/aggressive_tokenizer_hi_spec.ts

@@ -0,0 +1,70 @@
+/*
+file aggressive_tokenizer_hi_spec.js , located at spec\aggressive_tokenizer_hi_spec.js is licensed as follows:
+- (The MIT License)
+- Copyright (c) 2023 Mukesh Singh Bisht
+
+Permission is hereby granted, free of charge, to any person or entity obtaining a copy
+of  file aggressive_tokenizer_hi_spec.js and its content(the "Software"), to deal in the
+Software without restriction, including without limitation the rights to use, copy,
+modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following
+conditions:
+
+1. The above copyright notice and this permission notice shall be included in all copies
+   or substantial portions of the Software.
+
+2. Proper credit must be given to the original author Mukesh Singh Bisht, along with the
+   date of authorship specified as July 23, 2023, in any usage, distribution, or
+   modification of the Software.
+
+THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR
+OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerHi } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerHi()
+
+describe('aggressive_tokenizer_hi', function () {
+  it('should tokenize strings', function () {
+    const string = 'स्वतंत्रता दिवस की हार्दिक शुभकामनाएं'
+    const expectedArray = ['स्वतंत्रता', 'दिवस', 'की', 'हार्दिक', 'शुभकामनाएं']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+  it('should tokenize strings including english words', function () {
+    const string = 'स्वतंत्रता दिवस की हार्दिक शुभकामनाएं congrats mukesh'
+    const expectedArray = ['स्वतंत्रता', 'दिवस', 'की', 'हार्दिक', 'शुभकामनाएं', 'congrats', 'mukesh']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+
+  it('should swallow viram(stop) symbols', function () {
+    const string = 'स्वतंत्रता दिवस की हार्दिक शुभकामनाएं । congrats mukesh'
+    const expectedArray = ['स्वतंत्रता', 'दिवस', 'की', 'हार्दिक', 'शुभकामनाएं', 'congrats', 'mukesh']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+  it('should swallow dirgh viram(stop) symbols', function () {
+    const string = 'राजा बाजीराव ॥ '
+    const expectedArray = ['राजा', 'बाजीराव']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+  it('should swallow trailing off symbols', function () {
+    const string = 'राजा बाजीराव ...'
+    const expectedArray = ['राजा', 'बाजीराव']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+  it('should swallow question mark symbols', function () {
+    const string = 'राजा बाजीराव ?'
+    const expectedArray = ['राजा', 'बाजीराव']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+  it('should swallow comma mark symbols', function () {
+    const string = 'राजा, बाजीराव'
+    const expectedArray = ['राजा', 'बाजीराव']
+    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
+  })
+})

package/ts_spec/aggressive_tokenizer_nl_spec.ts

@@ -0,0 +1,39 @@
+/*
+Copyright (c) 2011, Chris Umbel, Martijn de Boer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWé.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerNl } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerNl()
+
+describe('aggressive_tokenizer_nl', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('\'s Morgens is het nog erg koud, vertelde de weerman over een van de radio\'s')).toEqual(['\'s', 'Morgens', 'is', 'het', 'nog', 'erg', 'koud', 'vertelde', 'de', 'weerman', 'over', 'een', 'van', 'de', 'radio\'s'])
+  })
+
+  it('should handle hyphens in words correctly', function () {
+    const sentence = 'clearing-systeem front-office-automatisering christelijk-historisch mond-op-mond, kant-en-klaar, kruidje-roer-me-niet, doe-het-zelver'
+    const res = tokenizer.tokenize(sentence)
+    const expectedRes = ['clearing-systeem', 'front-office-automatisering', 'christelijk-historisch', 'mond-op-mond', 'kant-en-klaar', 'kruidje-roer-me-niet', 'doe-het-zelver']
+    expect(res).toEqual(expectedRes)
+  })
+})

package/ts_spec/aggressive_tokenizer_pt_spec.ts

@@ -0,0 +1,48 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWé.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerPt } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerPt()
+
+describe('aggressive_tokenizer_pt', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('isso é coração')).toEqual(['isso', 'é', 'coração'])
+  })
+
+  it('should swallow punctuation', function () {
+    expect(tokenizer.tokenize('isso é coração, no')).toEqual(['isso', 'é', 'coração', 'no'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('isso é coração, no?')).toEqual(['isso', 'é', 'coração', 'no'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.isso é coração, no')).toEqual(['isso', 'é', 'coração', 'no'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('eu vou... pause')).toEqual(['eu', 'vou', 'pause'])
+  })
+})

package/ts_spec/aggressive_tokenizer_spec.ts

@@ -0,0 +1,57 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizer } from 'lib/natural'
+const tokenizer = new AggressiveTokenizer()
+
+describe('aggressive_tokenizer', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('these are things')).toEqual(['these', 'are', 'things'])
+  })
+
+  it('should swallow punctuation', function () {
+    expect(tokenizer.tokenize('these are things, no')).toEqual(['these', 'are', 'things', 'no'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('these are things, no?')).toEqual(['these', 'are', 'things', 'no'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.these are things, no')).toEqual(['these', 'are', 'things', 'no'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('i shal... pause')).toEqual(['i', 'shal', 'pause'])
+  })
+
+  it('should remove underscores', function () {
+    expect(tokenizer.tokenize('_ hi_this_is_a_test_case_ for__removing___underscores_')).toEqual(['hi', 'this', 'is', 'a', 'test', 'case', 'for', 'removing', 'underscores'])
+  })
+
+  it('should handle the use of hyphen inside words (issue #656)', function () {
+    expect(tokenizer.tokenize('links text-based opposed image-based links/CTA\'s')).toEqual(
+      ['links', 'text-based', 'opposed', 'image-based', 'links/CTA\'s'])
+  })
+})

package/ts_spec/aggressive_tokenizer_sv_spec.ts

@@ -0,0 +1,52 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerSv } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerSv()
+
+describe('aggressive_tokenizer_sv', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('Ett tu tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow punctuation', function () {
+    expect(tokenizer.tokenize('Ett, tu, tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('Ett, tu, tre?')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.Ett, tu, tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('Ett, tu... tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should not split on hyphen or Swedish letters', function () {
+    expect(tokenizer.tokenize('It-bolaget ägs av en 30-åring')).toEqual(['It-bolaget', 'ägs', 'av', 'en', '30-åring'])
+  })
+})

package/ts_spec/aggressive_tokenizer_vi_spec.ts

@@ -0,0 +1,48 @@
+/*
+Copyright (c) 2018, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerVi } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerVi()
+
+describe('aggressive_tokenizer_vi', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('Tôi bị lạc')).toEqual(['Tôi', 'bị', 'lạc'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('Làm ơn đợi một lát!')).toEqual(['Làm', 'ơn', 'đợi', 'một', 'lát'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('Tôi đang tìm John.')).toEqual(['Tôi', 'đang', 'tìm', 'John'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.Đi thẳng, sau đó rẽ trái / phải')).toEqual(['Đi', 'thẳng', 'sau', 'đó', 'rẽ', 'trái', 'phải'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('.Xin giữ máy!')).toEqual(['Xin', 'giữ', 'máy'])
+  })
+})

package/ts_spec/bayes_classifier_spec.ts

@@ -0,0 +1,165 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { BayesClassifier, PorterStemmer } from 'lib/natural'
+
+function setupClassifier (): BayesClassifier {
+  const classifier = new BayesClassifier()
+  classifier.addDocument(['fix', 'box'], 'computing')
+  classifier.addDocument(['write', 'code'], 'computing')
+  classifier.addDocument(['script', 'code'], 'computing')
+  classifier.addDocument(['write', 'book'], 'literature')
+  classifier.addDocument(['read', 'book'], 'literature')
+  classifier.addDocument(['study', 'book'], 'literature')
+  return classifier
+}
+
+describe('bayes classifier', function () {
+  describe('classifier', function () {
+    it('should classify with arrays', function () {
+      const classifier = setupClassifier()
+      classifier.train()
+      expect(classifier.classify(['bug', 'code'])).toBe('computing')
+      expect(classifier.classify(['read', 'thing'])).toBe('literature')
+    })
+
+    it('should classify with parallel training', function () {
+      const classifier = setupClassifier()
+      // Check for parallel method
+      if (classifier.Threads !== null) {
+        classifier.trainParallel(2, function (err) {
+          if (err !== null) {
+            console.log(err)
+            return
+          }
+          expect(classifier.classify(['bug', 'code'])).toBe('computing')
+          expect(classifier.classify(['read', 'thing'])).toBe('literature')
+        })
+      }
+    })
+
+    it('should classify with parallel batched training', function () {
+      const classifier = setupClassifier()
+      // Check for parallel method
+      if (classifier.Threads !== null) {
+        classifier.on('doneTraining', function () {
+          expect(classifier.classify(['bug', 'code'])).toBe('computing')
+          expect(classifier.classify(['read', 'thing'])).toBe('literature')
+          // asyncSpecDone();
+        })
+        classifier.trainParallelBatches({ numThreads: 2, batchSize: 2 })
+      }
+    })
+
+    it('should provide all classification scores', function () {
+      const classifier = setupClassifier()
+      classifier.train()
+
+      expect(classifier.getClassifications('i write code')[0].label).toBe('computing')
+      expect(classifier.getClassifications('i write code')[1].label).toBe('literature')
+    })
+
+    function setupClassifierWithSentences (): BayesClassifier {
+      const classifier = new BayesClassifier()
+      classifier.addDocument('i fixed the box', 'computing')
+      classifier.addDocument('i write code', 'computing')
+      classifier.addDocument('nasty script code', 'computing')
+      classifier.addDocument('write a book', 'literature')
+      classifier.addDocument('read a book', 'literature')
+      classifier.addDocument('study the books', 'literature')
+      return classifier
+    }
+
+    it('should classify with strings', function () {
+      const classifier = setupClassifierWithSentences()
+      classifier.train()
+      expect(classifier.classify('a bug in the code')).toBe('computing')
+      expect(classifier.classify('read all the books')).toBe('literature')
+    })
+
+    it('should classify and re-classify after document-removal', function () {
+      const classifier = new BayesClassifier()
+      const classifications = new Map<string, number>()
+
+      // Add some good/bad docs and train
+      classifier.addDocument('foo bar baz', 'good')
+      classifier.addDocument('qux zooby', 'bad')
+      classifier.addDocument('asdf qwer', 'bad')
+      classifier.train()
+
+      expect(classifier.classify('foo')).toBe('good')
+      expect(classifier.classify('qux')).toBe('bad')
+
+      // Remove one of the bad docs, retrain
+      classifier.removeDocument('qux zooby', 'bad')
+      classifier.retrain()
+
+      // Simple `classify` will still return a single result, even if
+      // ratio for each side is equal -- have to compare actual values in
+      // the classifications, should be equal since qux is unclassified
+      const arr = classifier.getClassifications('qux')
+      let item: { label: string, value: number }
+      for (let i = 0, ii = arr.length; i < ii; i++) {
+        item = arr[i]
+        classifications.set(item.label, item.value)
+      }
+      expect(classifications.get('good')).toEqual(classifications.get('bad'))
+
+      // Re-classify as good, retrain
+      classifier.addDocument('qux zooby', 'good')
+      classifier.retrain()
+
+      // Should now be good, original docs should be unaffected
+      expect(classifier.classify('foo')).toBe('good')
+      expect(classifier.classify('qux')).toBe('good')
+    })
+
+    it('should serialize and deserialize a working classifier', function () {
+      const classifier = setupClassifierWithSentences()
+      const objAsString = JSON.stringify(classifier)
+      const obj: Record<string, unknown> = JSON.parse(objAsString)
+      const newClassifier = BayesClassifier.restore(obj)
+
+      newClassifier.addDocument('kick a ball', 'sports')
+      newClassifier.addDocument('hit some balls', 'sports')
+      newClassifier.addDocument('kick and punch', 'sports')
+
+      newClassifier.train()
+
+      expect(newClassifier.classify('a bug in the code')).toBe('computing')
+      expect(newClassifier.classify('read all the books')).toBe('literature')
+      expect(newClassifier.classify('kick butt')).toBe('sports')
+    })
+
+    it('should accept an optional smoothing parameter for the Bayesian estimates', function () {
+      const defaultClassifier = new BayesClassifier()
+      const newClassifier1 = new BayesClassifier(PorterStemmer)
+      const newClassifier2 = new BayesClassifier(PorterStemmer, 0.1)
+
+      expect(defaultClassifier.classifier.smoothing).toBe(1.0)
+      expect(newClassifier1.classifier.smoothing).toBe(1.0)
+      expect(newClassifier2.classifier.smoothing).toBe(0.1)
+    })
+  })
+})