natural 7.0.3 → 7.0.5

package/ts_spec/aggressive_tokenizer_fr_spec.ts

@@ -1,111 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this softwé and associated documentation files (the "Softwé"), to deal
-in the Softwé without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Softwé, and to permit persons to whom the Softwé is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Softwé.
-
-THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE
-*/
-
-'use strict'
-
-import { AggressiveTokenizerFr } from 'lib/natural'
-const tokenizer = new AggressiveTokenizerFr()
-
-const text = "Affectueusement surnommé « Gabo » dans toute l'Amérique latine, le Colombien Gabriel Garcia Marquez, prix Nobel de littérature 1982, l'un des plus grands écrivains du XXe siècle, est mort À son domicile de Mexico jeudi 17 avril. Il était âgé de 87 ans. Son Œuvre a été traduite dans toutes les langues ou presque, et vendue à quelque 50 millions d'exemplaires."
-
-const tokenized = ['Affectueusement',
-  'surnommé',
-  'Gabo',
-  'dans',
-  'toute',
-  'l',
-  'Amérique',
-  'latine',
-  'le',
-  'Colombien',
-  'Gabriel',
-  'Garcia',
-  'Marquez',
-  'prix',
-  'Nobel',
-  'de',
-  'littérature',
-  '1982',
-  'l',
-  'un',
-  'des',
-  'plus',
-  'grands',
-  'écrivains',
-  'du',
-  'XXe',
-  'siècle',
-  'est',
-  'mort',
-  'À',
-  'son',
-  'domicile',
-  'de',
-  'Mexico',
-  'jeudi',
-  '17',
-  'avril',
-  'Il',
-  'était',
-  'âgé',
-  'de',
-  '87',
-  'ans',
-  'Son',
-  'Œuvre',
-  'a',
-  'été',
-  'traduite',
-  'dans',
-  'toutes',
-  'les',
-  'langues',
-  'ou',
-  'presque',
-  'et',
-  'vendue',
-  'à',
-  'quelque',
-  '50',
-  'millions',
-  'd',
-  'exemplaires']
-
-describe('aggressive_tokenizer_fr', function () {
-  it('should tokenize strings', function () {
-    expect(tokenizer.tokenize(text)).toEqual(tokenized)
-  })
-
-  it('should handle hyphens in words correctly', function () {
-    const sentence = 'Des sous-pages dans le sous-bois de la ville de Paris'
-    const res = tokenizer.tokenize(sentence)
-    const expectedRes = ['Des', 'sous-pages', 'dans', 'le', 'sous-bois', 'de', 'la', 'ville', 'de', 'Paris']
-    expect(res).toEqual(expectedRes)
-  })
-
-  /*
-  it('should tokenize strings via attached string method', function() {
-    tokenizer.attach();
-    expect(text.tokenize()).toEqual(tokenized);
-  });
-  */
-})

package/ts_spec/aggressive_tokenizer_hi_spec.ts

@@ -1,70 +0,0 @@
-/*
-file aggressive_tokenizer_hi_spec.js , located at spec\aggressive_tokenizer_hi_spec.js is licensed as follows:
-- (The MIT License)
-- Copyright (c) 2023 Mukesh Singh Bisht
-
-Permission is hereby granted, free of charge, to any person or entity obtaining a copy
-of  file aggressive_tokenizer_hi_spec.js and its content(the "Software"), to deal in the
-Software without restriction, including without limitation the rights to use, copy,
-modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
-and to permit persons to whom the Software is furnished to do so, subject to the following
-conditions:
-
-1. The above copyright notice and this permission notice shall be included in all copies
-   or substantial portions of the Software.
-
-2. Proper credit must be given to the original author Mukesh Singh Bisht, along with the
-   date of authorship specified as July 23, 2023, in any usage, distribution, or
-   modification of the Software.
-
-THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
-INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
-FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR
-OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-'use strict'
-
-import { AggressiveTokenizerHi } from 'lib/natural'
-const tokenizer = new AggressiveTokenizerHi()
-
-describe('aggressive_tokenizer_hi', function () {
-  it('should tokenize strings', function () {
-    const string = 'स्वतंत्रता दिवस की हार्दिक शुभकामनाएं'
-    const expectedArray = ['स्वतंत्रता', 'दिवस', 'की', 'हार्दिक', 'शुभकामनाएं']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-  it('should tokenize strings including english words', function () {
-    const string = 'स्वतंत्रता दिवस की हार्दिक शुभकामनाएं congrats mukesh'
-    const expectedArray = ['स्वतंत्रता', 'दिवस', 'की', 'हार्दिक', 'शुभकामनाएं', 'congrats', 'mukesh']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-
-  it('should swallow viram(stop) symbols', function () {
-    const string = 'स्वतंत्रता दिवस की हार्दिक शुभकामनाएं । congrats mukesh'
-    const expectedArray = ['स्वतंत्रता', 'दिवस', 'की', 'हार्दिक', 'शुभकामनाएं', 'congrats', 'mukesh']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-  it('should swallow dirgh viram(stop) symbols', function () {
-    const string = 'राजा बाजीराव ॥ '
-    const expectedArray = ['राजा', 'बाजीराव']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-  it('should swallow trailing off symbols', function () {
-    const string = 'राजा बाजीराव ...'
-    const expectedArray = ['राजा', 'बाजीराव']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-  it('should swallow question mark symbols', function () {
-    const string = 'राजा बाजीराव ?'
-    const expectedArray = ['राजा', 'बाजीराव']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-  it('should swallow comma mark symbols', function () {
-    const string = 'राजा, बाजीराव'
-    const expectedArray = ['राजा', 'बाजीराव']
-    expect(tokenizer.tokenize(string)).toEqual(expectedArray)
-  })
-})

package/ts_spec/aggressive_tokenizer_nl_spec.ts

@@ -1,39 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel, Martijn de Boer
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this softwé and associated documentation files (the "Softwé"), to deal
-in the Softwé without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Softwé, and to permit persons to whom the Softwé is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Softwé.
-
-THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
-THE SOFTWé.
-*/
-
-'use strict'
-
-import { AggressiveTokenizerNl } from 'lib/natural'
-const tokenizer = new AggressiveTokenizerNl()
-
-describe('aggressive_tokenizer_nl', function () {
-  it('should tokenize strings', function () {
-    expect(tokenizer.tokenize('\'s Morgens is het nog erg koud, vertelde de weerman over een van de radio\'s')).toEqual(['\'s', 'Morgens', 'is', 'het', 'nog', 'erg', 'koud', 'vertelde', 'de', 'weerman', 'over', 'een', 'van', 'de', 'radio\'s'])
-  })
-
-  it('should handle hyphens in words correctly', function () {
-    const sentence = 'clearing-systeem front-office-automatisering christelijk-historisch mond-op-mond, kant-en-klaar, kruidje-roer-me-niet, doe-het-zelver'
-    const res = tokenizer.tokenize(sentence)
-    const expectedRes = ['clearing-systeem', 'front-office-automatisering', 'christelijk-historisch', 'mond-op-mond', 'kant-en-klaar', 'kruidje-roer-me-niet', 'doe-het-zelver']
-    expect(res).toEqual(expectedRes)
-  })
-})

package/ts_spec/aggressive_tokenizer_pt_spec.ts

@@ -1,48 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this softwé and associated documentation files (the "Softwé"), to deal
-in the Softwé without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Softwé, and to permit persons to whom the Softwé is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Softwé.
-
-THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
-THE SOFTWé.
-*/
-
-'use strict'
-
-import { AggressiveTokenizerPt } from 'lib/natural'
-const tokenizer = new AggressiveTokenizerPt()
-
-describe('aggressive_tokenizer_pt', function () {
-  it('should tokenize strings', function () {
-    expect(tokenizer.tokenize('isso é coração')).toEqual(['isso', 'é', 'coração'])
-  })
-
-  it('should swallow punctuation', function () {
-    expect(tokenizer.tokenize('isso é coração, no')).toEqual(['isso', 'é', 'coração', 'no'])
-  })
-
-  it('should swallow final punctuation', function () {
-    expect(tokenizer.tokenize('isso é coração, no?')).toEqual(['isso', 'é', 'coração', 'no'])
-  })
-
-  it('should swallow initial punctuation', function () {
-    expect(tokenizer.tokenize('.isso é coração, no')).toEqual(['isso', 'é', 'coração', 'no'])
-  })
-
-  it('should swallow duplicate punctuation', function () {
-    expect(tokenizer.tokenize('eu vou... pause')).toEqual(['eu', 'vou', 'pause'])
-  })
-})

package/ts_spec/aggressive_tokenizer_spec.ts

@@ -1,57 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-'use strict'
-
-import { AggressiveTokenizer } from 'lib/natural'
-const tokenizer = new AggressiveTokenizer()
-
-describe('aggressive_tokenizer', function () {
-  it('should tokenize strings', function () {
-    expect(tokenizer.tokenize('these are things')).toEqual(['these', 'are', 'things'])
-  })
-
-  it('should swallow punctuation', function () {
-    expect(tokenizer.tokenize('these are things, no')).toEqual(['these', 'are', 'things', 'no'])
-  })
-
-  it('should swallow final punctuation', function () {
-    expect(tokenizer.tokenize('these are things, no?')).toEqual(['these', 'are', 'things', 'no'])
-  })
-
-  it('should swallow initial punctuation', function () {
-    expect(tokenizer.tokenize('.these are things, no')).toEqual(['these', 'are', 'things', 'no'])
-  })
-
-  it('should swallow duplicate punctuation', function () {
-    expect(tokenizer.tokenize('i shal... pause')).toEqual(['i', 'shal', 'pause'])
-  })
-
-  it('should remove underscores', function () {
-    expect(tokenizer.tokenize('_ hi_this_is_a_test_case_ for__removing___underscores_')).toEqual(['hi', 'this', 'is', 'a', 'test', 'case', 'for', 'removing', 'underscores'])
-  })
-
-  it('should handle the use of hyphen inside words (issue #656)', function () {
-    expect(tokenizer.tokenize('links text-based opposed image-based links/CTA\'s')).toEqual(
-      ['links', 'text-based', 'opposed', 'image-based', 'links/CTA\'s'])
-  })
-})

package/ts_spec/aggressive_tokenizer_sv_spec.ts

@@ -1,52 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-'use strict'
-
-import { AggressiveTokenizerSv } from 'lib/natural'
-const tokenizer = new AggressiveTokenizerSv()
-
-describe('aggressive_tokenizer_sv', function () {
-  it('should tokenize strings', function () {
-    expect(tokenizer.tokenize('Ett tu tre')).toEqual(['Ett', 'tu', 'tre'])
-  })
-
-  it('should swallow punctuation', function () {
-    expect(tokenizer.tokenize('Ett, tu, tre')).toEqual(['Ett', 'tu', 'tre'])
-  })
-
-  it('should swallow final punctuation', function () {
-    expect(tokenizer.tokenize('Ett, tu, tre?')).toEqual(['Ett', 'tu', 'tre'])
-  })
-
-  it('should swallow initial punctuation', function () {
-    expect(tokenizer.tokenize('.Ett, tu, tre')).toEqual(['Ett', 'tu', 'tre'])
-  })
-
-  it('should swallow duplicate punctuation', function () {
-    expect(tokenizer.tokenize('Ett, tu... tre')).toEqual(['Ett', 'tu', 'tre'])
-  })
-
-  it('should not split on hyphen or Swedish letters', function () {
-    expect(tokenizer.tokenize('It-bolaget ägs av en 30-åring')).toEqual(['It-bolaget', 'ägs', 'av', 'en', '30-åring'])
-  })
-})

package/ts_spec/aggressive_tokenizer_vi_spec.ts

@@ -1,48 +0,0 @@
-/*
-Copyright (c) 2018, Hugo W.L. ter Doest
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this softwé and associated documentation files (the "Softwé"), to deal
-in the Softwé without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Softwé, and to permit persons to whom the Softwé is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Softwé.
-
-THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-'use strict'
-
-import { AggressiveTokenizerVi } from 'lib/natural'
-const tokenizer = new AggressiveTokenizerVi()
-
-describe('aggressive_tokenizer_vi', function () {
-  it('should tokenize strings', function () {
-    expect(tokenizer.tokenize('Tôi bị lạc')).toEqual(['Tôi', 'bị', 'lạc'])
-  })
-
-  it('should swallow final punctuation', function () {
-    expect(tokenizer.tokenize('Làm ơn đợi một lát!')).toEqual(['Làm', 'ơn', 'đợi', 'một', 'lát'])
-  })
-
-  it('should swallow final punctuation', function () {
-    expect(tokenizer.tokenize('Tôi đang tìm John.')).toEqual(['Tôi', 'đang', 'tìm', 'John'])
-  })
-
-  it('should swallow initial punctuation', function () {
-    expect(tokenizer.tokenize('.Đi thẳng, sau đó rẽ trái / phải')).toEqual(['Đi', 'thẳng', 'sau', 'đó', 'rẽ', 'trái', 'phải'])
-  })
-
-  it('should swallow duplicate punctuation', function () {
-    expect(tokenizer.tokenize('.Xin giữ máy!')).toEqual(['Xin', 'giữ', 'máy'])
-  })
-})

package/ts_spec/bayes_classifier_spec.ts

@@ -1,165 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-'use strict'
-
-import { BayesClassifier, PorterStemmer } from 'lib/natural'
-
-function setupClassifier (): BayesClassifier {
-  const classifier = new BayesClassifier()
-  classifier.addDocument(['fix', 'box'], 'computing')
-  classifier.addDocument(['write', 'code'], 'computing')
-  classifier.addDocument(['script', 'code'], 'computing')
-  classifier.addDocument(['write', 'book'], 'literature')
-  classifier.addDocument(['read', 'book'], 'literature')
-  classifier.addDocument(['study', 'book'], 'literature')
-  return classifier
-}
-
-describe('bayes classifier', function () {
-  describe('classifier', function () {
-    it('should classify with arrays', function () {
-      const classifier = setupClassifier()
-      classifier.train()
-      expect(classifier.classify(['bug', 'code'])).toBe('computing')
-      expect(classifier.classify(['read', 'thing'])).toBe('literature')
-    })
-
-    it('should classify with parallel training', function () {
-      const classifier = setupClassifier()
-      // Check for parallel method
-      if (classifier.Threads !== null) {
-        classifier.trainParallel(2, function (err) {
-          if (err !== null) {
-            console.log(err)
-            return
-          }
-          expect(classifier.classify(['bug', 'code'])).toBe('computing')
-          expect(classifier.classify(['read', 'thing'])).toBe('literature')
-        })
-      }
-    })
-
-    it('should classify with parallel batched training', function () {
-      const classifier = setupClassifier()
-      // Check for parallel method
-      if (classifier.Threads !== null) {
-        classifier.on('doneTraining', function () {
-          expect(classifier.classify(['bug', 'code'])).toBe('computing')
-          expect(classifier.classify(['read', 'thing'])).toBe('literature')
-          // asyncSpecDone();
-        })
-        classifier.trainParallelBatches({ numThreads: 2, batchSize: 2 })
-      }
-    })
-
-    it('should provide all classification scores', function () {
-      const classifier = setupClassifier()
-      classifier.train()
-
-      expect(classifier.getClassifications('i write code')[0].label).toBe('computing')
-      expect(classifier.getClassifications('i write code')[1].label).toBe('literature')
-    })
-
-    function setupClassifierWithSentences (): BayesClassifier {
-      const classifier = new BayesClassifier()
-      classifier.addDocument('i fixed the box', 'computing')
-      classifier.addDocument('i write code', 'computing')
-      classifier.addDocument('nasty script code', 'computing')
-      classifier.addDocument('write a book', 'literature')
-      classifier.addDocument('read a book', 'literature')
-      classifier.addDocument('study the books', 'literature')
-      return classifier
-    }
-
-    it('should classify with strings', function () {
-      const classifier = setupClassifierWithSentences()
-      classifier.train()
-      expect(classifier.classify('a bug in the code')).toBe('computing')
-      expect(classifier.classify('read all the books')).toBe('literature')
-    })
-
-    it('should classify and re-classify after document-removal', function () {
-      const classifier = new BayesClassifier()
-      const classifications = new Map<string, number>()
-
-      // Add some good/bad docs and train
-      classifier.addDocument('foo bar baz', 'good')
-      classifier.addDocument('qux zooby', 'bad')
-      classifier.addDocument('asdf qwer', 'bad')
-      classifier.train()
-
-      expect(classifier.classify('foo')).toBe('good')
-      expect(classifier.classify('qux')).toBe('bad')
-
-      // Remove one of the bad docs, retrain
-      classifier.removeDocument('qux zooby', 'bad')
-      classifier.retrain()
-
-      // Simple `classify` will still return a single result, even if
-      // ratio for each side is equal -- have to compare actual values in
-      // the classifications, should be equal since qux is unclassified
-      const arr = classifier.getClassifications('qux')
-      let item: { label: string, value: number }
-      for (let i = 0, ii = arr.length; i < ii; i++) {
-        item = arr[i]
-        classifications.set(item.label, item.value)
-      }
-      expect(classifications.get('good')).toEqual(classifications.get('bad'))
-
-      // Re-classify as good, retrain
-      classifier.addDocument('qux zooby', 'good')
-      classifier.retrain()
-
-      // Should now be good, original docs should be unaffected
-      expect(classifier.classify('foo')).toBe('good')
-      expect(classifier.classify('qux')).toBe('good')
-    })
-
-    it('should serialize and deserialize a working classifier', function () {
-      const classifier = setupClassifierWithSentences()
-      const objAsString = JSON.stringify(classifier)
-      const obj: Record<string, unknown> = JSON.parse(objAsString)
-      const newClassifier = BayesClassifier.restore(obj)
-
-      newClassifier.addDocument('kick a ball', 'sports')
-      newClassifier.addDocument('hit some balls', 'sports')
-      newClassifier.addDocument('kick and punch', 'sports')
-
-      newClassifier.train()
-
-      expect(newClassifier.classify('a bug in the code')).toBe('computing')
-      expect(newClassifier.classify('read all the books')).toBe('literature')
-      expect(newClassifier.classify('kick butt')).toBe('sports')
-    })
-
-    it('should accept an optional smoothing parameter for the Bayesian estimates', function () {
-      const defaultClassifier = new BayesClassifier()
-      const newClassifier1 = new BayesClassifier(PorterStemmer)
-      const newClassifier2 = new BayesClassifier(PorterStemmer, 0.1)
-
-      expect(defaultClassifier.classifier.smoothing).toBe(1.0)
-      expect(newClassifier1.classifier.smoothing).toBe(1.0)
-      expect(newClassifier2.classifier.smoothing).toBe(0.1)
-    })
-  })
-})

package/ts_spec/brill_pos_tagger_spec.ts

@@ -1,79 +0,0 @@
-/*
-Unit test for Brill's POS Tagger: test against the pos module
-Copyright (C) 2019 Hugo W.L. ter Doest
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-'use strict'
-
-import { WordTokenizer, Lexicon, RuleSet, BrillPOSTagger } from 'lib/natural'
-import type { Sentence, TagResults } from 'lib/natural'
-
-import * as englishSentences from '../spec/test_data/NYT-20150205-picassos-granddaughter-plans-to-sell-art-worrying-the-market.json'
-
-import jsonData from '../spec/test_data/NYT-20150205-picassos-granddaughter-plans_expected_tag_results.json'
-
-import * as dutchSentences from '../spec/test_data/Volkskrant-20150205-Knot-geldpers-aanzetten-is-paardenmiddel-voor-half-procent-inflatie.json'
-
-const englishTagResults: TagResults = jsonData as TagResults
-const DEBUG = false
-
-// Compares two tagged sentences. First one is in the old POSJS format, i.e.
-// an array of two position arrays. The second one is a Sentence object
-// that holds an array of objects for each position {token: "string", tag: "string"}
-function compareTaggedSentences (sentenceInOldFormat: string[][], sentenceInNewFormat: Sentence): boolean {
-  let equal = true
-  sentenceInOldFormat.forEach(function (wordPlusTag: string[], index: number) {
-    equal = equal &&
-      (wordPlusTag[1] === sentenceInNewFormat.taggedWords[index].tag)
-    DEBUG && console.log(wordPlusTag[1] + ' ' + sentenceInNewFormat.taggedWords[index].tag)
-  })
-  return equal
-}
-
-describe('Brill\'s POS Tagger', function () {
-  let brillPOSTagger: BrillPOSTagger
-  let lexicon: Lexicon
-  let ruleSet: RuleSet
-  const tokenizer = new WordTokenizer()
-
-  it('should process an English newspaper article just like the dariusk/pos-js module', function () {
-    lexicon = new Lexicon('EN', 'NN')
-    expect(lexicon.nrEntries()).toBeGreaterThan(0)
-    ruleSet = new RuleSet('EN')
-    expect(ruleSet.nrRules()).toBeGreaterThan(0)
-    brillPOSTagger = new BrillPOSTagger(lexicon, ruleSet)
-
-    englishSentences.sentences.forEach(function (sentence: string, index: number) {
-      const tokenizedSentence = tokenizer.tokenize(sentence)
-      const taggedSentence = brillPOSTagger.tag(tokenizedSentence)
-      expect(compareTaggedSentences(englishTagResults.results[index], taggedSentence)).toBe(true)
-    })
-  })
-
-  it('should tag a Dutch news paper article', function () {
-    lexicon = new Lexicon('DU', 'N')
-    expect(lexicon.nrEntries()).toBeGreaterThan(0)
-    const ruleSet = new RuleSet('DU')
-    expect(ruleSet.nrRules()).toBeGreaterThan(0)
-    brillPOSTagger = new BrillPOSTagger(lexicon, ruleSet)
-
-    dutchSentences.sentences.forEach(function (sentence, index) {
-      const tokenizedSentence = tokenizer.tokenize(sentence)
-      const taggedSentence = brillPOSTagger.tag(tokenizedSentence)
-      expect(tokenizedSentence.length).toEqual(taggedSentence.taggedWords.length)
-    })
-  })
-})