natural 6.12.0 → 7.0.0

package/ts_spec/aggressive_tokenizer_pt_spec.ts

@@ -0,0 +1,48 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWé.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerPt } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerPt()
+
+describe('aggressive_tokenizer_pt', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('isso é coração')).toEqual(['isso', 'é', 'coração'])
+  })
+
+  it('should swallow punctuation', function () {
+    expect(tokenizer.tokenize('isso é coração, no')).toEqual(['isso', 'é', 'coração', 'no'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('isso é coração, no?')).toEqual(['isso', 'é', 'coração', 'no'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.isso é coração, no')).toEqual(['isso', 'é', 'coração', 'no'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('eu vou... pause')).toEqual(['eu', 'vou', 'pause'])
+  })
+})

package/ts_spec/aggressive_tokenizer_spec.ts

@@ -0,0 +1,57 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizer } from 'lib/natural'
+const tokenizer = new AggressiveTokenizer()
+
+describe('aggressive_tokenizer', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('these are things')).toEqual(['these', 'are', 'things'])
+  })
+
+  it('should swallow punctuation', function () {
+    expect(tokenizer.tokenize('these are things, no')).toEqual(['these', 'are', 'things', 'no'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('these are things, no?')).toEqual(['these', 'are', 'things', 'no'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.these are things, no')).toEqual(['these', 'are', 'things', 'no'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('i shal... pause')).toEqual(['i', 'shal', 'pause'])
+  })
+
+  it('should remove underscores', function () {
+    expect(tokenizer.tokenize('_ hi_this_is_a_test_case_ for__removing___underscores_')).toEqual(['hi', 'this', 'is', 'a', 'test', 'case', 'for', 'removing', 'underscores'])
+  })
+
+  it('should handle the use of hyphen inside words (issue #656)', function () {
+    expect(tokenizer.tokenize('links text-based opposed image-based links/CTA\'s')).toEqual(
+      ['links', 'text-based', 'opposed', 'image-based', 'links/CTA\'s'])
+  })
+})

package/ts_spec/aggressive_tokenizer_sv_spec.ts

@@ -0,0 +1,52 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerSv } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerSv()
+
+describe('aggressive_tokenizer_sv', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('Ett tu tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow punctuation', function () {
+    expect(tokenizer.tokenize('Ett, tu, tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('Ett, tu, tre?')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.Ett, tu, tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('Ett, tu... tre')).toEqual(['Ett', 'tu', 'tre'])
+  })
+
+  it('should not split on hyphen or Swedish letters', function () {
+    expect(tokenizer.tokenize('It-bolaget ägs av en 30-åring')).toEqual(['It-bolaget', 'ägs', 'av', 'en', '30-åring'])
+  })
+})

package/ts_spec/aggressive_tokenizer_vi_spec.ts

@@ -0,0 +1,48 @@
+/*
+Copyright (c) 2018, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { AggressiveTokenizerVi } from 'lib/natural'
+const tokenizer = new AggressiveTokenizerVi()
+
+describe('aggressive_tokenizer_vi', function () {
+  it('should tokenize strings', function () {
+    expect(tokenizer.tokenize('Tôi bị lạc')).toEqual(['Tôi', 'bị', 'lạc'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('Làm ơn đợi một lát!')).toEqual(['Làm', 'ơn', 'đợi', 'một', 'lát'])
+  })
+
+  it('should swallow final punctuation', function () {
+    expect(tokenizer.tokenize('Tôi đang tìm John.')).toEqual(['Tôi', 'đang', 'tìm', 'John'])
+  })
+
+  it('should swallow initial punctuation', function () {
+    expect(tokenizer.tokenize('.Đi thẳng, sau đó rẽ trái / phải')).toEqual(['Đi', 'thẳng', 'sau', 'đó', 'rẽ', 'trái', 'phải'])
+  })
+
+  it('should swallow duplicate punctuation', function () {
+    expect(tokenizer.tokenize('.Xin giữ máy!')).toEqual(['Xin', 'giữ', 'máy'])
+  })
+})

package/ts_spec/bayes_classifier_spec.ts

@@ -0,0 +1,165 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { BayesClassifier, PorterStemmer } from 'lib/natural'
+
+function setupClassifier (): BayesClassifier {
+  const classifier = new BayesClassifier()
+  classifier.addDocument(['fix', 'box'], 'computing')
+  classifier.addDocument(['write', 'code'], 'computing')
+  classifier.addDocument(['script', 'code'], 'computing')
+  classifier.addDocument(['write', 'book'], 'literature')
+  classifier.addDocument(['read', 'book'], 'literature')
+  classifier.addDocument(['study', 'book'], 'literature')
+  return classifier
+}
+
+describe('bayes classifier', function () {
+  describe('classifier', function () {
+    it('should classify with arrays', function () {
+      const classifier = setupClassifier()
+      classifier.train()
+      expect(classifier.classify(['bug', 'code'])).toBe('computing')
+      expect(classifier.classify(['read', 'thing'])).toBe('literature')
+    })
+
+    it('should classify with parallel training', function () {
+      const classifier = setupClassifier()
+      // Check for parallel method
+      if (classifier.Threads !== null) {
+        classifier.trainParallel(2, function (err) {
+          if (err !== null) {
+            console.log(err)
+            return
+          }
+          expect(classifier.classify(['bug', 'code'])).toBe('computing')
+          expect(classifier.classify(['read', 'thing'])).toBe('literature')
+        })
+      }
+    })
+
+    it('should classify with parallel batched training', function () {
+      const classifier = setupClassifier()
+      // Check for parallel method
+      if (classifier.Threads !== null) {
+        classifier.on('doneTraining', function () {
+          expect(classifier.classify(['bug', 'code'])).toBe('computing')
+          expect(classifier.classify(['read', 'thing'])).toBe('literature')
+          // asyncSpecDone();
+        })
+        classifier.trainParallelBatches({ numThreads: 2, batchSize: 2 })
+      }
+    })
+
+    it('should provide all classification scores', function () {
+      const classifier = setupClassifier()
+      classifier.train()
+
+      expect(classifier.getClassifications('i write code')[0].label).toBe('computing')
+      expect(classifier.getClassifications('i write code')[1].label).toBe('literature')
+    })
+
+    function setupClassifierWithSentences (): BayesClassifier {
+      const classifier = new BayesClassifier()
+      classifier.addDocument('i fixed the box', 'computing')
+      classifier.addDocument('i write code', 'computing')
+      classifier.addDocument('nasty script code', 'computing')
+      classifier.addDocument('write a book', 'literature')
+      classifier.addDocument('read a book', 'literature')
+      classifier.addDocument('study the books', 'literature')
+      return classifier
+    }
+
+    it('should classify with strings', function () {
+      const classifier = setupClassifierWithSentences()
+      classifier.train()
+      expect(classifier.classify('a bug in the code')).toBe('computing')
+      expect(classifier.classify('read all the books')).toBe('literature')
+    })
+
+    it('should classify and re-classify after document-removal', function () {
+      const classifier = new BayesClassifier()
+      const classifications = new Map<string, number>()
+
+      // Add some good/bad docs and train
+      classifier.addDocument('foo bar baz', 'good')
+      classifier.addDocument('qux zooby', 'bad')
+      classifier.addDocument('asdf qwer', 'bad')
+      classifier.train()
+
+      expect(classifier.classify('foo')).toBe('good')
+      expect(classifier.classify('qux')).toBe('bad')
+
+      // Remove one of the bad docs, retrain
+      classifier.removeDocument('qux zooby', 'bad')
+      classifier.retrain()
+
+      // Simple `classify` will still return a single result, even if
+      // ratio for each side is equal -- have to compare actual values in
+      // the classifications, should be equal since qux is unclassified
+      const arr = classifier.getClassifications('qux')
+      let item: { label: string, value: number }
+      for (let i = 0, ii = arr.length; i < ii; i++) {
+        item = arr[i]
+        classifications.set(item.label, item.value)
+      }
+      expect(classifications.get('good')).toEqual(classifications.get('bad'))
+
+      // Re-classify as good, retrain
+      classifier.addDocument('qux zooby', 'good')
+      classifier.retrain()
+
+      // Should now be good, original docs should be unaffected
+      expect(classifier.classify('foo')).toBe('good')
+      expect(classifier.classify('qux')).toBe('good')
+    })
+
+    it('should serialize and deserialize a working classifier', function () {
+      const classifier = setupClassifierWithSentences()
+      const objAsString = JSON.stringify(classifier)
+      const obj: Record<string, unknown> = JSON.parse(objAsString)
+      const newClassifier = BayesClassifier.restore(obj)
+
+      newClassifier.addDocument('kick a ball', 'sports')
+      newClassifier.addDocument('hit some balls', 'sports')
+      newClassifier.addDocument('kick and punch', 'sports')
+
+      newClassifier.train()
+
+      expect(newClassifier.classify('a bug in the code')).toBe('computing')
+      expect(newClassifier.classify('read all the books')).toBe('literature')
+      expect(newClassifier.classify('kick butt')).toBe('sports')
+    })
+
+    it('should accept an optional smoothing parameter for the Bayesian estimates', function () {
+      const defaultClassifier = new BayesClassifier()
+      const newClassifier1 = new BayesClassifier(PorterStemmer)
+      const newClassifier2 = new BayesClassifier(PorterStemmer, 0.1)
+
+      expect(defaultClassifier.classifier.smoothing).toBe(1.0)
+      expect(newClassifier1.classifier.smoothing).toBe(1.0)
+      expect(newClassifier2.classifier.smoothing).toBe(0.1)
+    })
+  })
+})

package/ts_spec/brill_pos_tagger_spec.ts

@@ -0,0 +1,79 @@
+/*
+Unit test for Brill's POS Tagger: test against the pos module
+Copyright (C) 2019 Hugo W.L. ter Doest
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict'
+
+import { WordTokenizer, Lexicon, RuleSet, BrillPOSTagger } from 'lib/natural'
+import type { Sentence, TagResults } from 'lib/natural'
+
+import * as englishSentences from '../spec/test_data/NYT-20150205-picassos-granddaughter-plans-to-sell-art-worrying-the-market.json'
+
+import jsonData from '../spec/test_data/NYT-20150205-picassos-granddaughter-plans_expected_tag_results.json'
+
+import * as dutchSentences from '../spec/test_data/Volkskrant-20150205-Knot-geldpers-aanzetten-is-paardenmiddel-voor-half-procent-inflatie.json'
+
+const englishTagResults: TagResults = jsonData as TagResults
+const DEBUG = false
+
+// Compares two tagged sentences. First one is in the old POSJS format, i.e.
+// an array of two position arrays. The second one is a Sentence object
+// that holds an array of objects for each position {token: "string", tag: "string"}
+function compareTaggedSentences (sentenceInOldFormat: string[][], sentenceInNewFormat: Sentence): boolean {
+  let equal = true
+  sentenceInOldFormat.forEach(function (wordPlusTag: string[], index: number) {
+    equal = equal &&
+      (wordPlusTag[1] === sentenceInNewFormat.taggedWords[index].tag)
+    DEBUG && console.log(wordPlusTag[1] + ' ' + sentenceInNewFormat.taggedWords[index].tag)
+  })
+  return equal
+}
+
+describe('Brill\'s POS Tagger', function () {
+  let brillPOSTagger: BrillPOSTagger
+  let lexicon: Lexicon
+  let ruleSet: RuleSet
+  const tokenizer = new WordTokenizer()
+
+  it('should process an English newspaper article just like the dariusk/pos-js module', function () {
+    lexicon = new Lexicon('EN', 'NN')
+    expect(lexicon.nrEntries()).toBeGreaterThan(0)
+    ruleSet = new RuleSet('EN')
+    expect(ruleSet.nrRules()).toBeGreaterThan(0)
+    brillPOSTagger = new BrillPOSTagger(lexicon, ruleSet)
+
+    englishSentences.sentences.forEach(function (sentence: string, index: number) {
+      const tokenizedSentence = tokenizer.tokenize(sentence)
+      const taggedSentence = brillPOSTagger.tag(tokenizedSentence)
+      expect(compareTaggedSentences(englishTagResults.results[index], taggedSentence)).toBe(true)
+    })
+  })
+
+  it('should tag a Dutch news paper article', function () {
+    lexicon = new Lexicon('DU', 'N')
+    expect(lexicon.nrEntries()).toBeGreaterThan(0)
+    const ruleSet = new RuleSet('DU')
+    expect(ruleSet.nrRules()).toBeGreaterThan(0)
+    brillPOSTagger = new BrillPOSTagger(lexicon, ruleSet)
+
+    dutchSentences.sentences.forEach(function (sentence, index) {
+      const tokenizedSentence = tokenizer.tokenize(sentence)
+      const taggedSentence = brillPOSTagger.tag(tokenizedSentence)
+      expect(tokenizedSentence.length).toEqual(taggedSentence.taggedWords.length)
+    })
+  })
+})

package/ts_spec/brill_pos_trainer_spec.ts

@@ -0,0 +1,107 @@
+/*
+Unit test for Brill's POS Trainer
+Copyright (C) 2019 Hugo W.L. ter Doest
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict'
+
+import {
+  Corpus,
+  ruleTemplates,
+  RuleTemplate,
+  Sentence,
+  BrillPOSTrainer,
+  BrillPOSTagger,
+  BrillPOSTester
+} from 'lib/natural'
+
+import type {
+  Lexicon,
+  RuleSet
+} from 'lib/natural'
+
+// Import a JSON file with the brown corpus
+import brownCorpus from '../spec/test_data/browntag_nolines_excerpt.json'
+
+const DEBUG = false
+const JSON_FLAG = 2
+
+function selectRuleTemplates (templateNames: string[]): RuleTemplate[] {
+  const templates: RuleTemplate[] = []
+  templateNames.forEach(function (name: string) {
+    if (ruleTemplates[name] !== undefined) {
+      const template = new RuleTemplate(name, ruleTemplates[name])
+      templates.push(template)
+    }
+  })
+  return templates
+}
+
+describe('Brill\'s POS Trainer', function () {
+  let corpus = null
+  let corpora: Corpus[]
+  const percentageTrain = 60
+  let trainLexicon: Lexicon
+  // Templates consider only tags, no words
+  const templateNames = [
+    'NEXT-TAG',
+    'PREV-TAG',
+    'PREV-1-OR-2-OR-3-TAG',
+    'PREV-1-OR-2-TAG',
+    'NEXT1OR2TAG',
+    'NEXT1OR2OR3TAG',
+    'SURROUNDTAG',
+    'PREV2TAG',
+    'NEXT2TAG'
+  ]
+  let templates: RuleTemplate[]
+  let trainer = null
+  let ruleSet: RuleSet
+
+  it('should split the corpus in a training and testing corpus', function () {
+    corpus = new Corpus(brownCorpus, JSON_FLAG, Sentence)
+    DEBUG && console.log('Corpus: ' + JSON.stringify(corpus, null, 2))
+    corpora = corpus.splitInTrainAndTest(percentageTrain)
+    expect(corpora[0].nrSentences() + corpora[1].nrSentences()).toEqual(corpus.nrSentences())
+  })
+
+  it('should build a lexicon from the training corpus', function () {
+    trainLexicon = corpora[0].buildLexicon()
+    // Set default category to noun (NN)
+    // and default category for capitalised words to proper noun (NP)
+    trainLexicon.setDefaultCategories('NN', 'NP')
+    expect(trainLexicon.nrEntries()).not.toEqual(0)
+  })
+
+  it('should set up the rule templates', function () {
+    templates = selectRuleTemplates(templateNames)
+    expect(templates.length).toEqual(templateNames.length)
+  })
+
+  it('should train on the training corpus to derive transformation rules', function () {
+    trainer = new BrillPOSTrainer(1)
+    ruleSet = trainer.train(corpora[0], templates, trainLexicon)
+    expect(ruleSet.nrRules()).toBeGreaterThan(0)
+    expect(trainer.printRulesWithScores()).not.toEqual('')
+  })
+
+  it('should test the derived transformation rules on the test corpus', function () {
+    const tagger = new BrillPOSTagger(trainLexicon, ruleSet)
+    const tester = new BrillPOSTester()
+    const scores = tester.test(corpora[1], tagger)
+    expect(scores[1]).toBeGreaterThan(0)
+  })
+})

package/ts_spec/carry_stemmer_fr_spec.ts

@@ -0,0 +1,97 @@
+/*
+Copyright (c) 2020, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { CarryStemmerFr as stemmer } from 'lib/natural'
+
+// const snowBallDict = require('spec/test_data/snowball_fr.json')
+
+const tests = [
+  {
+    input: 'action',
+    output: 'ac'
+  },
+  {
+    input: 'acteur',
+    output: 'ac'
+  },
+  {
+    input: 'actrices',
+    output: 'ac'
+  },
+  {
+    input: 'Dleyton',
+    output: 'Dleyton'
+  },
+  {
+    input: 'volera',
+    output: 'vol'
+  }
+]
+
+const testSentences = [
+  {
+    input: 'Le petit cheval de manège',
+    output: ['pet', 'cheval', 'manèg']
+  }
+]
+
+describe('Carry stemmer', function () {
+  tests.forEach(test => {
+    it('stems ' + test.input + ' correctly', function () {
+      expect(stemmer.stem(test.input)).toBe(test.output)
+    })
+  })
+
+  testSentences.forEach(test => {
+    it('tokenizes and stems ' + test.input + ' correctly', function () {
+      expect(stemmer.tokenizeAndStem(test.input)).toEqual(test.output)
+    })
+  })
+
+  /*
+  it('should perform stemming on a lot of words', function() {
+    var ok = [];
+    var ko = [];
+
+    Object.keys(snowBallDict).forEach(word => {
+
+      var stemmed = stemmer.stem(word);
+      var expectedStem = snowBallDict[word];
+
+      if (stemmed === expectedStem) {
+        ok.push(word);
+      }
+      else {
+        ko.push({
+          word: word,
+          expected: expectedStem,
+          actual: stemmed
+        });
+      }
+    });
+
+    expect(ko.length).toBe(0);
+  });
+*/
+})