natural 6.12.0 → 7.0.0

package/ts_spec/sentence_tokenizer_spec.ts

@@ -0,0 +1,189 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { SentenceTokenizer as Tokenizer } from 'lib/natural'
+const tokenizer = new Tokenizer()
+
+describe('sentence_tokenizer', function () {
+  it('should tokenize strings and trim whitespace', function () {
+    expect(
+      tokenizer.tokenize('This is a sentence. This is another sentence.')
+    )
+      .toEqual([
+        'This is a sentence.',
+        'This is another sentence.'
+      ])
+  })
+
+  /*
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect(
+        'This is a sentence. This is another sentence.'.tokenize()
+    ).toEqual([
+        'This is a sentence.',
+        'This is another sentence.'
+    ]);
+  });
+
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect(
+        'This is a sentence. This is another sentence.'.tokenize()
+    ).toEqual([
+        'This is a sentence.',
+        'This is another sentence.'
+    ]);
+  });
+  */
+
+  it('should include quotation marks', function () {
+    expect(
+      tokenizer.tokenize('"This is a sentence." This is another sentence.')
+    ).toEqual([
+      '"This is a sentence."',
+      'This is another sentence.'
+    ])
+  })
+
+  it('should include brackets', function () {
+    expect(
+      tokenizer.tokenize('This is a sentence. [This is another sentence.]')
+    ).toEqual([
+      'This is a sentence.',
+      '[This is another sentence.]'
+    ])
+  })
+
+  it('should handle repetitive punctuation', function () {
+    expect(
+      tokenizer.tokenize('I love you!! Do you love me??')
+    ).toEqual(['I love you!!', 'Do you love me??'
+    ])
+  })
+
+  it('should handle repetitive punctuation with space', function () {
+    expect(
+      tokenizer.tokenize('I love you! ! Do you love me? ?')
+    ).toEqual(['I love you! !', 'Do you love me? ?'
+    ])
+  })
+
+  it('should handle decimal points in numbers', function () {
+    expect(
+      tokenizer.tokenize('Pi is approximately equal to 3.14.')
+    ).toEqual([
+      'Pi is approximately equal to 3.14.'
+    ])
+  })
+
+  it('should handle periods in email addresses', function () {
+    expect(
+      tokenizer.tokenize('My email address is batman@example.com.')
+    ).toEqual([
+      'My email address is batman@example.com.'
+    ])
+  })
+
+  it('should handle periods in web addresses', function () {
+    expect(
+      tokenizer.tokenize('My twitter feed can be found at https://twitter.com/user1.')
+    ).toEqual([
+      'My twitter feed can be found at https://twitter.com/user1.'
+    ])
+  })
+
+  it('should handle an ellipsis followed by punctuation', function () {
+    expect(
+      tokenizer.tokenize('Is this the end for our heroes...?')
+    ).toEqual([
+      'Is this the end for our heroes...?'
+    ])
+  })
+
+  it('should handle multiple spaces separating sentences', function () {
+    expect(
+      tokenizer.tokenize('Tune in tomorrow and find out!  Same Bat-Time!  Same Bat-Channel!')
+    ).toEqual([
+      'Tune in tomorrow and find out!',
+      'Same Bat-Time!',
+      'Same Bat-Channel!'
+    ])
+  })
+
+  it('should handle braces and quotes (issue #591)', function () {
+    expect(
+      tokenizer.tokenize('Teste. Test test. Test test: “Test.”')
+    ).toEqual([
+      'Teste.',
+      'Test test.',
+      'Test test: “Test.”'
+    ])
+    expect(
+      tokenizer.tokenize('Test Test. Test test, test test (test test) test: “Test.”')
+    ).toEqual([
+      'Test Test.',
+      'Test test, test test (test test) test: “Test.”'
+    ])
+    expect(
+      tokenizer.tokenize('Test Test. Test test, test (test) test (test test) test: “Test.”')
+    ).toEqual([
+      'Test Test.',
+      'Test test, test (test) test (test test) test: “Test.”'
+    ])
+    expect(
+      tokenizer.tokenize('Test: Test (test) test “Test.”')
+    ).toEqual([
+      'Test: Test (test) test “Test.”'
+    ])
+  })
+
+  it('Should handle text with the ellipsis symbol … (issue #648)', function () {
+    expect(
+      tokenizer.tokenize('We’re heading for a catastrophic global temperature rise… Fires are blazing from the Amazon to the Arctic.')
+    ).toEqual([
+      'We’re heading for a catastrophic global temperature rise…',
+      'Fires are blazing from the Amazon to the Arctic.'
+    ])
+  })
+  it('It should handle last sentence without punctuation (issue #648)', function () {
+    expect(
+      tokenizer.tokenize('We’re heading for a catastrophic global temperature rise. Fires are blazing from the Amazon to the Arctic')
+    ).toEqual([
+      'We’re heading for a catastrophic global temperature rise.',
+      'Fires are blazing from the Amazon to the Arctic'
+    ])
+  })
+  it('It should handle the example from issue #689 correctly', function () {
+    const testInput = `
+      This is some test content.
+
+      We're trying to figure out variations in versions of the package.
+    `.trim()
+    expect(tokenizer.tokenize(testInput)).toEqual([
+      'This is some test content.',
+      'We\'re trying to figure out variations in versions of the package.'
+    ])
+  })
+})

package/ts_spec/shortest_path_tree_spec.ts

@@ -0,0 +1,141 @@
+/*
+Copyright (c) 2014, Lee Wenzhu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import {
+  ShortestPathTree as SPT,
+  Topological,
+  EdgeWeightedDigraph
+} from 'lib/natural'
+
+describe('shortest path tree', function () {
+  const digraph = new EdgeWeightedDigraph()
+  digraph.add(5, 4, 0.35)
+  digraph.add(4, 7, 0.37)
+  digraph.add(5, 7, 0.28)
+  digraph.add(5, 1, 0.32)
+  digraph.add(4, 0, 0.38)
+  digraph.add(0, 2, 0.26)
+  digraph.add(3, 7, 0.39)
+  digraph.add(1, 3, 0.29)
+  digraph.add(7, 2, 0.34)
+  digraph.add(6, 2, 0.40)
+  digraph.add(3, 6, 0.52)
+  digraph.add(6, 0, 0.58)
+  digraph.add(6, 4, 0.93)
+  describe('edge weighted digraph normal operations', function () {
+    it('should get the numbers of vertexs', function () {
+      expect(digraph.v()).toBe(8)
+    })
+    it('should get the numbers of edges', function () {
+      expect(digraph.e()).toBe(13)
+    })
+    it('should print all item in digraph', function () {
+      expect(digraph.toString()).toBe('0 -> 2, 0.26\n' +
+                                            '1 -> 3, 0.29\n' +
+                                            '3 -> 7, 0.39\n' +
+                                            '3 -> 6, 0.52\n' +
+                                            '4 -> 7, 0.37\n' +
+                                            '4 -> 0, 0.38\n' +
+                                            '5 -> 4, 0.35\n' +
+                                            '5 -> 7, 0.28\n' +
+                                            '5 -> 1, 0.32\n' +
+                                            '6 -> 2, 0.4\n' +
+                                            '6 -> 0, 0.58\n' +
+                                            '6 -> 4, 0.93\n' +
+                                            '7 -> 2, 0.34')
+    })
+  })
+
+  describe('topo sort for digraph', function () {
+    it('should sort all the vertexs', function () {
+      const topoSort = new Topological(digraph)
+      expect(topoSort.isDAG()).toBe(true)
+      expect(topoSort.order()).toEqual([5, 1, 3, 6, 4, 7, 0, 2])
+    })
+  })
+
+  describe('shortest path tree normal operations', function () {
+    const lpt = new SPT(digraph, 5)
+    it('should determine existence of paths', function () {
+      expect(lpt.hasPathTo(0)).toBe(true)
+      expect(lpt.hasPathTo(1)).toBe(true)
+      expect(lpt.hasPathTo(2)).toBe(true)
+      expect(lpt.hasPathTo(3)).toBe(true)
+      expect(lpt.hasPathTo(4)).toBe(true)
+      expect(lpt.hasPathTo(5)).toBe(false)
+      expect(lpt.hasPathTo(6)).toBe(true)
+      expect(lpt.hasPathTo(7)).toBe(true)
+    })
+    it('should determine paths', function () {
+      expect(lpt.pathTo(0)).toEqual([5, 4, 0])
+      expect(lpt.pathTo(1)).toEqual([5, 1])
+      expect(lpt.pathTo(2)).toEqual([5, 7, 2])
+      expect(lpt.pathTo(3)).toEqual([5, 1, 3])
+      expect(lpt.pathTo(4)).toEqual([5, 4])
+      expect(lpt.pathTo(5)).toEqual([])
+      expect(lpt.pathTo(6)).toEqual([5, 1, 3, 6])
+      expect(lpt.pathTo(7)).toEqual([5, 7])
+    })
+    it('should calculate distances', function () {
+      expect(lpt.getDistTo(0)).toBe(0.73)
+      expect(lpt.getDistTo(1)).toBe(0.32)
+      expect(lpt.getDistTo(2)).toBe(0.62)
+      expect(lpt.getDistTo(3)).toBe(0.61)
+      expect(lpt.getDistTo(4)).toBe(0.35)
+      expect(lpt.getDistTo(5)).toBe(0)
+      expect(lpt.getDistTo(6)).toBe(1.13)
+      expect(lpt.getDistTo(7)).toBe(0.28)
+    })
+  })
+
+  /**
+     * Check the issue #150
+     **/
+  describe('lightest weight path', function () {
+    it('should find the lightest weight path', function () {
+      const digraph = new EdgeWeightedDigraph()
+
+      digraph.add(1, 3, 0.29)
+      digraph.add(1, 6, 0)
+      digraph.add(3, 6, 0)
+
+      const spt = new SPT(digraph, 1)
+      const path = spt.pathTo(6)
+      expect(path).toEqual([1, 6])
+    })
+    it('should find the lightest weight path', function () {
+      const digraph = new EdgeWeightedDigraph()
+
+      digraph.add(1, 3, -1)
+      digraph.add(1, 6, 0)
+      digraph.add(3, 6, 0)
+
+      const spt = new SPT(digraph, 1)
+      const path = spt.pathTo(6)
+      const hasPath = spt.hasPathTo(6)
+      expect(hasPath).toBe(true)
+      expect(path).toEqual([1, 3, 6])
+    })
+  })
+})

package/ts_spec/soundex_spec.ts

@@ -0,0 +1,162 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { SoundEx } from 'lib/natural'
+const soundex = new SoundEx()
+
+describe('soundex', function () {
+  describe('transformLipps', function () {
+    it('should replace B, F, P, and V with 1', function () {
+      expect(soundex.transformLipps('bopper')).toBe('1o11er')
+      expect(soundex.transformLipps('valumf')).toBe('1alum1')
+    })
+  })
+
+  describe('transformThroats', function () {
+    it('should replace C, G, J, K, Q, S, X, Z with 2', function () {
+      expect(soundex.transformLipps('bopper')).toBe('1o11er')
+      expect(soundex.transformLipps('valumf')).toBe('1alum1')
+    })
+  })
+
+  describe('transformTongue', function () {
+    it('should replace D and T with 3', function () {
+      expect(soundex.transformToungue('dat')).toBe('3a3')
+    })
+  })
+
+  describe('transformL', function () {
+    it('should replace L with 4', function () {
+      expect(soundex.transformL('lala')).toBe('4a4a')
+    })
+  })
+
+  describe('transformHum', function () {
+    it('should replace M and N with 5', function () {
+      expect(soundex.transformHum('mummification')).toBe('5u55ificatio5')
+    })
+  })
+
+  describe('transformR', function () {
+    it('should replace R with 6', function () {
+      expect(soundex.transformR('render')).toBe('6ende6')
+    })
+  })
+
+  describe('condense', function () {
+    it('sound condense sequences', function () {
+      expect(soundex.condense('11222556')).toBe('1256')
+    })
+  })
+
+  describe('padRight0', function () {
+    it('sound padd zeros on the right to a lenght of 3', function () {
+      expect(soundex.padRight0('1')).toBe('100')
+      expect(soundex.padRight0('11')).toBe('110')
+      expect(soundex.padRight0('111')).toBe('111')
+    })
+  })
+
+  it('should not code the first character', function () {
+    expect(soundex.process('render').charAt(0)).toBe('R')
+  })
+
+  it('should pad right with zeros', function () {
+    expect(soundex.process('super')).toBe('S160')
+    expect(soundex.process('butt')).toBe('B300')
+    expect(soundex.process('a')).toBe('A000')
+  })
+
+  it('should pad right with zeros', function () {
+    expect(soundex.process('but')).toBe('B300')
+  })
+
+  it('should perform soundex', function () {
+    expect(soundex.process('BLACKBERRY')).toBe('B421')
+    expect(soundex.process('blackberry')).toBe('B421')
+    expect(soundex.process('calculate')).toBe('C424')
+    expect(soundex.process('CALCULATE')).toBe('C424')
+    expect(soundex.process('fox')).toBe('F200')
+    expect(soundex.process('FOX')).toBe('F200')
+    expect(soundex.process('jump')).toBe('J510')
+    expect(soundex.process('JUMP')).toBe('J510')
+    expect(soundex.process('phonetics')).toBe('P532')
+    expect(soundex.process('PHONETICS')).toBe('P532')
+  })
+
+  it('should perform soundex via compare method', function () {
+    expect(soundex.compare('ant', 'and')).toBeTruthy()
+    expect(soundex.compare('ant', 'anne')).toBeFalsy()
+    expect(soundex.compare('band', 'bant')).toBeTruthy()
+    expect(soundex.compare('band', 'gand')).toBeFalsy()
+  })
+
+  /*
+    it('should perform soundex via attached soundsLike String patch', function() {
+        soundex.attach();
+        expect('ant'.soundsLike('ant')).toBeTruthy();
+        expect('ant'.soundsLike('anne')).toBeFalsy();
+        expect('band'.soundsLike('bant')).toBeTruthy();
+        expect('band'.soundsLike('gand')).toBeFalsy();
+    });
+
+    it('should return string phonetics with string patch', function() {
+        soundex.attach();
+        expect('phonetics'.phonetics()).toBe('P532');
+        expect('PHONETICS'.phonetics()).toBe('P532');
+    });
+
+    it('issue 221 -- inital vowels that duplicate cons codes', function() {
+        soundex.attach();
+        expect('Lloyd'.phonetics()).toBe('L300');
+        expect('Pfister'.phonetics()).toBe('P236');
+
+        expect('manhattan'.phonetics()).toBe('M535');
+        expect('Lukasiewicz'.phonetics()).toBe('L222');
+
+        expect('Gauss'.phonetics()).toBe('G200');
+        expect('Tymczak'.phonetics()).toBe('T522');
+        expect('Jackson'.phonetics()).toBe('J250');
+    });
+
+    it('should tokenize and return string phonetics with string patch', function() {
+        soundex.attach();
+        expect('phonetics'.tokenizeAndPhoneticize()).toEqual(['P532']);
+        expect('phonetics jump'.tokenizeAndPhoneticize()).toEqual(['P532', 'J510']);
+        expect('phonetics jump calculate'.tokenizeAndPhoneticize()).toEqual(['P532', 'J510', 'C424']);
+    });
+    */
+
+  it('should max out at four characters long by default', function () {
+    expect(soundex.process('supercalifragilisticexpialidocious').length).toBe(4)
+  })
+
+  it('should truncate to specified length if maxLength passed', function () {
+    expect(soundex.process('supercalifragilisticexpialidocious', 8).length).toBe(8)
+  })
+
+  it('should handle a maxLength beyond code length', function () {
+    expect(soundex.process('JUMP', 8)).toBe('J510')
+  })
+})

package/ts_spec/spellcheck_spec.ts

@@ -0,0 +1,112 @@
+/*
+Copyright (c) 2011
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { Spellcheck } from 'lib/natural'
+
+describe('spellcheck', function () {
+  describe('distance 1 corrections', function () {
+    const spellcheck = new Spellcheck(['something'])
+
+    it('should fix deletions', function () {
+      expect(spellcheck.getCorrections('smething')).toContain('something')
+    })
+    it('should fix insertions', function () {
+      expect(spellcheck.getCorrections('somethhing')).toContain('something')
+    })
+    it('should fix transpositions', function () {
+      expect(spellcheck.getCorrections('somehting')).toContain('something')
+    })
+    it('should fix replacements', function () {
+      expect(spellcheck.getCorrections('sometzing')).toContain('something')
+    })
+  })
+
+  describe('distance 2 corrections', function () {
+    const spellcheck = new Spellcheck(['something'])
+
+    it('should fix a deletion then a transposition', function () {
+      expect(spellcheck.getCorrections('smtehing', 2)).toContain('something')
+    })
+  })
+
+  describe('edits', function () {
+    const spellcheck = new Spellcheck([])
+    const edits = spellcheck.edits('ab')
+
+    it('should compute insertions', function () {
+      const insertions = ['xab', 'axb', 'abx']
+      insertions.forEach(insertion => {
+        expect(edits.indexOf(insertion)).toBeGreaterThan(-1)
+      })
+    })
+    it('should compute deletions', function () {
+      const deletions = ['a', 'b']
+      deletions.forEach(deletion => {
+        expect(edits.indexOf(deletion)).toBeGreaterThan(-1)
+      })
+    })
+    it('should compute transpositions', function () {
+      const transpositions = ['ba']
+      transpositions.forEach(transposition => {
+        expect(edits.indexOf(transposition)).toBeGreaterThan(-1)
+      })
+    })
+    it('should compute replacements', function () {
+      const replacements = ['zb', 'af']
+      replacements.forEach(replacement => {
+        expect(edits.indexOf(replacement)).toBeGreaterThan(-1)
+      })
+    })
+  })
+
+  describe('edits up to distance', function () {
+    const spellcheck = new Spellcheck([])
+
+    it('should correctly compute edits at distance', function () {
+      const edits = spellcheck.editsWithMaxDistance('abc', 2)
+      expect(edits[1].indexOf('abzc')).toBeGreaterThan(-1) // 1 insertion
+      expect(edits[2].indexOf('a')).toBeGreaterThan(-1) // 2 deletions
+    })
+  })
+
+  describe('boolean spellcheck', function () {
+    const spellcheck = new Spellcheck(['cat'])
+
+    it('should consider cat a word', function () {
+      expect(spellcheck.isCorrect('cat')).toBe(true)
+    })
+
+    it('should not consider dog a word', function () {
+      expect(spellcheck.isCorrect('dog')).toBe(false)
+    })
+  })
+
+  describe('special cases', function () {
+    const spellcheck = new Spellcheck(['cat'])
+
+    it('should return the input word as the most probable correction if it is already correct', function () {
+      expect(spellcheck.getCorrections('cat').indexOf('cat')).toBe(0)
+    })
+  })
+})