natural 6.12.0 → 7.0.0

package/ts_spec/present_verb_inflector_spec.ts

@@ -0,0 +1,127 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+import { PresentVerbInflector } from 'lib/natural'
+const inflector = new PresentVerbInflector()
+
+describe('present verb inflector', function () {
+  describe('singularization', function () {
+    it('should singularize regular ES forms', function () {
+      expect(inflector.singularize('catch')).toBe('catches')
+      expect(inflector.singularize('do')).toBe('does')
+      expect(inflector.singularize('go')).toBe('goes')
+    })
+
+    it('should handle [CS]HES forms', function () {
+      expect(inflector.singularize('cash')).toBe('cashes')
+      expect(inflector.singularize('ach')).toBe('aches')
+    })
+
+    it('should ignore XES forms', function () {
+      expect(inflector.singularize('annex')).toBe('annexes')
+    })
+
+    it('should handle SSES forms', function () {
+      expect(inflector.singularize('access')).toBe('accesses')
+    })
+
+    it('should ignore ZZES forms', function () {
+      expect(inflector.singularize('buzz')).toBe('buzzes')
+    })
+
+    it('should singularize regular S forms', function () {
+      expect(inflector.singularize('claim')).toBe('claims')
+      expect(inflector.singularize('drink')).toBe('drinks')
+      expect(inflector.singularize('become')).toBe('becomes')
+    })
+
+    it('should singularize irregular forms', function () {
+      expect(inflector.singularize('are')).toBe('is')
+      expect(inflector.singularize('were')).toBe('was')
+      expect(inflector.singularize('have')).toBe('has')
+    })
+
+    it('should singularize ies forms', function () {
+      expect(inflector.singularize('fly')).toBe('flies')
+      expect(inflector.singularize('try')).toBe('tries')
+    })
+
+    it('should handle ambiguous forms', function () {
+      expect(inflector.singularize('will')).toBe('will')
+    })
+  })
+
+  describe('pluralization', function () {
+    it('should pluralize regular ES forms', function () {
+      expect(inflector.pluralize('catches')).toBe('catch')
+      expect(inflector.pluralize('does')).toBe('do')
+      expect(inflector.pluralize('goes')).toBe('go')
+    })
+
+    it('should handle [CS]HES forms', function () {
+      expect(inflector.pluralize('cashes')).toBe('cash')
+      expect(inflector.pluralize('aches')).toBe('ach')
+    })
+
+    it('should handle XES forms', function () {
+      expect(inflector.pluralize('annexes')).toBe('annex')
+    })
+
+    it('should handle SSES forms', function () {
+      expect(inflector.pluralize('accesses')).toBe('access')
+    })
+
+    it('should handle ZZES forms', function () {
+      expect(inflector.pluralize('buzzes')).toBe('buzz')
+    })
+
+    it('should pluralize regular S forms that done drop e', function () {
+      expect(inflector.pluralize('becomes')).toBe('become')
+    })
+
+    it('should pluralize regular S forms', function () {
+      expect(inflector.pluralize('drinks')).toBe('drink')
+      expect(inflector.pluralize('claims')).toBe('claim')
+    })
+
+    it('should pluralize irregular forms', function () {
+      expect(inflector.pluralize('was')).toBe('were')
+      expect(inflector.pluralize('is')).toBe('are')
+      expect(inflector.pluralize('am')).toBe('are')
+      expect(inflector.pluralize('has')).toBe('have')
+    })
+
+    it('should pluralize ies forms', function () {
+      expect(inflector.pluralize('flies')).toBe('fly')
+      expect(inflector.pluralize('tries')).toBe('try')
+    })
+
+    it('should handle ambiguous forms', function () {
+      expect(inflector.pluralize('will')).toBe('will')
+    })
+  })
+
+  it('should pluralize and singularize string', function () {
+    expect(inflector.pluralize('becomes')).toBe('become')
+    expect(inflector.singularize('become')).toBe('becomes')
+  })
+})

package/ts_spec/remove_diacritics_spec.ts

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2013, Ken Koch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+import { removeDiacritics } from 'lib/natural'
+
+describe('remove_diacritics', function () {
+  it('should correctly remove diacritics', function () {
+    const originalPhrase = 'piñon ça va über résumé œdipe'
+    expect(removeDiacritics(originalPhrase)).toEqual('pinon ca va uber resume oedipe')
+  })
+})

package/ts_spec/sentence_analyzer_spec.ts

@@ -0,0 +1,293 @@
+/*
+Copyright (c) 2011, Rob Ellis, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+/* eslint-disable no-new */
+import { SentenceAnalyzer } from 'lib/natural'
+import { SenType } from 'lib/natural/analyzers/SenType'
+import type {
+  TaggedSentence,
+  TaggedWord,
+  PunctuationFunction,
+  PunctuationMapping,
+  CallbackFunction
+} from 'lib/natural'
+
+declare type Tests = Array<{
+  token?: string
+  result: string | boolean
+  argument?: string
+  index?: string
+}>
+
+describe('sentence analyzer', function () {
+  function testSentenceAnalyzer (sentenceTags: TaggedWord[], tests: Tests, callback: CallbackFunction): void {
+    new SentenceAnalyzer({ tags: sentenceTags, punct: function () { return [] } }, function (analyzer) {
+      analyzer.part(function (part) {
+        const posTags = part.posObj.tags
+        for (let tagNum = 0; tagNum < posTags.length; tagNum++) {
+          const posTag = posTags[tagNum]
+          tests.forEach(t => {
+            if (posTag.token === t.token && t.argument !== undefined) {
+              expect(posTag[t.argument]).toEqual(t.result)
+            }
+          })
+        }
+        const lastTagNum = posTags.length - 1
+        tests.forEach(t => {
+          if (t.index !== undefined && t.argument !== undefined) {
+            expect(posTags[lastTagNum][t.argument]).toEqual(t.result)
+          }
+        })
+      })
+      callback(analyzer)
+    })
+  }
+
+  it('should determine PP and SP, given a POS', function () {
+    const sentenceTags: TaggedWord[] = [
+      { token: 'The', pos: 'DT' },
+      { token: 'angry', pos: 'JJ' },
+      { token: 'bear', pos: 'NN' },
+      { token: 'chased', pos: 'VB' },
+      { token: 'the', pos: 'DT' },
+      { token: 'frightened', pos: 'JJ' },
+      { token: 'little', pos: 'JJ' },
+      { token: 'squirrel', pos: 'NN' }
+    ]
+
+    const tests: Tests = [
+      {
+        token: 'angry',
+        argument: 'spos',
+        result: 'SP'
+      },
+      {
+        token: 'squirrel',
+        argument: 'spos',
+        result: 'PP'
+      }
+    ]
+
+    testSentenceAnalyzer(sentenceTags, tests, function (analyzer) {
+      expect(analyzer.subjectToString().trim()).toEqual('The angry bear')
+      expect(analyzer.predicateToString().trim()).toEqual('chased the frightened little squirrel')
+      expect(analyzer.toString().trim()).toEqual('The angry bear chased the frightened little squirrel')
+      expect(analyzer.implicitYou()).toEqual(false)
+    })
+  })
+
+  it('should determine PP and SP given a POS that begins with a verb', function () {
+    const sentenceTags = [
+      { token: 'Vote', pos: 'VB' },
+      { token: 'for', pos: 'IN' },
+      { token: 'me', pos: 'PRP' }
+    ]
+
+    const tests: Tests = [
+      {
+        token: 'Vote',
+        argument: 'spos',
+        result: 'PP'
+      },
+      {
+        token: 'me',
+        argument: 'pp',
+        result: true
+      },
+      {
+        index: 'LAST',
+        argument: 'token',
+        result: 'You'
+      },
+      {
+        index: 'LAST',
+        argument: 'pos',
+        result: 'PRP'
+      },
+      {
+        index: 'LAST',
+        argument: 'added',
+        result: true
+      }
+    ]
+
+    testSentenceAnalyzer(sentenceTags, tests, function (analyzer) {
+      expect(analyzer.implicitYou()).toEqual(true)
+    })
+  })
+
+  it('should look for EX before VB', function () {
+    const sentenceTags = [
+      { token: 'There', pos: 'EX' },
+      { token: 'is', pos: 'VB' },
+      { token: 'a', pos: 'DT' },
+      { token: 'house', pos: 'NN' },
+      { token: 'in', pos: 'IN' },
+      { token: 'the', pos: 'DT' },
+      { token: 'valley', pos: 'DT' }
+    ]
+
+    const tests = [
+      {
+        token: 'There',
+        argument: 'spos',
+        result: 'SP'
+      },
+      {
+        token: 'is',
+        argument: 'spos',
+        result: 'SP'
+      }
+    ]
+
+    testSentenceAnalyzer(sentenceTags, tests, function (analyzer) {
+      expect(analyzer.subjectToString().trim()).toEqual('There is a house')
+      expect(analyzer.predicateToString().trim()).toEqual('')
+      expect(analyzer.toString().trim()).toEqual('There is a house in the valley')
+      expect(analyzer.implicitYou()).toEqual(false)
+    })
+  })
+
+  function testSentenceType (args: TaggedSentence, sentenceType: SenType, callback: CallbackFunction | null): void {
+    new SentenceAnalyzer(args, function (analyzer) {
+      analyzer.part(function () {
+        analyzer.type(function (type) {
+          expect(analyzer.senType).toEqual(sentenceType)
+        })
+      })
+      if (callback !== null) {
+        callback(analyzer)
+      }
+    })
+  }
+
+  describe('#type', function () {
+    it('should determine a command without punctuation', function () {
+      const sentenceTags = [
+        { token: 'Vote', pos: 'VB' },
+        { token: 'for', pos: 'IN' },
+        { token: 'me', pos: 'PRP' }
+      ]
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return []
+      }
+
+      testSentenceType({ tags: sentenceTags, punct }, SenType.Command, null)
+    })
+
+    it('should determine an interrogative beginning with who', function () {
+      const sentenceTags = [
+        { token: 'Who', pos: 'WP' },
+        { token: 'voted', pos: 'VB' }
+      ]
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return []
+      }
+
+      testSentenceType({ tags: sentenceTags, punct }, SenType.Interrogative, null)
+    })
+
+    it('should determine an interrogative ending with a personal pronoun', function () {
+      const sentenceTags = [
+        { token: 'Should', pos: 'MD' },
+        { token: 'we', pos: 'PRP' }
+      ]
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return ''
+      }
+
+      testSentenceType({ tags: sentenceTags, punct }, SenType.Interrogative, null)
+    })
+
+    it('should classify other sentences as unknown', function () {
+      const sentenceTags = [
+        { token: 'I', pos: 'PRP' },
+        { token: 'am', pos: 'VB' },
+        { token: 'unsure', pos: 'JJ' }
+      ]
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return ''
+      }
+
+      testSentenceType({ tags: sentenceTags, punct }, SenType.Unknown, null)
+    })
+
+    it('should determine an interrogative ending with a question mark', function () {
+      const sentenceTags = [
+        { token: 'Do', pos: 'VB' },
+        { token: 'I', pos: 'PRP' },
+        { token: 'care', pos: 'VB' }
+      ]
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return [{ token: '?', pos: '.' }]
+      }
+
+      testSentenceType({ tags: sentenceTags, punct }, SenType.Interrogative, null)
+    })
+
+    const taggedSentForCommand = [
+      { token: 'Vote', pos: 'VB' },
+      { token: 'for', pos: 'IN' },
+      { token: 'me', pos: 'PRP' }
+    ]
+
+    it('should determine a command ending in an exclamation point', function () {
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return [{ token: '!', pos: '.' }]
+      }
+
+      testSentenceType({ tags: taggedSentForCommand, punct }, SenType.Command, null)
+    })
+
+    it('should determine a command ending with .', function () {
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return [{ token: '.', pos: '.' }]
+      }
+
+      testSentenceType({ tags: taggedSentForCommand, punct }, SenType.Command, null)
+    })
+
+    const taggedSentenceForExclam = [
+      { token: 'We', pos: 'PRP' },
+      { token: 'like', pos: 'VB' },
+      { token: 'sheep', pos: 'NN' }
+    ]
+
+    it('should determine an exclamation ending in an exclamation point', function () {
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return [{ token: '!', pos: '.' }]
+      }
+
+      testSentenceType({ tags: taggedSentenceForExclam, punct }, SenType.Exclamatory, null)
+    })
+
+    it('should determine a declaration ending with a .', function () {
+      const punct: PunctuationFunction = function (): PunctuationMapping {
+        return [{ token: '.', pos: '.' }]
+      }
+
+      testSentenceType({ tags: taggedSentenceForExclam, punct }, SenType.Declarative, null)
+    })
+  })
+})

package/ts_spec/sentence_tokenizer_parser_spec.ts

@@ -0,0 +1,200 @@
+/*
+Copyright (c) 2011, 2020 Chris Umbel, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { SentenceTokenizerNew as Tokenizer } from 'lib/natural'
+const tokenizer = new Tokenizer()
+
+describe('sentence_tokenizer', function () {
+  it('should tokenize strings and trim whitespace', function () {
+    expect(
+      tokenizer.tokenize('This is a sentence. This is another sentence.')
+    )
+      .toEqual([
+        'This is a sentence.',
+        'This is another sentence.'
+      ])
+  })
+
+  /*
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect(
+        'This is a sentence. This is another sentence.'.tokenize()
+    ).toEqual([
+        'This is a sentence.',
+        'This is another sentence.'
+    ]);
+  });
+
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect(
+        'This is a sentence. This is another sentence.'.tokenize()
+    ).toEqual([
+        'This is a sentence.',
+        'This is another sentence.'
+    ]);
+  });
+  */
+
+  it('should include quotation marks', function () {
+    expect(
+      tokenizer.tokenize('"This is a sentence." This is another sentence.')
+    ).toEqual([
+      '"This is a sentence."',
+      'This is another sentence.'
+    ])
+  })
+
+  it('should include brackets', function () {
+    expect(
+      tokenizer.tokenize('This is a sentence. [This is another sentence.]')
+    ).toEqual([
+      'This is a sentence.',
+      '[This is another sentence.]'
+    ])
+  })
+
+  it('should handle repetitive punctuation', function () {
+    expect(
+      tokenizer.tokenize('I love you!! Do you love me??')
+    ).toEqual(['I love you!!', 'Do you love me??'
+    ])
+  })
+
+  it('should handle repetitive punctuation with space', function () {
+    expect(
+      tokenizer.tokenize('I love you! ! Do you love me? ?')
+    ).toEqual(['I love you! !', 'Do you love me? ?'
+    ])
+  })
+
+  it('should handle decimal numbers in sentences', function () {
+    expect(
+      tokenizer.tokenize('Pi is approximately equal to 3.14.')
+    ).toEqual([
+      'Pi is approximately equal to 3.14.'
+    ])
+  })
+
+  it('should tokenize text with a number like "1)" present ', function () {
+    expect(
+      tokenizer.tokenize("This is a sentence that can't 1) be parsed with SentenceTokenizerNew. Here is another sentence.")
+    ).toEqual([
+      "This is a sentence that can't 1) be parsed with SentenceTokenizerNew.",
+      'Here is another sentence.'
+    ])
+  })
+
+  it('should handle periods in email addresses', function () {
+    expect(
+      tokenizer.tokenize('My email address is batman@example.com.')
+    ).toEqual([
+      'My email address is batman@example.com.'
+    ])
+  })
+
+  it('should handle periods in web addresses', function () {
+    expect(
+      tokenizer.tokenize('My twitter feed can be found at https://twitter.com/user1.')
+    ).toEqual([
+      'My twitter feed can be found at https://twitter.com/user1.'
+    ])
+  })
+
+  it('should handle an ellipsis followed by punctuation', function () {
+    expect(
+      tokenizer.tokenize('Is this the end for our heroes...?')
+    ).toEqual([
+      'Is this the end for our heroes...?'
+    ])
+  })
+
+  it('should handle multiple spaces separating sentences', function () {
+    expect(
+      tokenizer.tokenize('Tune in tomorrow and find out!  Same Bat-Time!  Same Bat-Channel!')
+    ).toEqual([
+      'Tune in tomorrow and find out!',
+      'Same Bat-Time!',
+      'Same Bat-Channel!'
+    ])
+  })
+
+  it('should handle a sentence that does not end with punctuation (issue #549))', function () {
+    expect(
+      tokenizer.tokenize('This is a sentence. But is this also one')
+    ).toEqual([
+      'This is a sentence.',
+      'But is this also one'
+    ])
+  })
+
+  it('should handle a sentence that contains a quoted phrase (issue #550 but with . and ’ reversed))', function () {
+    expect(
+      tokenizer.tokenize('This is a sentence. And another where ‘Someone says something’.')
+    ).toEqual([
+      'This is a sentence.',
+      'And another where ‘Someone says something’.'
+    ])
+  })
+
+  it('should handle sentences with an abbreviation', function () {
+    expect(
+      tokenizer.tokenize('Acme, Inc. is creating exciting products. Use at your own risk.')
+    ).toEqual([
+      'Acme, Inc. is creating exciting products.',
+      'Use at your own risk.'
+    ])
+    expect(
+      tokenizer.tokenize('I need the parts A.S.A.P. please. Send them when they are ready')
+    ).toEqual(
+      [
+        'I need the parts A.S.A.P. please.',
+        'Send them when they are ready'
+      ]
+    )
+    expect(
+      tokenizer.tokenize('I need the parts from Inc.. Send them when they are ready!')
+    ).toEqual(
+      [
+        'I need the parts from Inc..',
+        'Send them when they are ready!'
+      ]
+    )
+  })
+  it('Should correctly parse multiple sentences with a subset of the sentences surrounded with quotes', function () {
+    expect(
+      tokenizer.tokenize('"All ticketed passengers should now be in the Blue Concourse sleep lounge. Make sure your validation papers are in order. Thank you." The upstairs lounge was not at all grungy.')
+    ).toEqual(
+      [
+        '"',
+        'All ticketed passengers should now be in the Blue Concourse sleep lounge.',
+        'Make sure your validation papers are in order.',
+        'Thank you.',
+        '"',
+        'The upstairs lounge was not at all grungy.'
+      ]
+    )
+  })
+})