natural 6.12.0 → 7.0.0

package/ts_spec/transliterator_ja_spec.ts

@@ -0,0 +1,105 @@
+/*
+Copyright (c) 2012, Guillaume Marty
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { TransliterateJa as transliterateJa } from 'lib/natural'
+
+describe('transliterateJa', function () {
+  it('should transliterate hiragana', function () {
+    expect(transliterateJa('あいうえお かきくけこ')).toEqual('aiueo kakikukeko')
+    expect(transliterateJa('さしすせそ たちつてと')).toEqual('sashisuseso tachitsuteto')
+  })
+
+  it('should transliterate katakana', function () {
+    expect(transliterateJa('アイウエオ カキクケコ')).toEqual('aiueo kakikukeko')
+    expect(transliterateJa('サシスセソ タチツテト')).toEqual('sashisuseso tachitsuteto')
+  })
+
+  it('should transliterate small tsu differently depending on context', function () {
+    expect(transliterateJa('まっか ざっし たった はっぱ')).toEqual('makka zasshi tatta happa')
+    expect(transliterateJa('マッカ ザッシ タッタ ハッパ')).toEqual('makka zasshi tatta happa')
+  })
+
+  it('should transliterate final small tsu', function () {
+    expect(transliterateJa('ああっ')).toEqual('aat')
+    expect(transliterateJa('アアッ')).toEqual('aat')
+  })
+
+  it('should transliterate n differently depending on context', function () {
+    expect(transliterateJa('まんと ばんび ほんや')).toEqual("manto bambi hon'ya")
+    expect(transliterateJa('マント バンビ ホンヤ')).toEqual("manto bambi hon'ya")
+  })
+
+  it('should transliterate long vowels', function () {
+    expect(transliterateJa('ああ　いい　うう　ええ　おお　おう')).toEqual('aa　ii　ū　ee　oo　ō')
+    expect(transliterateJa('あー　いー　うー　えー　おー')).toEqual('ā　ī　ū　ē　ō')
+    expect(transliterateJa('アア　イイ　ウウ　エエ　オオ　オウ')).toEqual('aa　ii　ū　ee　oo　ō')
+    expect(transliterateJa('アー　イー　ウー　エー　オー')).toEqual('ā　ī　ū　ē　ō')
+  })
+
+  it('should leave non fullwidth kana unchanged', function () {
+    expect(transliterateJa('abc ABC 漢字 (.)')).toEqual('abc ABC 漢字 (.)')
+    expect(transliterateJa('ｱｲｳｴｵ ｶｷｸｹｺ')).toEqual('ｱｲｳｴｵ ｶｷｸｹｺ')
+  })
+
+  it('should transliterate misc. words correctly', function () {
+    expect(transliterateJa('アヴァンギャルド')).toEqual('avangyarudo') // Avant-garde
+    expect(transliterateJa('アクサン・スィルコンフレックス')).toEqual('akusan sirukonfurekkusu') // Accent circonflexe
+    expect(transliterateJa('アドレシッング')).toEqual('adoreshinngu') // Addressing
+    expect(transliterateJa('イェス')).toEqual('yesu') // Yes
+    expect(transliterateJa('インテリジェンス')).toEqual('interijensu') // Intelligence
+    expect(transliterateJa('インテルメッツォ')).toEqual('interumettso') // Intermezzo
+    expect(transliterateJa('グァテマラ')).toEqual('gwatemara') // Guatemala
+    expect(transliterateJa('クァルテット')).toEqual('kwarutetto') // Quartet
+    expect(transliterateJa('クィンテット')).toEqual('kwintetto') // Quintet
+    expect(transliterateJa('クォーター')).toEqual('kwōtā') // Quarter
+    expect(transliterateJa('サブウーファー')).toEqual('sabuūfā') // Sub woofer
+    expect(transliterateJa('ソフトウェア')).toEqual('sofutowea') // Software
+    expect(transliterateJa('ツィーター')).toEqual('tsītā') // Tweeter
+    expect(transliterateJa('デューティー')).toEqual('dyūtī') // Duty
+    expect(transliterateJa('ドキュメントィンドウ')).toEqual('dokyumentwindō') // Document window
+    expect(transliterateJa('ヌーヴェルヴァーグ')).toEqual('nūveruvāgu') // Nouvelle vague
+    expect(transliterateJa('ハイジャッンプ')).toEqual('haijanmpu') // High jump
+    expect(transliterateJa('バッファ')).toEqual('baffa') // Buffer
+    expect(transliterateJa('フーホェア')).toEqual('fūhwea') // WhoWhere?
+    expect(transliterateJa('フェイズィング')).toEqual('feizingu') // Phasing
+    expect(transliterateJa('フッロピー')).toEqual('furropī') // Floppy (alternative transcription)
+    expect(transliterateJa('ブュー')).toEqual('byū') // View
+    expect(transliterateJa('フューチャー')).toEqual('fyūchā') // Future
+    expect(transliterateJa('フロッピィ')).toEqual('furoppī') // Floppy
+    expect(transliterateJa('ボージョレー・ヌーヴォー')).toEqual('bōjorē nūvō') // Beaujolais nouveau
+    expect(transliterateJa('ボスニア・ヘルツェゴビナ')).toEqual('bosunia herutsegobina') // Bosnia and Herzegovina
+    expect(transliterateJa('マッハ')).toEqual('mahha') // Mach
+    expect(transliterateJa('レヴュー')).toEqual('revyū') // Review
+    expect(transliterateJa('レクリェーション')).toEqual('rekuryēshon') // Recreation
+  })
+
+  it('should use fallback for small vowels on special case', function () {
+    // These words use rare combination of small vowels.
+    expect(transliterateJa('アゲィンスト')).toEqual('ageinsuto') // Against (alternative transcription)
+    expect(transliterateJa('エッセィ')).toEqual('essei') // Essay (alternative transcription)
+    expect(transliterateJa('ゾゥアラジィ')).toEqual('zouarajī') // Zoology (alternative transcription)
+    expect(transliterateJa('ゾゥァラジカル')).toEqual('zouarajikaru') // Zoological (alternative transcription)
+    expect(transliterateJa('ボランテァ')).toEqual('borantea') // Volunteer (alternative transcription)
+  })
+})

package/ts_spec/treebank_word_tokenizer_spec.ts

@@ -0,0 +1,38 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { TreebankWordTokenizer as Tokenizer } from 'lib/natural'
+const tokenizer = new Tokenizer()
+
+describe('treebank', function () {
+  it('should tokenize', function () {
+    let tokens = tokenizer.tokenize("If we 'all' can't go. I'll stay home.")
+    expect(tokens).toEqual(['If', 'we', "'all", "'", 'ca', "n't", 'go.',
+      'I', "'ll", 'stay', 'home', '.'])
+
+    tokens = tokenizer.tokenize("If we 'all' can't go. I'll stay home. If we 'all' can't go. I'll stay home.")
+    expect(tokens).toEqual(['If', 'we', "'all", "'", 'ca', "n't", 'go.',
+      'I', "'ll", 'stay', 'home.', 'If', 'we', "'all", "'", 'ca', "n't", 'go.', 'I', "'ll", 'stay', 'home', '.'])
+  })
+})

package/ts_spec/trie_spec.ts

@@ -0,0 +1,197 @@
+/*
+Copyright (c) 2014 Ken Koch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+import { Trie } from 'lib/natural'
+
+describe('trie', function () {
+  describe('adding strings', function () {
+    it('should add words one at a time', function () {
+      const trie = new Trie()
+      trie.addString('test')
+      expect(trie.contains('test')).toBe(true)
+
+      trie.addString('abcd')
+      expect(trie.contains('abcd')).toBe(true)
+    })
+
+    it('should return true if a string is added that already existed', function () {
+      const trie = new Trie()
+      expect(trie.addString('test')).toBe(false)
+      expect(trie.addString('test')).toBe(true)
+    })
+
+    it('Should add an array of strings', function () {
+      const trie = new Trie()
+      const testWords = ['test', 'abcd', 'ffff']
+      trie.addStrings(testWords)
+
+      for (let i = testWords.length - 1; i >= 0; i--) {
+        expect(trie.contains(testWords[i])).toBe(true)
+      };
+    })
+  })
+
+  describe('getSize', function () {
+    it('should return 1 for an empty trie', function () {
+      const trie = new Trie()
+      expect(trie.getSize()).toBe(1)
+    })
+
+    it('should return the correct size', function () {
+      const trie = new Trie()
+      trie.addString('a')
+      expect(trie.getSize()).toBe(2)
+
+      trie.addString('ab')
+      expect(trie.getSize()).toBe(3)
+    })
+
+    it('should count all branches', function () {
+      const trie = new Trie()
+      trie.addString('a')
+      expect(trie.getSize()).toBe(2)
+
+      trie.addString('ba')
+      expect(trie.getSize()).toBe(4)
+    })
+  })
+
+  describe('searching', function () {
+    it("should not find words that haven't been added", function () {
+      const trie = new Trie()
+      expect(trie.contains('aaaaa')).toBe(false)
+    })
+
+    it('should not return prefixes of words that have been added as words', function () {
+      const trie = new Trie()
+      trie.addString('test')
+      expect(trie.contains('test')).toBe(true)
+      expect(trie.contains('tes')).toBe(false)
+    })
+
+    it('should not return suffixes of words that have been added as words', function () {
+      const trie = new Trie()
+      trie.addString('test')
+      expect(trie.contains('test')).toBe(true)
+      expect(trie.contains('est')).toBe(false)
+    })
+
+    it('should not find a word that falls in between two other words but has not been added', function () {
+      const trie = new Trie()
+      trie.addString('test')
+      trie.addString('tested')
+      expect(trie.contains('test')).toBe(true)
+      expect(trie.contains('tested')).toBe(true)
+      expect(trie.contains('teste')).toBe(false)
+    })
+  })
+
+  function expectResults (results: string[]): void {
+    expect(results).toContain('a')
+    expect(results).toContain('ab')
+    expect(results).toContain('abc')
+    expect(results).not.toContain('bc')
+    expect(results).not.toContain('cd')
+  }
+
+  describe('prefix searching', function () {
+    it('should be able to find all full prefix matched words along a path.', function () {
+      const trie = new Trie()
+      trie.addStrings(['a', 'ab', 'bc', 'cd', 'abc'])
+      const results = trie.findMatchesOnPath('abcd')
+      expectResults(results)
+    })
+
+    it('should be able to guess all full prefix matched words.', function () {
+      const trie = new Trie()
+      trie.addStrings(['a', 'ab', 'bc', 'cd', 'abc'])
+      let results = trie.keysWithPrefix('a')
+      expectResults(results)
+      results = trie.keysWithPrefix('ab')
+      expect(results).toContain('ab')
+      expect(results).toContain('abc')
+      expect(results).not.toContain('a')
+      expect(results).not.toContain('bc')
+      expect(results).not.toContain('cd')
+    })
+
+    it('should be able to execute the find_prefix search with a match', function () {
+      const trie = new Trie()
+      trie.addStrings(['their', 'and', 'they'])
+      const results = trie.findPrefix('theyre')
+      expect(results[0]).toBe('they')
+      expect(results[1]).toBe('re')
+    })
+
+    it('should return empty array if no full prefix matches found.', function () {
+      const trie = new Trie()
+      trie.addStrings(['a', 'ab', 'bc', 'cd', 'abc'])
+      const results = trie.keysWithPrefix('not-found')
+      expect(results.length).toEqual(0)
+    })
+
+    it('should be able to execute the find_prefix search without a match', function () {
+      const trie = new Trie()
+      trie.addStrings(['their', 'and'])
+      const results = trie.findPrefix('theyre')
+      expect(results[0]).toBe(null)
+      expect(results[1]).toBe('yre')
+    })
+  })
+
+  describe('case sensitivity', function () {
+    it('should be case sensitive by default', function () {
+      const trie = new Trie()
+      trie.addString('test')
+      expect(trie.contains('TEST')).toBe(false)
+    })
+
+    it('should have contains in case-insensitive mode', function () {
+      const trie = new Trie(false)
+      trie.addString('test')
+      expect(trie.contains('TEST')).toBe(true)
+    })
+
+    it('should have case-insensitive contains when the strings are added with case', function () {
+      const trie = new Trie(false)
+      trie.addString('teSt')
+      expect(trie.contains('test')).toBe(true)
+    })
+
+    it('should have findMatchesOnPath in case-insensitive mode', function () {
+      const trie = new Trie(false)
+      trie.addStrings(['a', 'ab', 'bc', 'cd', 'abc'])
+      const results = trie.findMatchesOnPath('ABcD')
+      expectResults(results)
+    })
+
+    it('should have findPrefix in case-insensitive mode', function () {
+      const trie = new Trie(false)
+      trie.addStrings(['thEIr', 'And', 'theY'])
+      const results = trie.findPrefix('ThEyRe')
+      expect(results[0]).toBe('they')
+      expect(results[1]).toBe('re')
+    })
+  })
+})

package/tsconfig.json

@@ -7,6 +7,7 @@
     ],
     "module": "commonjs",
     "rootDir": "./",
+    "baseUrl": "./",
     "moduleResolution": "node",
     "allowJs": true,
     "outDir": "./dist",
@@ -15,9 +16,10 @@
     "strict": true
   },
   "include": [
-    "lib/natural/**/*",
-    "spec/*"
-, "io_spec/StorageBackend_spec.js"  ],
+    "lib/natural/**/*.ts",
+    "ts_spec/*.ts",
+    "io_spec/*.ts"
+, "lib/natural/analyzers/SenType.ts"  ],
   "exclude": [
     "./node_modules"
   ]

package/example.txt

@@ -1 +0,0 @@
-{"attr1":"val1","attr2":"val2"}