suika 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41135bca7252d6e52d8c13c52ca08446ebe79ffe388f585a1739976e5b26ad4f
-  data.tar.gz: 22e021383f5169c264392cf4e1c4cd185319f810d86c2c8e85000b80ac493ccd
+  metadata.gz: bfb2e78e15c648ee309868bdfd3f386a66b1cff633cb546880132cdb9b8f3806
+  data.tar.gz: d398f4de11a4af80b7c62c4e468fa2e3f9393bbb2211e0f2e317a08ed05c73b5
 SHA512:
-  metadata.gz: 534e3c4c67612fcf0c052057dbb85b649cba037517c9c500c2a3e20e8f0cae97e734b71fcce7ff47ba1a55d4b99fd5fcdab0be26f14addddc3728c784d1e8966
-  data.tar.gz: 8f929702297098e8c3439170323205e9c58f6f5f9735192604bd08566f4f20c8ae1d603185126636097846df45e45c6e7141380fea3e2b449bea3e6c73bd983d
+  metadata.gz: 6fea777a4229725a174aa0955bcf5f775cc6e319fa73cf6f204385b3a47d84997b74b8798f78269f442aa11dcf151b1319c079c315dfea103dcd7a148cf0b5c5
+  data.tar.gz: 17d2fd7f248c965b6d585542c4ec4cb90e87663f96b0522f06f1c3e94c55a18e59f08e4ab1e7724f9dd48a114c2c0f35a0499100fb8850a89d2822de87927988

data/.rubocop.yml

@@ -3,7 +3,8 @@ require:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.4
+  NewCops: enable
+  TargetRubyVersion: 2.5
   DisplayCopNames: true
   DisplayStyleGuide: true
   Exclude:

data/.travis.yml

@@ -1,6 +1,12 @@
 ---
+os: linux
+dist: xenial
 language: ruby
 cache: bundler
 rvm:
-  - 2.7.0
-before_install: gem install bundler -v 2.1.2
+  - '2.5'
+  - '2.6'
+  - '2.7'
+
+before_install:
+  - gem install bundler -v 2.1.4

data/CHANGELOG.md

@@ -0,0 +1,37 @@
+## 0.2.0
+
+### Breaking Change
+- Change to use dartsclone for trie library.
+
+
+## 0.1.4
+
+### Bug Fixes
+- Fix CharDef.char_type to return 'DEFAULT' when unknown character code is given.
+
+### Features
+- Add character code of square era name Reiwa.
+
+## 0.1.3
+
+### Bug Fixes
+- Fix unknown word processing.
+
+### Changes
+- Remove redundant spaces from output.
+
+
+## 0.1.2
+
+### Bug Fixes
+- Fix local variable typo in Tagger.parse.
+
+
+## 0.1.1
+
+### Bug Fixes
+- Fix specification of class in CharDef.char_type.
+
+
+## 0.1.0
+- First release.

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    suika (0.2.0)
+      dartsclone (>= 0.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    dartsclone (0.2.0)
+    diff-lcs (1.4.4)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  suika!
+
+BUNDLED WITH
+   2.1.2

data/README.md

@@ -1,5 +1,6 @@
 # Suika
 
+[![Build Status](https://travis-ci.org/yoshoku/suika.svg?branch=master)](https://travis-ci.org/yoshoku/suika)
 [![Gem Version](https://badge.fury.io/rb/suika.svg)](https://badge.fury.io/rb/suika)
 [![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/suika/blob/master/LICENSE.txt)
 [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://rubydoc.info/gems/suika)
@@ -30,13 +31,13 @@ require 'suika'
 tagger = Suika::Tagger.new
 tagger.parse('すもももももももものうち').each { |token| puts token }
 
-# すもも  名詞, 一般, *, *, *, *, すもも, スモモ, スモモ
-# も      助詞, 係助詞, *, *, *, *, も, モ, モ
-# もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-# も      助詞, 係助詞, *, *, *, *, も, モ, モ
-# もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-# の      助詞, 連体化, *, *, *, *, の, ノ, ノ
-# うち    名詞, 非自立, 副詞可能, *, *, *, うち, ウチ, ウチ
+# すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+# も      助詞,係助詞,*,*,*,*,も,モ,モ
+# もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+# も      助詞,係助詞,*,*,*,*,も,モ,モ
+# もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+# の      助詞,連体化,*,*,*,*,の,ノ,ノ
+# うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
 ```
 
 Since the Tagger class loads the binary dictionary at initialization, it is recommended to reuse the instance.
@@ -51,6 +52,25 @@ sentences.each do |sentence|
 end
 ```
 
+## Test
+Suika was able to parse all sentences in the [Livedoor news corpus](https://www.rondhuit.com/download.html#ldcc)
+without any error.
+
+```ruby
+require 'suika'
+
+tagger = Suika::Tagger.new
+
+Dir.glob('ldcc-20140209/text/*/*.txt').each do |filename|
+  File.foreach(filename) do |sentence|
+    sentence.strip!
+    puts tagger.parse(sentence) unless sentence.empty?
+  end
+end
+```
+
+![suika_test](https://user-images.githubusercontent.com/5562409/90264778-8f593f80-de8c-11ea-81f1-20831e3c8b12.gif)
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yoshoku/suika.

data/lib/suika.rb

@@ -2,5 +2,6 @@
 
 require 'suika/version'
 require 'suika/char_def'
+require 'suika/node'
 require 'suika/lattice'
 require 'suika/tagger'

data/lib/suika/char_def.rb

@@ -5,10 +5,11 @@ module Suika
   class CharDef
     # @!visibility private
     def self.char_type(ch)
-      code = ch.unpack1('U*')
-      CHAR_TYPES.find do |ctype|
-        Object.const_get("CharDef::#{ctype}").any? { |r| r.include?(code) }
+      ch_code = ch.unpack1('U*')
+      ch_type = CHAR_TYPES.find do |ct|
+        Object.const_get("::Suika::CharDef::#{ct}").any? { |r| r.include?(ch_code) }
       end
+      ch_type || 'DEFAULT'
     end
 
     # @!visibility private
@@ -16,39 +17,41 @@ module Suika
       CHAR_CATEGORY[char_type(ch)]
     end
 
+    MAX_GROUPING_SIZE = 24
+
     CHAR_CATEGORY = {
       'DEFAULT' => {
-        invoke: 0, group: 1, length: 0
+        invoke: false, group: true, length: 0
       },
       'SPACE' => {
-        invoke: 0, group: 1, length: 0
+        invoke: false, group: true, length: 0
       },
       'KANJI' => {
-        invoke: 0, group: 0, length: 2
+        invoke: false, group: false, length: 2
       },
       'SYMBOL' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'NUMERIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'ALPHA' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'HIRAGANA' => {
-        invoke: 0, group: 1, length: 2
+        invoke: false, group: true, length: 2
       },
       'KATAKANA' => {
-        invoke: 1, group: 1, length: 2
+        invoke: true, group: true, length: 2
       },
       'KANJINUMERIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'GREEK' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'CYRILLIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       }
     }.freeze
 
@@ -117,6 +120,7 @@ module Suika
       0x2B00..0x2BFF,  # Miscellaneous Symbols and Arrows
       0x2A00..0x2AFF,  # Supplemental Mathematical Operators
       0x3300..0x33FF,
+      0x32FF..0x32FF,  # Square era name REIWA
       0x3200..0x32FE,  # ENclosed CJK Letters and Months
       0x3000..0x303F,  # CJK Symbol and Punctuation
       0xFE30..0xFE4F,  # CJK Compatibility Forms
@@ -171,8 +175,6 @@ module Suika
       0xF900..0xFA2D,
       0xFA30..0xFA6A
     ].freeze
-
-    # rubocop:disable Style/AsciiComments
     # KANJI-NUMERIC （一 二 三 四 五 六 七 八 九 十 百 千 万 億 兆）
     # 0x4E00 KANJINUMERIC KANJI
     KANJINUMERIC = [
@@ -192,7 +194,6 @@ module Suika
       0x5104..0x5104,
       0x5146..0x5146
     ].freeze
-    # rubocop:enable Style/AsciiComments
 
     private_constant :CHAR_CATEGORY, :CHAR_TYPES
 

data/lib/suika/lattice.rb

@@ -4,8 +4,6 @@ module Suika
   # @!visibility private
   class Lattice
     # @!visibility private
-    Node = Struct.new(:surface, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs, keyword_init: true)
-
     attr_reader :begin_nodes, :end_nodes, :length
 
     # @!visibility private
@@ -13,17 +11,17 @@ module Suika
       @length = length
       @begin_nodes = Array.new(length + 1) { [] }
       @end_nodes = Array.new(length + 1) { [] }
-      bos = Node.new(surface: 'BOS', left_id: 0, right_id: 0, cost: 0, attrs: [])
-      @end_nodes[0].append(bos)
-      eos = Node.new(surface: 'EOS', left_id: 0, right_id: 0, cost: 0, attrs: [])
-      @begin_nodes[length].append(eos)
+      bos = Node.new(surface: 'BOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @end_nodes[0].push(bos)
+      eos = Node.new(surface: 'EOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @begin_nodes[length].push(eos)
     end
 
     # @!visibility private
-    def insert(begin_id, end_id, surface, left_id, right_id, cost, attrs)
-      node = Node.new(surface: surface, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
-      @begin_nodes[begin_id].append(node)
-      @end_nodes[end_id].append(node)
+    def insert(begin_id, end_id, surface, unknown, left_id, right_id, cost, attrs)
+      node = Node.new(surface: surface, unknown: unknown, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
+      @begin_nodes[begin_id].push(node)
+      @end_nodes[end_id].push(node)
     end
   end
 end

data/lib/suika/node.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Suika
+  # @!visibility private
+  class Node
+    # @!visibility private
+    attr_accessor :surface, :unknown, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs
+
+    # @!visibility private
+    def initialize(surface: '', unknown: false, min_cost: 0, min_prev: nil, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @surface = surface
+      @unknown = unknown
+      @min_cost = min_cost
+      @min_prev = min_prev
+      @left_id = left_id
+      @right_id = right_id
+      @cost = cost
+      @attrs = attrs
+    end
+  end
+end

data/lib/suika/tagger.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require 'rambling-trie'
+require 'dartsclone'
+require 'rubygems/package'
 require 'zlib'
 
 module Suika
@@ -12,22 +13,22 @@ module Suika
   #   tagger = Suika::Tagger.new
   #   tagger.parse('すもももももももものうち').each { |token| puts token }
   #
-  #   # すもも  名詞, 一般, *, *, *, *, すもも, スモモ, スモモ
-  #   # も      助詞, 係助詞, *, *, *, *, も, モ, モ
-  #   # もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-  #   # も      助詞, 係助詞, *, *, *, *, も, モ, モ
-  #   # もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-  #   # の      助詞, 連体化, *, *, *, *, の, ノ, ノ
-  #   # うち    名詞, 非自立, 副詞可能, *, *, *, うち, ウチ, ウチ
+  #   # すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+  #   # も      助詞,係助詞,*,*,*,*,も,モ,モ
+  #   # もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+  #   # も      助詞,係助詞,*,*,*,*,も,モ,モ
+  #   # もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+  #   # の      助詞,連体化,*,*,*,*,の,ノ,ノ
+  #   # うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
   #
   class Tagger
     # Create a new tagger by loading the built-in binary dictionary.
     def initialize
-      ipadic = Marshal.load(Zlib::GzipReader.open(__dir__ + '/../../dict/ipadic.gz', &:read))
-      @trie = ipadic[:trie]
-      @dictionary = ipadic[:dictionary]
-      @unknown_dictionary = ipadic[:unknown_dictionary]
-      @cost_mat = ipadic[:cost_matrix]
+      raise IOError, 'SHA1 digest of dictionary file does not match.' unless DICTIONARY_KEY == Digest::SHA1.file(DICTIONARY_PATH).to_s
+
+      @sysdic = Marshal.load(Zlib::GzipReader.open(DICTIONARY_PATH, &:read))
+      @trie = DartsClone::DoubleArray.new
+      @trie.set_array(@sysdic[:trie])
     end
 
     # Parse the given sentence.
@@ -39,44 +40,40 @@ module Suika
       terminal = sentence.length
 
       while start < terminal
-        word = sentence[start]
-        pos = start
-        is_unknown = true
-        while @trie.match?(word) && pos < terminal
-          if @dictionary.key?(word)
-            @dictionary[word].each do |el|
-              lattice.insert(start, start + word.length,
-                             word, el[0].to_i, el[1].to_i, el[2].to_i,
-                             el[3..-1])
+        step = terminal - start
+
+        query = sentence[start..-1]
+        result = trie.common_prefix_search(query)
+        unless result.empty?
+          words, indices = result
+          words.each_with_index do |word, i|
+            features[indices[i]].each do |el|
+              lattice.insert(start, start + word.length, word, false,
+                             el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
             end
-            is_unknown = false
           end
-          pos += 1
-          word = sentence[start..pos]
-        end
-
-        unless is_unknown
-          start += 1
-          next
+          step = words.map(&:size).min
         end
 
         word = sentence[start]
-        char_type = CharDef.char_type(sentence[start])
         char_cate = CharDef.char_category(sentence[start])
-        if char_cate[:group] == 1
-          unk_terminal = char_cate[:length].zero? ? terminal : start + char_cate[:length]
+        char_type = CharDef.char_type(sentence[start])
+        if char_cate[:invoke]
+          char_length = char_cate[:group] ? CharDef::MAX_GROUPING_SIZE : char_cate[:length]
+          unk_terminal = [start + char_length, terminal].min
           pos = start + 1
-          while pos < unk_terminal && char_type == CharDef.char_type(text[t])
-            word << text[t]
+          while pos < unk_terminal && char_type == CharDef.char_type(sentence[pos])
+            word << sentence[pos]
             pos += 1
           end
         end
-        @unknown_dictionary[char_type].each do |el|
-          lattice.insert(start, start + word.length,
-                         word, el[0].to_i, el[1].to_i, el[2].to_i,
-                         el[3..-1])
+        unknowns[char_type].each do |el|
+          lattice.insert(start, start + word.length, word, true,
+                         el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
         end
-        start += 1
+        step = [step, word.length].min
+
+        start += step
       end
 
       viterbi(lattice)
@@ -84,9 +81,25 @@ module Suika
 
     private
 
+    DICTIONARY_PATH = "#{__dir__}/../../dict/sysdic.gz"
+    DICTIONARY_KEY = '562e53853b8a5b9f4857536b0748847a0878ebf0'
     INT_MAX = 2**(([42].pack('i').size * 16) - 2) - 1
 
-    private_constant :INT_MAX
+    private_constant :DICTIONARY_PATH, :DICTIONARY_KEY, :INT_MAX
+
+    attr_reader :trie
+
+    def features
+      @sysdic[:dictionary]
+    end
+
+    def unknowns
+      @sysdic[:unknown_dictionary]
+    end
+
+    def costmat
+      @sysdic[:cost_matrix]
+    end
 
     def viterbi(lattice)
       bos = lattice.end_nodes[0].first
@@ -98,7 +111,7 @@ module Suika
           rnode.min_cost = INT_MAX
           rnode.min_prev = nil
           lattice.end_nodes[n].each do |lnode|
-            cost = lnode.min_cost + @cost_mat[lnode.right_id][rnode.left_id] + rnode.cost
+            cost = lnode.min_cost + costmat[lnode.right_id][rnode.left_id] + rnode.cost
             if cost < rnode.min_cost
               rnode.min_cost = cost
               rnode.min_prev = lnode
@@ -111,9 +124,10 @@ module Suika
       prev_node = eos.min_prev
       res = []
       until prev_node.nil?
-        res.append("#{prev_node.surface}\t#{prev_node.attrs.join(', ')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
+        res.push("#{prev_node.surface}\t#{prev_node.attrs.join(',')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
         prev_node = prev_node.min_prev
       end
+
       res.reverse
     end
   end

data/lib/suika/version.rb

@@ -3,5 +3,5 @@
 # Suika is a Japanese morphological analyzer written in pure Ruby.
 module Suika
   # The version of Suika you are using.
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/suika.gemspec

@@ -12,11 +12,10 @@ Gem::Specification.new do |spec|
   spec.description   = 'Suika is a Japanese morphological analyzer written in pure Ruby.'
   spec.homepage      = 'https://github.com/yoshoku/suika'
   spec.license       = 'BSD-3-Clause'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/magro/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/suika/blob/master/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://rubydoc.info/gems/suika'
 
   # Specify which files should be added to the gem when it is released.
@@ -28,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'rambling-trie', '~> 2.1'
+  spec.add_runtime_dependency 'dartsclone', '>= 0.2.0'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: suika
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-04 00:00:00.000000000 Z
+date: 2020-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rambling-trie
+  name: dartsclone
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.2.0
 description: Suika is a Japanese morphological analyzer written in pure Ruby.
 email:
 - yoshoku@outlook.com
@@ -35,18 +35,21 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - NOTICE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
-- dict/ipadic.gz
+- dict/sysdic.gz
 - lib/suika.rb
 - lib/suika/char_def.rb
 - lib/suika/lattice.rb
+- lib/suika/node.rb
 - lib/suika/tagger.rb
 - lib/suika/version.rb
 - suika.gemspec
@@ -56,7 +59,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/yoshoku/suika
   source_code_uri: https://github.com/yoshoku/suika
-  changelog_uri: https://github.com/yoshoku/magro/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/yoshoku/suika/blob/master/CHANGELOG.md
   documentation_uri: https://rubydoc.info/gems/suika
 post_install_message:
 rdoc_options: []
@@ -66,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="