suika 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2aafd394952b381595891fab5652ed86c1c71a18d09befd1ffa2cd4bfdb05e8e
-  data.tar.gz: 33385a74bac5c831a7116aa34ee31c5470154c928e971926b7f0f112b3cf1264
+  metadata.gz: bfb2e78e15c648ee309868bdfd3f386a66b1cff633cb546880132cdb9b8f3806
+  data.tar.gz: d398f4de11a4af80b7c62c4e468fa2e3f9393bbb2211e0f2e317a08ed05c73b5
 SHA512:
-  metadata.gz: 80de1122cb9aae4a3313bb1ee95594435415dcacb197ef9bbf33a2695f57b9cc14c8f466aab19c8f487f1f4c3e4107ecd3851fad217921f61511655b5309fb21
-  data.tar.gz: e0e9b76d04c847f6c1a8013db4b8a05ab9050dbde59d25c76d66fee078ac6b2ca230652b03ee61f3d97b7eebdc1139a34fb790fc5c954b072b55747e1933415a
+  metadata.gz: 6fea777a4229725a174aa0955bcf5f775cc6e319fa73cf6f204385b3a47d84997b74b8798f78269f442aa11dcf151b1319c079c315dfea103dcd7a148cf0b5c5
+  data.tar.gz: 17d2fd7f248c965b6d585542c4ec4cb90e87663f96b0522f06f1c3e94c55a18e59f08e4ab1e7724f9dd48a114c2c0f35a0499100fb8850a89d2822de87927988

data/.rubocop.yml

@@ -3,7 +3,8 @@ require:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.4
+  NewCops: enable
+  TargetRubyVersion: 2.5
   DisplayCopNames: true
   DisplayStyleGuide: true
   Exclude:

data/.travis.yml

@@ -1,6 +1,12 @@
 ---
+os: linux
+dist: xenial
 language: ruby
 cache: bundler
 rvm:
-  - 2.7.0
-before_install: gem install bundler -v 2.1.2
+  - '2.5'
+  - '2.6'
+  - '2.7'
+
+before_install:
+  - gem install bundler -v 2.1.4

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.0
+
+### Breaking Change
+- Change to use dartsclone for trie library.
+
+
 ## 0.1.4
 
 ### Bug Fixes

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    suika (0.1.2)
-      rambling-trie (~> 2.1)
+    suika (0.2.0)
+      dartsclone (>= 0.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    dartsclone (0.2.0)
     diff-lcs (1.4.4)
     rake (12.3.3)
-    rambling-trie (2.1.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)

data/README.md

@@ -1,5 +1,6 @@
 # Suika
 
+[![Build Status](https://travis-ci.org/yoshoku/suika.svg?branch=master)](https://travis-ci.org/yoshoku/suika)
 [![Gem Version](https://badge.fury.io/rb/suika.svg)](https://badge.fury.io/rb/suika)
 [![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/suika/blob/master/LICENSE.txt)
 [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://rubydoc.info/gems/suika)
@@ -51,6 +52,25 @@ sentences.each do |sentence|
 end
 ```
 
+## Test
+Suika was able to parse all sentences in the [Livedoor news corpus](https://www.rondhuit.com/download.html#ldcc)
+without any error.
+
+```ruby
+require 'suika'
+
+tagger = Suika::Tagger.new
+
+Dir.glob('ldcc-20140209/text/*/*.txt').each do |filename|
+  File.foreach(filename) do |sentence|
+    sentence.strip!
+    puts tagger.parse(sentence) unless sentence.empty?
+  end
+end
+```
+
+![suika_test](https://user-images.githubusercontent.com/5562409/90264778-8f593f80-de8c-11ea-81f1-20831e3c8b12.gif)
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yoshoku/suika.

data/lib/suika.rb

@@ -2,5 +2,6 @@
 
 require 'suika/version'
 require 'suika/char_def'
+require 'suika/node'
 require 'suika/lattice'
 require 'suika/tagger'

data/lib/suika/lattice.rb

@@ -4,8 +4,6 @@ module Suika
   # @!visibility private
   class Lattice
     # @!visibility private
-    Node = Struct.new(:surface, :unknown, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs, keyword_init: true)
-
     attr_reader :begin_nodes, :end_nodes, :length
 
     # @!visibility private
@@ -14,16 +12,16 @@ module Suika
       @begin_nodes = Array.new(length + 1) { [] }
       @end_nodes = Array.new(length + 1) { [] }
       bos = Node.new(surface: 'BOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
-      @end_nodes[0].append(bos)
+      @end_nodes[0].push(bos)
       eos = Node.new(surface: 'EOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
-      @begin_nodes[length].append(eos)
+      @begin_nodes[length].push(eos)
     end
 
     # @!visibility private
     def insert(begin_id, end_id, surface, unknown, left_id, right_id, cost, attrs)
       node = Node.new(surface: surface, unknown: unknown, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
-      @begin_nodes[begin_id].append(node)
-      @end_nodes[end_id].append(node)
+      @begin_nodes[begin_id].push(node)
+      @end_nodes[end_id].push(node)
     end
   end
 end

data/lib/suika/node.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Suika
+  # @!visibility private
+  class Node
+    # @!visibility private
+    attr_accessor :surface, :unknown, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs
+
+    # @!visibility private
+    def initialize(surface: '', unknown: false, min_cost: 0, min_prev: nil, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @surface = surface
+      @unknown = unknown
+      @min_cost = min_cost
+      @min_prev = min_prev
+      @left_id = left_id
+      @right_id = right_id
+      @cost = cost
+      @attrs = attrs
+    end
+  end
+end

data/lib/suika/tagger.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require 'rambling-trie'
+require 'dartsclone'
+require 'rubygems/package'
 require 'zlib'
 
 module Suika
@@ -23,11 +24,11 @@ module Suika
   class Tagger
     # Create a new tagger by loading the built-in binary dictionary.
     def initialize
-      ipadic = Marshal.load(Zlib::GzipReader.open(__dir__ + '/../../dict/ipadic.gz', &:read))
-      @trie = ipadic[:trie]
-      @dictionary = ipadic[:dictionary]
-      @unknown_dictionary = ipadic[:unknown_dictionary]
-      @cost_mat = ipadic[:cost_matrix]
+      raise IOError, 'SHA1 digest of dictionary file does not match.' unless DICTIONARY_KEY == Digest::SHA1.file(DICTIONARY_PATH).to_s
+
+      @sysdic = Marshal.load(Zlib::GzipReader.open(DICTIONARY_PATH, &:read))
+      @trie = DartsClone::DoubleArray.new
+      @trie.set_array(@sysdic[:trie])
     end
 
     # Parse the given sentence.
@@ -39,39 +40,40 @@ module Suika
       terminal = sentence.length
 
       while start < terminal
-        word = sentence[start]
-        pos = start
-        matched = false
-        while @trie.match?(word) && pos < terminal
-          if @dictionary.key?(word)
-            matched = true
-            @dictionary[word].each do |el|
+        step = terminal - start
+
+        query = sentence[start..-1]
+        result = trie.common_prefix_search(query)
+        unless result.empty?
+          words, indices = result
+          words.each_with_index do |word, i|
+            features[indices[i]].each do |el|
               lattice.insert(start, start + word.length, word, false,
                              el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
             end
           end
-          pos += 1
-          word = sentence[start..pos]
+          step = words.map(&:size).min
         end
 
         word = sentence[start]
         char_cate = CharDef.char_category(sentence[start])
-        unless !char_cate[:invoke] && matched
+        char_type = CharDef.char_type(sentence[start])
+        if char_cate[:invoke]
           char_length = char_cate[:group] ? CharDef::MAX_GROUPING_SIZE : char_cate[:length]
           unk_terminal = [start + char_length, terminal].min
           pos = start + 1
-          char_type = CharDef.char_type(sentence[start])
           while pos < unk_terminal && char_type == CharDef.char_type(sentence[pos])
             word << sentence[pos]
             pos += 1
           end
-          @unknown_dictionary[char_type].each do |el|
-            lattice.insert(start, start + word.length, word, true,
-                           el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
-          end
         end
+        unknowns[char_type].each do |el|
+          lattice.insert(start, start + word.length, word, true,
+                         el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
+        end
+        step = [step, word.length].min
 
-        start += 1
+        start += step
       end
 
       viterbi(lattice)
@@ -79,9 +81,25 @@ module Suika
 
     private
 
+    DICTIONARY_PATH = "#{__dir__}/../../dict/sysdic.gz"
+    DICTIONARY_KEY = '562e53853b8a5b9f4857536b0748847a0878ebf0'
     INT_MAX = 2**(([42].pack('i').size * 16) - 2) - 1
 
-    private_constant :INT_MAX
+    private_constant :DICTIONARY_PATH, :DICTIONARY_KEY, :INT_MAX
+
+    attr_reader :trie
+
+    def features
+      @sysdic[:dictionary]
+    end
+
+    def unknowns
+      @sysdic[:unknown_dictionary]
+    end
+
+    def costmat
+      @sysdic[:cost_matrix]
+    end
 
     def viterbi(lattice)
       bos = lattice.end_nodes[0].first
@@ -93,7 +111,7 @@ module Suika
           rnode.min_cost = INT_MAX
           rnode.min_prev = nil
           lattice.end_nodes[n].each do |lnode|
-            cost = lnode.min_cost + @cost_mat[lnode.right_id][rnode.left_id] + rnode.cost
+            cost = lnode.min_cost + costmat[lnode.right_id][rnode.left_id] + rnode.cost
             if cost < rnode.min_cost
               rnode.min_cost = cost
               rnode.min_prev = lnode
@@ -106,7 +124,7 @@ module Suika
       prev_node = eos.min_prev
       res = []
       until prev_node.nil?
-        res.append("#{prev_node.surface}\t#{prev_node.attrs.join(',')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
+        res.push("#{prev_node.surface}\t#{prev_node.attrs.join(',')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
         prev_node = prev_node.min_prev
       end
 

data/lib/suika/version.rb

@@ -3,5 +3,5 @@
 # Suika is a Japanese morphological analyzer written in pure Ruby.
 module Suika
   # The version of Suika you are using.
-  VERSION = '0.1.4'
+  VERSION = '0.2.0'
 end

data/suika.gemspec

@@ -12,7 +12,6 @@ Gem::Specification.new do |spec|
   spec.description   = 'Suika is a Japanese morphological analyzer written in pure Ruby.'
   spec.homepage      = 'https://github.com/yoshoku/suika'
   spec.license       = 'BSD-3-Clause'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
@@ -28,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'rambling-trie', '~> 2.1'
+  spec.add_runtime_dependency 'dartsclone', '>= 0.2.0'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: suika
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-08 00:00:00.000000000 Z
+date: 2020-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rambling-trie
+  name: dartsclone
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.2.0
 description: Suika is a Japanese morphological analyzer written in pure Ruby.
 email:
 - yoshoku@outlook.com
@@ -45,10 +45,11 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- dict/ipadic.gz
+- dict/sysdic.gz
 - lib/suika.rb
 - lib/suika/char_def.rb
 - lib/suika/lattice.rb
+- lib/suika/node.rb
 - lib/suika/tagger.rb
 - lib/suika/version.rb
 - suika.gemspec
@@ -68,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="