suika 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24d1a6401851f6ca9fa7b4ea11d1966110b33c736499e5ad4facffda746d135d
-  data.tar.gz: 46763eabe75de50858ecfd98e92adee305d837033c996c8da2faeac5d56c694f
+  metadata.gz: f2aaecfc53b051cfbd06052bf22ac14614e459d42789670df1a7a31f601d533f
+  data.tar.gz: 39d4ac2b9fc0f4f164f2e3408b3dfcfe478f922d889334243c572a48ddf781de
 SHA512:
-  metadata.gz: '01690ace19b17fb68a368fc1d7a572a079c1d19876204ed85a239efbf6a5a2f4adcd5bab5f9d4f6c7c83dc54ff63e0c5a3c99819f88fed81065a96925bc27c5f'
-  data.tar.gz: 35b773639f6379a788f9dd3178e857752090420e406783e1abc27b276aa0c89f17b393a9c5ff60dc273f762b26c55f5155f5baa2a3723faaa1d8026d27b71c64
+  metadata.gz: dfdf6c08812109d7f6c6a52b64b8d8f7e6febd858bae23ccead4b3e598c76acd24a9c56cd893d958199b6545e37a4e01d6058c2acf39241a472260fda6accd90
+  data.tar.gz: 495016afc1854269d20b35e27bbb65cdffc307a9a94ab0149b8dcee884ac8560077ce0367a65f70b11bbdf0d27175134a2f66d67cc9bccc8f91a45353f2f1cf8

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: github-ci

data/.github/workflows/build.yml

@@ -0,0 +1,21 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [ '2.6', '2.7', '3.0' ]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        run: |
+          gem install --no-document bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rake

data/.github/workflows/coverage.yml

@@ -0,0 +1,26 @@
+name: coverage
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: '2.7'
+      - name: Build and test with Rake
+        run: |
+          gem install --no-document bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rake
+      - name: Coveralls GitHub Action
+        uses: coverallsapp/github-action@v1.1.2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

data/.rubocop.yml

@@ -3,7 +3,8 @@ require:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.4
+  NewCops: enable
+  TargetRubyVersion: 2.5
   DisplayCopNames: true
   DisplayStyleGuide: true
   Exclude:

data/CHANGELOG.md

@@ -1,5 +1,45 @@
-# 0.1.1
+## 0.3.0
+
+### Changes
+- Add type declaration files.
+- Refactor to avoid assigning null to variables.
+- Fix some configuration files.
+
+
+## 0.2.0
+
+### Breaking Change
+- Change to use dartsclone for trie library.
+
+
+## 0.1.4
+
+### Bug Fixes
+- Fix CharDef.char_type to return 'DEFAULT' when unknown character code is given.
+
+### Features
+- Add character code of square era name Reiwa.
+
+## 0.1.3
+
+### Bug Fixes
+- Fix unknown word processing.
+
+### Changes
+- Remove redundant spaces from output.
+
+
+## 0.1.2
+
+### Bug Fixes
+- Fix local variable typo in Tagger.parse.
+
+
+## 0.1.1
+
+### Bug Fixes
 - Fix specification of class in CharDef.char_type.
 
-# 0.1.0
+
+## 0.1.0
 - First release.

data/Gemfile

@@ -5,5 +5,9 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in suika.gemspec
 gemspec
 
-gem 'rake', '~> 12.0'
+gem 'rake', '~> 13.0'
 gem 'rspec', '~> 3.0'
+gem 'simplecov', '~> 0.21'
+gem 'simplecov-lcov', '~> 0.8'
+gem 'rbs', '~> 1.2'
+gem 'steep', '~> 0.44'

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2020 Atsushi Tatsuma
+Copyright (c) 2020-2021 Atsushi Tatsuma
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

data/README.md

@@ -1,7 +1,9 @@
 # Suika
 
+[![Build Status](https://github.com/yoshoku/suika/workflows/build/badge.svg)](https://github.com/yoshoku/suika/actions?query=workflow%3Abuild)
+[![Coverage Status](https://coveralls.io/repos/github/yoshoku/suika/badge.svg?branch=main)](https://coveralls.io/github/yoshoku/suika?branch=main)
 [![Gem Version](https://badge.fury.io/rb/suika.svg)](https://badge.fury.io/rb/suika)
-[![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/suika/blob/master/LICENSE.txt)
+[![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/suika/blob/main/LICENSE.txt)
 [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://rubydoc.info/gems/suika)
 
 Suika 🍉 is a Japanese morphological analyzer written in pure Ruby.
@@ -30,13 +32,13 @@ require 'suika'
 tagger = Suika::Tagger.new
 tagger.parse('すもももももももものうち').each { |token| puts token }
 
-# すもも  名詞, 一般, *, *, *, *, すもも, スモモ, スモモ
-# も      助詞, 係助詞, *, *, *, *, も, モ, モ
-# もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-# も      助詞, 係助詞, *, *, *, *, も, モ, モ
-# もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-# の      助詞, 連体化, *, *, *, *, の, ノ, ノ
-# うち    名詞, 非自立, 副詞可能, *, *, *, うち, ウチ, ウチ
+# すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+# も      助詞,係助詞,*,*,*,*,も,モ,モ
+# もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+# も      助詞,係助詞,*,*,*,*,も,モ,モ
+# もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+# の      助詞,連体化,*,*,*,*,の,ノ,ノ
+# うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
 ```
 
 Since the Tagger class loads the binary dictionary at initialization, it is recommended to reuse the instance.
@@ -51,17 +53,36 @@ sentences.each do |sentence|
 end
 ```
 
+## Test
+Suika was able to parse all sentences in the [Livedoor news corpus](https://www.rondhuit.com/download.html#ldcc)
+without any error.
+
+```ruby
+require 'suika'
+
+tagger = Suika::Tagger.new
+
+Dir.glob('ldcc-20140209/text/*/*.txt').each do |filename|
+  File.foreach(filename) do |sentence|
+    sentence.strip!
+    puts tagger.parse(sentence) unless sentence.empty?
+  end
+end
+```
+
+![suika_test](https://user-images.githubusercontent.com/5562409/90264778-8f593f80-de8c-11ea-81f1-20831e3c8b12.gif)
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yoshoku/suika.
-This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/yoshoku/suika/blob/master/CODE_OF_CONDUCT.md).
+This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/yoshoku/suika/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 
 The gem is available as open source under the terms of the [BSD-3-Clause License](https://opensource.org/licenses/BSD-3-Clause).
 In addition, the gem includes binary data generated from mecab-ipadic.
-The details of the license can be found in [LICENSE.txt](https://github.com/yoshoku/suika/blob/master/LICENSE.txt)
-and [NOTICE.txt](https://github.com/yoshoku/suika/blob/master/NOTICE.txt).
+The details of the license can be found in [LICENSE.txt](https://github.com/yoshoku/suika/blob/main/LICENSE.txt)
+and [NOTICE.txt](https://github.com/yoshoku/suika/blob/main/NOTICE.txt).
 
 ## Respect
 
@@ -74,4 +95,4 @@ Janome, a morphological analyzer written in scripting language, gives me the cou
 
 ## Code of Conduct
 
-Everyone interacting in the Suika project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/yoshoku/suika/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Suika project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/yoshoku/suika/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -1,6 +1,79 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+require 'csv'
+require 'dartsclone'
+require 'nkf'
+require 'rubygems/package'
+require 'zlib'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+
+desc 'Build suika system dictionary'
+task :dictionary do
+  base_dir = "#{__dir__}/dict/mecab-ipadic-2.7.0-20070801"
+  unless File.directory?(base_dir)
+    puts "Download mecab-ipadic file and expand that under dict directory:  #{__dir__}/dict/mecab-ipadic-2.7.0-20070801"
+    puts
+    puts 'Example:'
+    puts 'wget -O dict/mecab-ipadic.tgz https://drive.google.com/uc?export=download&id=0B4y35FiV1wh7MWVlSDBCSXZMTXM'
+    puts 'cd dict'
+    puts 'tar xzf mecab-ipadic.tgz'
+    puts 'cd ../'
+    next # exit
+  end
+
+  File.open("#{__dir__}/dict/mecab-ipadic-2.7.0-20070801/Reiwa.csv", 'w') do |f|
+    f.puts('令和,1288,1288,5904,名詞,固有名詞,一般,*,*,*,令和,レイワ,レイワ')
+  end
+
+  unknowns = {}
+  File.open("#{base_dir}/unk.def") do |f|
+    f.each_line do |line|
+      row = NKF.nkf('-w', line.chomp).split(',')
+      unknowns[row[0]] ||= []
+      unknowns[row[0]] << [row[1].to_i, row[2].to_i, row[3].to_i, *row[4..-1]]
+    end
+  end
+
+  dict = {}
+  Dir.glob("#{base_dir}/*.csv").each do |filename|
+    File.open(filename) do |f|
+      f.each_line do |line|
+        row = NKF.nkf('-w', line.chomp).split(',')
+        dict[row[0]] ||= []
+        dict[row[0]] << [row[1].to_i, row[2].to_i, row[3].to_i, *row[4..-1]]
+      end
+    end
+  end
+
+  da = DartsClone::DoubleArray.new
+  words = dict.keys.sort
+  da.build(words)
+  features = words.map { |w| dict[w] }
+
+  concosts = nil
+  File.open("#{base_dir}/matrix.def") do |f|
+    n_entries = f.readline.chomp.split.map(&:to_i).first
+    concosts = Array.new(n_entries) { Array.new(n_entries) }
+    f.each_line do |line|
+      row, col, cost = line.chomp.split.map(&:to_i)
+      concosts[row][col] = cost
+    end
+  end
+
+  ipadic = {
+    trie: da.get_array,
+    features: features,
+    unknowns: unknowns,
+    concosts: concosts
+  }
+
+  Zlib::GzipWriter.open("#{__dir__}/dict/sysdic.gz", Zlib::BEST_SPEED) { |f| f.write(Marshal.dump(ipadic)) }
+
+  puts 'The system dictionary has been successfully built:'
+  puts "#{__dir__}/dict/sysdic.gz"
+  puts Digest::SHA1.file("#{__dir__}/dict/sysdic.gz").to_s
+end

data/Steepfile

@@ -0,0 +1,20 @@
+target :lib do
+  signature "sig"
+#
+  check "lib"                       # Directory name
+#   check "Gemfile"                   # File name
+#   check "app/models/**/*.rb"        # Glob
+#   # ignore "lib/templates/*.rb"
+#
+#   # library "pathname", "set"       # Standard libraries
+  library "dartsclone"           # Gems
+end
+
+# target :spec do
+#   signature "sig", "sig-private"
+#
+#   check "spec"
+#
+#   # library "pathname", "set"       # Standard libraries
+#   # library "rspec"
+# end

data/lib/suika.rb

@@ -2,5 +2,6 @@
 
 require 'suika/version'
 require 'suika/char_def'
+require 'suika/node'
 require 'suika/lattice'
 require 'suika/tagger'

data/lib/suika/char_def.rb

@@ -5,10 +5,11 @@ module Suika
   class CharDef
     # @!visibility private
     def self.char_type(ch)
-      code = ch.unpack1('U*')
-      CHAR_TYPES.find do |ctype|
-        Object.const_get("::Suika::CharDef::#{ctype}").any? { |r| r.include?(code) }
+      ch_code = ch.unpack1('U*')
+      ch_type = CHAR_TYPES.find do |ct|
+        Object.const_get("::Suika::CharDef::#{ct}").any? { |r| r.include?(ch_code) }
       end
+      ch_type || 'DEFAULT'
     end
 
     # @!visibility private
@@ -16,39 +17,41 @@ module Suika
       CHAR_CATEGORY[char_type(ch)]
     end
 
+    MAX_GROUPING_SIZE = 24
+
     CHAR_CATEGORY = {
       'DEFAULT' => {
-        invoke: 0, group: 1, length: 0
+        invoke: false, group: true, length: 0
       },
       'SPACE' => {
-        invoke: 0, group: 1, length: 0
+        invoke: false, group: true, length: 0
       },
       'KANJI' => {
-        invoke: 0, group: 0, length: 2
+        invoke: false, group: false, length: 2
       },
       'SYMBOL' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'NUMERIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'ALPHA' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'HIRAGANA' => {
-        invoke: 0, group: 1, length: 2
+        invoke: false, group: true, length: 2
       },
       'KATAKANA' => {
-        invoke: 1, group: 1, length: 2
+        invoke: true, group: true, length: 2
       },
       'KANJINUMERIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'GREEK' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'CYRILLIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       }
     }.freeze
 
@@ -117,6 +120,7 @@ module Suika
       0x2B00..0x2BFF,  # Miscellaneous Symbols and Arrows
       0x2A00..0x2AFF,  # Supplemental Mathematical Operators
       0x3300..0x33FF,
+      0x32FF..0x32FF,  # Square era name REIWA
       0x3200..0x32FE,  # ENclosed CJK Letters and Months
       0x3000..0x303F,  # CJK Symbol and Punctuation
       0xFE30..0xFE4F,  # CJK Compatibility Forms
@@ -171,8 +175,6 @@ module Suika
       0xF900..0xFA2D,
       0xFA30..0xFA6A
     ].freeze
-
-    # rubocop:disable Style/AsciiComments
     # KANJI-NUMERIC （一 二 三 四 五 六 七 八 九 十 百 千 万 億 兆）
     # 0x4E00 KANJINUMERIC KANJI
     KANJINUMERIC = [
@@ -192,7 +194,6 @@ module Suika
       0x5104..0x5104,
       0x5146..0x5146
     ].freeze
-    # rubocop:enable Style/AsciiComments
 
     private_constant :CHAR_CATEGORY, :CHAR_TYPES
 

data/lib/suika/lattice.rb

@@ -4,8 +4,6 @@ module Suika
   # @!visibility private
   class Lattice
     # @!visibility private
-    Node = Struct.new(:surface, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs, keyword_init: true)
-
     attr_reader :begin_nodes, :end_nodes, :length
 
     # @!visibility private
@@ -13,17 +11,17 @@ module Suika
       @length = length
       @begin_nodes = Array.new(length + 1) { [] }
       @end_nodes = Array.new(length + 1) { [] }
-      bos = Node.new(surface: 'BOS', left_id: 0, right_id: 0, cost: 0, attrs: [])
-      @end_nodes[0].append(bos)
-      eos = Node.new(surface: 'EOS', left_id: 0, right_id: 0, cost: 0, attrs: [])
-      @begin_nodes[length].append(eos)
+      bos = Node.new(surface: 'BOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @end_nodes[0].push(bos)
+      eos = Node.new(surface: 'EOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @begin_nodes[length].push(eos)
     end
 
     # @!visibility private
-    def insert(begin_id, end_id, surface, left_id, right_id, cost, attrs)
-      node = Node.new(surface: surface, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
-      @begin_nodes[begin_id].append(node)
-      @end_nodes[end_id].append(node)
+    def insert(begin_id, end_id, surface, unknown, left_id, right_id, cost, attrs)
+      node = Node.new(surface: surface, unknown: unknown, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
+      @begin_nodes[begin_id].push(node)
+      @end_nodes[end_id].push(node)
     end
   end
 end

data/lib/suika/node.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Suika
+  # @!visibility private
+  class Node
+    # @!visibility private
+    attr_accessor :surface, :unknown, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs
+
+    # @!visibility private
+    def initialize(surface: '', unknown: false, min_cost: 0, min_prev: nil, left_id: 0, right_id: 0, cost: 0, attrs: [])
+      @surface = surface
+      @unknown = unknown
+      @min_cost = min_cost
+      @min_prev = min_prev
+      @left_id = left_id
+      @right_id = right_id
+      @cost = cost
+      @attrs = attrs
+    end
+  end
+end

data/lib/suika/tagger.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require 'rambling-trie'
+require 'dartsclone'
+require 'rubygems/package'
 require 'zlib'
 
 module Suika
@@ -12,22 +13,22 @@ module Suika
   #   tagger = Suika::Tagger.new
   #   tagger.parse('すもももももももものうち').each { |token| puts token }
   #
-  #   # すもも  名詞, 一般, *, *, *, *, すもも, スモモ, スモモ
-  #   # も      助詞, 係助詞, *, *, *, *, も, モ, モ
-  #   # もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-  #   # も      助詞, 係助詞, *, *, *, *, も, モ, モ
-  #   # もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-  #   # の      助詞, 連体化, *, *, *, *, の, ノ, ノ
-  #   # うち    名詞, 非自立, 副詞可能, *, *, *, うち, ウチ, ウチ
+  #   # すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+  #   # も      助詞,係助詞,*,*,*,*,も,モ,モ
+  #   # もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+  #   # も      助詞,係助詞,*,*,*,*,も,モ,モ
+  #   # もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+  #   # の      助詞,連体化,*,*,*,*,の,ノ,ノ
+  #   # うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
   #
   class Tagger
     # Create a new tagger by loading the built-in binary dictionary.
     def initialize
-      ipadic = Marshal.load(Zlib::GzipReader.open(__dir__ + '/../../dict/ipadic.gz', &:read))
-      @trie = ipadic[:trie]
-      @dictionary = ipadic[:dictionary]
-      @unknown_dictionary = ipadic[:unknown_dictionary]
-      @cost_mat = ipadic[:cost_matrix]
+      raise IOError, 'SHA1 digest of dictionary file does not match.' unless DICTIONARY_KEY == Digest::SHA1.file(DICTIONARY_PATH).to_s
+
+      @sysdic = Marshal.load(Zlib::GzipReader.open(DICTIONARY_PATH, &:read))
+      @trie = DartsClone::DoubleArray.new
+      @trie.set_array(@sysdic[:trie])
     end
 
     # Parse the given sentence.
@@ -39,44 +40,42 @@ module Suika
       terminal = sentence.length
 
       while start < terminal
-        word = sentence[start]
-        pos = start
-        is_unknown = true
-        while @trie.match?(word) && pos < terminal
-          if @dictionary.key?(word)
-            @dictionary[word].each do |el|
-              lattice.insert(start, start + word.length,
-                             word, el[0].to_i, el[1].to_i, el[2].to_i,
-                             el[3..-1])
+        step = terminal - start
+
+        query = sentence[start..-1] || ''
+        result = trie.common_prefix_search(query)
+        unless result.empty?
+          words, indices = result
+          unless words.empty?
+            step = INT_MAX
+            words.each_with_index do |word, i|
+              features[indices[i]].each do |el|
+                lattice.insert(start, start + word.length, word, false, el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
+              end
+              step = word.length if word.length < step
             end
-            is_unknown = false
           end
-          pos += 1
-          word = sentence[start..pos]
-        end
-
-        unless is_unknown
-          start += 1
-          next
         end
 
-        word = sentence[start]
-        char_type = CharDef.char_type(sentence[start])
-        char_cate = CharDef.char_category(sentence[start])
-        if char_cate[:group] == 1
-          unk_terminal = char_cate[:length].zero? ? terminal : start + char_cate[:length]
+        word = sentence[start] || ''
+        char_cate = CharDef.char_category(sentence[start] || '')
+        char_type = CharDef.char_type(sentence[start] || '')
+        if char_cate[:invoke]
+          unk_terminal = start + (char_cate[:group] ? CharDef::MAX_GROUPING_SIZE : char_cate[:length])
+          unk_terminal = terminal if terminal < unk_terminal
           pos = start + 1
-          while pos < unk_terminal && char_type == CharDef.char_type(text[t])
-            word << text[t]
+          while pos < unk_terminal && char_type == CharDef.char_type(sentence[pos] || '')
+            word << (sentence[pos] || '')
             pos += 1
           end
         end
-        @unknown_dictionary[char_type].each do |el|
-          lattice.insert(start, start + word.length,
-                         word, el[0].to_i, el[1].to_i, el[2].to_i,
-                         el[3..-1])
+        unknowns[char_type].each do |el|
+          lattice.insert(start, start + word.length, word, true,
+                         el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
         end
-        start += 1
+        step = word.length if word.length < step
+
+        start += step
       end
 
       viterbi(lattice)
@@ -84,12 +83,28 @@ module Suika
 
     private
 
+    DICTIONARY_PATH = "#{__dir__}/../../dict/sysdic.gz"
+    DICTIONARY_KEY = 'eb921bf5e67f5733188527b21adbf9dabdda0c7a'
     INT_MAX = 2**(([42].pack('i').size * 16) - 2) - 1
 
-    private_constant :INT_MAX
+    private_constant :DICTIONARY_PATH, :DICTIONARY_KEY, :INT_MAX
+
+    attr_reader :trie
+
+    def features
+      @sysdic[:features]
+    end
+
+    def unknowns
+      @sysdic[:unknowns]
+    end
+
+    def connect_cost(r_id, l_id)
+      @sysdic[:concosts][r_id][l_id]
+    end
 
     def viterbi(lattice)
-      bos = lattice.end_nodes[0].first
+      bos = lattice.end_nodes[0][0]
       bos.min_cost = 0
       bos.min_prev = nil
 
@@ -98,7 +113,7 @@ module Suika
           rnode.min_cost = INT_MAX
           rnode.min_prev = nil
           lattice.end_nodes[n].each do |lnode|
-            cost = lnode.min_cost + @cost_mat[lnode.right_id][rnode.left_id] + rnode.cost
+            cost = lnode.min_cost + connect_cost(lnode.right_id, rnode.left_id) + rnode.cost
             if cost < rnode.min_cost
               rnode.min_cost = cost
               rnode.min_prev = lnode
@@ -107,13 +122,14 @@ module Suika
         end
       end
 
-      eos = lattice.begin_nodes[-1].first
+      eos = lattice.begin_nodes[-1][0]
       prev_node = eos.min_prev
       res = []
       until prev_node.nil?
-        res.append("#{prev_node.surface}\t#{prev_node.attrs.join(', ')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
+        res.push("#{prev_node.surface}\t#{prev_node.attrs.join(',')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
         prev_node = prev_node.min_prev
       end
+
       res.reverse
     end
   end

data/lib/suika/version.rb

@@ -3,5 +3,5 @@
 # Suika is a Japanese morphological analyzer written in pure Ruby.
 module Suika
   # The version of Suika you are using.
-  VERSION = '0.1.1'
+  VERSION = '0.3.0'
 end

data/sig/suika.rbs

@@ -0,0 +1,3 @@
+module Suika
+  VERSION: String
+end

data/sig/suika/char_def.rbs

@@ -0,0 +1,25 @@
+module Suika
+  class CharDef
+    def self.char_type: (String ch) -> String
+    def self.char_category: (String ch) -> { invoke: bool, group: bool, length: Integer }
+
+    MAX_GROUPING_SIZE: Integer
+
+    private
+
+    #CHAR_CATEGORY: Hash[String, { invoke: bool, group: bool, length: Integer }]
+    CHAR_CATEGORY: Hash[String, untyped]
+    CHAR_TYPES: Array[String]
+
+    SPACE: Array[Range[Integer]]
+    NUMERIC: Array[Range[Integer]]
+    SYMBOL: Array[Range[Integer]]
+    ALPHA: Array[Range[Integer]]
+    CYRILLIC: Array[Range[Integer]]
+    GREEK: Array[Range[Integer]]
+    HIRAGANA: Array[Range[Integer]]
+    KATAKANA: Array[Range[Integer]]
+    KANJI: Array[Range[Integer]]
+    KANJINUMERIC: Array[Range[Integer]]
+  end
+end

data/sig/suika/lattice.rbs

@@ -0,0 +1,11 @@
+module Suika
+  class Lattice
+    attr_reader begin_nodes: Array[Array[::Suika::Node]]
+    attr_reader end_nodes: Array[Array[::Suika::Node]]
+    attr_reader length: Integer
+
+    def initialize: (Integer length) -> void
+    def insert: (Integer begin_id, Integer end_id, String surface, bool unknown,
+                 Integer left_id, Integer right_id, Integer cost, Array[String] attrs) -> void
+  end
+end

data/sig/suika/node.rbs

@@ -0,0 +1,18 @@
+module Suika
+  class Node
+    attr_accessor surface: String
+    attr_accessor unknown: bool
+    attr_accessor min_cost: Integer
+    # attr_accessor min_prev: ::Suika::Node?
+    attr_accessor min_prev: untyped
+    attr_accessor left_id: Integer
+    attr_accessor right_id: Integer
+    attr_accessor cost: Integer
+    attr_accessor attrs: Array[String]
+
+    def initialize: (?surface: String surface, ?unknown: bool unknown,
+                     ?min_cost: Integer min_cost, ?min_prev: ::Suika::Node? min_prev,
+                     ?left_id: ::Integer left_id, ?right_id: ::Integer right_id,
+                     ?cost: ::Integer cost, ?attrs: Array[String] attrs) -> void
+  end
+end

data/sig/suika/tagger.rbs

@@ -0,0 +1,23 @@
+module Suika
+  class Tagger
+    def initialize: () -> void
+    def parse: (String sentence) -> Array[String]
+
+    private
+
+    DICTIONARY_PATH: String
+    DICTIONARY_KEY: String
+    INT_MAX: untyped
+
+    attr_reader trie: ::DartsClone::DoubleArray
+
+    # type feature = [Integer, Integer, Integer, String, String, String, String, String, String, String]
+
+    # def features: () -> Array[Array[feature]]
+    def features: () -> Array[Array[untyped]]
+    # def unknowns: () -> Hash[String, Array[feature]]
+    def unknowns: () -> Hash[String, Array[untyped]]
+    def connect_cost: (Integer r_id, Integer l_id) -> Integer
+    def viterbi: (::Suika::Lattice lattice) -> Array[String]
+  end
+end

data/suika.gemspec

@@ -12,11 +12,10 @@ Gem::Specification.new do |spec|
   spec.description   = 'Suika is a Japanese morphological analyzer written in pure Ruby.'
   spec.homepage      = 'https://github.com/yoshoku/suika'
   spec.license       = 'BSD-3-Clause'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/magro/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/suika/blob/main/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://rubydoc.info/gems/suika'
 
   # Specify which files should be added to the gem when it is released.
@@ -28,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'rambling-trie', '~> 2.1'
+  spec.add_runtime_dependency 'dartsclone', '>= 0.2.0'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: suika
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - yoshoku
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-05 00:00:00.000000000 Z
+date: 2021-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rambling-trie
+  name: dartsclone
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.2.0
 description: Suika is a Japanese morphological analyzer written in pure Ruby.
 email:
 - yoshoku@outlook.com
@@ -31,10 +31,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
+- ".github/workflows/build.yml"
+- ".github/workflows/coverage.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -42,14 +44,21 @@ files:
 - NOTICE.txt
 - README.md
 - Rakefile
+- Steepfile
 - bin/console
 - bin/setup
-- dict/ipadic.gz
+- dict/sysdic.gz
 - lib/suika.rb
 - lib/suika/char_def.rb
 - lib/suika/lattice.rb
+- lib/suika/node.rb
 - lib/suika/tagger.rb
 - lib/suika/version.rb
+- sig/suika.rbs
+- sig/suika/char_def.rbs
+- sig/suika/lattice.rbs
+- sig/suika/node.rbs
+- sig/suika/tagger.rbs
 - suika.gemspec
 homepage: https://github.com/yoshoku/suika
 licenses:
@@ -57,9 +66,9 @@ licenses:
 metadata:
   homepage_uri: https://github.com/yoshoku/suika
   source_code_uri: https://github.com/yoshoku/suika
-  changelog_uri: https://github.com/yoshoku/magro/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/yoshoku/suika/blob/main/CHANGELOG.md
   documentation_uri: https://rubydoc.info/gems/suika
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -67,15 +76,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: Suika is a Japanese morphological analyzer written in pure Ruby.
 test_files: []

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.7.0
-before_install: gem install bundler -v 2.1.2