suika 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfb2e78e15c648ee309868bdfd3f386a66b1cff633cb546880132cdb9b8f3806
-  data.tar.gz: d398f4de11a4af80b7c62c4e468fa2e3f9393bbb2211e0f2e317a08ed05c73b5
+  metadata.gz: f2aaecfc53b051cfbd06052bf22ac14614e459d42789670df1a7a31f601d533f
+  data.tar.gz: 39d4ac2b9fc0f4f164f2e3408b3dfcfe478f922d889334243c572a48ddf781de
 SHA512:
-  metadata.gz: 6fea777a4229725a174aa0955bcf5f775cc6e319fa73cf6f204385b3a47d84997b74b8798f78269f442aa11dcf151b1319c079c315dfea103dcd7a148cf0b5c5
-  data.tar.gz: 17d2fd7f248c965b6d585542c4ec4cb90e87663f96b0522f06f1c3e94c55a18e59f08e4ab1e7724f9dd48a114c2c0f35a0499100fb8850a89d2822de87927988
+  metadata.gz: dfdf6c08812109d7f6c6a52b64b8d8f7e6febd858bae23ccead4b3e598c76acd24a9c56cd893d958199b6545e37a4e01d6058c2acf39241a472260fda6accd90
+  data.tar.gz: 495016afc1854269d20b35e27bbb65cdffc307a9a94ab0149b8dcee884ac8560077ce0367a65f70b11bbdf0d27175134a2f66d67cc9bccc8f91a45353f2f1cf8

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: github-ci

data/.github/workflows/build.yml

@@ -0,0 +1,21 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [ '2.6', '2.7', '3.0' ]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        run: |
+          gem install --no-document bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rake

data/.github/workflows/coverage.yml

@@ -0,0 +1,26 @@
+name: coverage
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: '2.7'
+      - name: Build and test with Rake
+        run: |
+          gem install --no-document bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rake
+      - name: Coveralls GitHub Action
+        uses: coverallsapp/github-action@v1.1.2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.3.0
+
+### Changes
+- Add type declaration files.
+- Refactor to avoid assigning null to variables.
+- Fix some configuration files.
+
+
 ## 0.2.0
 
 ### Breaking Change

data/Gemfile

@@ -5,5 +5,9 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in suika.gemspec
 gemspec
 
-gem 'rake', '~> 12.0'
+gem 'rake', '~> 13.0'
 gem 'rspec', '~> 3.0'
+gem 'simplecov', '~> 0.21'
+gem 'simplecov-lcov', '~> 0.8'
+gem 'rbs', '~> 1.2'
+gem 'steep', '~> 0.44'

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2020 Atsushi Tatsuma
+Copyright (c) 2020-2021 Atsushi Tatsuma
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

data/README.md

@@ -1,8 +1,9 @@
 # Suika
 
-[![Build Status](https://travis-ci.org/yoshoku/suika.svg?branch=master)](https://travis-ci.org/yoshoku/suika)
+[![Build Status](https://github.com/yoshoku/suika/workflows/build/badge.svg)](https://github.com/yoshoku/suika/actions?query=workflow%3Abuild)
+[![Coverage Status](https://coveralls.io/repos/github/yoshoku/suika/badge.svg?branch=main)](https://coveralls.io/github/yoshoku/suika?branch=main)
 [![Gem Version](https://badge.fury.io/rb/suika.svg)](https://badge.fury.io/rb/suika)
-[![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/suika/blob/master/LICENSE.txt)
+[![BSD 3-Clause License](https://img.shields.io/badge/License-BSD%203--Clause-orange.svg)](https://github.com/yoshoku/suika/blob/main/LICENSE.txt)
 [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://rubydoc.info/gems/suika)
 
 Suika 🍉 is a Japanese morphological analyzer written in pure Ruby.
@@ -74,14 +75,14 @@ end
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yoshoku/suika.
-This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/yoshoku/suika/blob/master/CODE_OF_CONDUCT.md).
+This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/yoshoku/suika/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 
 The gem is available as open source under the terms of the [BSD-3-Clause License](https://opensource.org/licenses/BSD-3-Clause).
 In addition, the gem includes binary data generated from mecab-ipadic.
-The details of the license can be found in [LICENSE.txt](https://github.com/yoshoku/suika/blob/master/LICENSE.txt)
-and [NOTICE.txt](https://github.com/yoshoku/suika/blob/master/NOTICE.txt).
+The details of the license can be found in [LICENSE.txt](https://github.com/yoshoku/suika/blob/main/LICENSE.txt)
+and [NOTICE.txt](https://github.com/yoshoku/suika/blob/main/NOTICE.txt).
 
 ## Respect
 
@@ -94,4 +95,4 @@ Janome, a morphological analyzer written in scripting language, gives me the cou
 
 ## Code of Conduct
 
-Everyone interacting in the Suika project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/yoshoku/suika/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Suika project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/yoshoku/suika/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -1,6 +1,79 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+require 'csv'
+require 'dartsclone'
+require 'nkf'
+require 'rubygems/package'
+require 'zlib'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+
+desc 'Build suika system dictionary'
+task :dictionary do
+  base_dir = "#{__dir__}/dict/mecab-ipadic-2.7.0-20070801"
+  unless File.directory?(base_dir)
+    puts "Download mecab-ipadic file and expand that under dict directory:  #{__dir__}/dict/mecab-ipadic-2.7.0-20070801"
+    puts
+    puts 'Example:'
+    puts 'wget -O dict/mecab-ipadic.tgz https://drive.google.com/uc?export=download&id=0B4y35FiV1wh7MWVlSDBCSXZMTXM'
+    puts 'cd dict'
+    puts 'tar xzf mecab-ipadic.tgz'
+    puts 'cd ../'
+    next # exit
+  end
+
+  File.open("#{__dir__}/dict/mecab-ipadic-2.7.0-20070801/Reiwa.csv", 'w') do |f|
+    f.puts('令和,1288,1288,5904,名詞,固有名詞,一般,*,*,*,令和,レイワ,レイワ')
+  end
+
+  unknowns = {}
+  File.open("#{base_dir}/unk.def") do |f|
+    f.each_line do |line|
+      row = NKF.nkf('-w', line.chomp).split(',')
+      unknowns[row[0]] ||= []
+      unknowns[row[0]] << [row[1].to_i, row[2].to_i, row[3].to_i, *row[4..-1]]
+    end
+  end
+
+  dict = {}
+  Dir.glob("#{base_dir}/*.csv").each do |filename|
+    File.open(filename) do |f|
+      f.each_line do |line|
+        row = NKF.nkf('-w', line.chomp).split(',')
+        dict[row[0]] ||= []
+        dict[row[0]] << [row[1].to_i, row[2].to_i, row[3].to_i, *row[4..-1]]
+      end
+    end
+  end
+
+  da = DartsClone::DoubleArray.new
+  words = dict.keys.sort
+  da.build(words)
+  features = words.map { |w| dict[w] }
+
+  concosts = nil
+  File.open("#{base_dir}/matrix.def") do |f|
+    n_entries = f.readline.chomp.split.map(&:to_i).first
+    concosts = Array.new(n_entries) { Array.new(n_entries) }
+    f.each_line do |line|
+      row, col, cost = line.chomp.split.map(&:to_i)
+      concosts[row][col] = cost
+    end
+  end
+
+  ipadic = {
+    trie: da.get_array,
+    features: features,
+    unknowns: unknowns,
+    concosts: concosts
+  }
+
+  Zlib::GzipWriter.open("#{__dir__}/dict/sysdic.gz", Zlib::BEST_SPEED) { |f| f.write(Marshal.dump(ipadic)) }
+
+  puts 'The system dictionary has been successfully built:'
+  puts "#{__dir__}/dict/sysdic.gz"
+  puts Digest::SHA1.file("#{__dir__}/dict/sysdic.gz").to_s
+end

data/Steepfile

@@ -0,0 +1,20 @@
+target :lib do
+  signature "sig"
+#
+  check "lib"                       # Directory name
+#   check "Gemfile"                   # File name
+#   check "app/models/**/*.rb"        # Glob
+#   # ignore "lib/templates/*.rb"
+#
+#   # library "pathname", "set"       # Standard libraries
+  library "dartsclone"           # Gems
+end
+
+# target :spec do
+#   signature "sig", "sig-private"
+#
+#   check "spec"
+#
+#   # library "pathname", "set"       # Standard libraries
+#   # library "rspec"
+# end

data/lib/suika/tagger.rb

@@ -42,28 +42,30 @@ module Suika
       while start < terminal
         step = terminal - start
 
-        query = sentence[start..-1]
+        query = sentence[start..-1] || ''
         result = trie.common_prefix_search(query)
         unless result.empty?
           words, indices = result
-          words.each_with_index do |word, i|
-            features[indices[i]].each do |el|
-              lattice.insert(start, start + word.length, word, false,
-                             el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
+          unless words.empty?
+            step = INT_MAX
+            words.each_with_index do |word, i|
+              features[indices[i]].each do |el|
+                lattice.insert(start, start + word.length, word, false, el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
+              end
+              step = word.length if word.length < step
             end
           end
-          step = words.map(&:size).min
         end
 
-        word = sentence[start]
-        char_cate = CharDef.char_category(sentence[start])
-        char_type = CharDef.char_type(sentence[start])
+        word = sentence[start] || ''
+        char_cate = CharDef.char_category(sentence[start] || '')
+        char_type = CharDef.char_type(sentence[start] || '')
         if char_cate[:invoke]
-          char_length = char_cate[:group] ? CharDef::MAX_GROUPING_SIZE : char_cate[:length]
-          unk_terminal = [start + char_length, terminal].min
+          unk_terminal = start + (char_cate[:group] ? CharDef::MAX_GROUPING_SIZE : char_cate[:length])
+          unk_terminal = terminal if terminal < unk_terminal
           pos = start + 1
-          while pos < unk_terminal && char_type == CharDef.char_type(sentence[pos])
-            word << sentence[pos]
+          while pos < unk_terminal && char_type == CharDef.char_type(sentence[pos] || '')
+            word << (sentence[pos] || '')
             pos += 1
           end
         end
@@ -71,7 +73,7 @@ module Suika
           lattice.insert(start, start + word.length, word, true,
                          el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
         end
-        step = [step, word.length].min
+        step = word.length if word.length < step
 
         start += step
       end
@@ -82,7 +84,7 @@ module Suika
     private
 
     DICTIONARY_PATH = "#{__dir__}/../../dict/sysdic.gz"
-    DICTIONARY_KEY = '562e53853b8a5b9f4857536b0748847a0878ebf0'
+    DICTIONARY_KEY = 'eb921bf5e67f5733188527b21adbf9dabdda0c7a'
     INT_MAX = 2**(([42].pack('i').size * 16) - 2) - 1
 
     private_constant :DICTIONARY_PATH, :DICTIONARY_KEY, :INT_MAX
@@ -90,19 +92,19 @@ module Suika
     attr_reader :trie
 
     def features
-      @sysdic[:dictionary]
+      @sysdic[:features]
     end
 
     def unknowns
-      @sysdic[:unknown_dictionary]
+      @sysdic[:unknowns]
     end
 
-    def costmat
-      @sysdic[:cost_matrix]
+    def connect_cost(r_id, l_id)
+      @sysdic[:concosts][r_id][l_id]
     end
 
     def viterbi(lattice)
-      bos = lattice.end_nodes[0].first
+      bos = lattice.end_nodes[0][0]
       bos.min_cost = 0
       bos.min_prev = nil
 
@@ -111,7 +113,7 @@ module Suika
           rnode.min_cost = INT_MAX
           rnode.min_prev = nil
           lattice.end_nodes[n].each do |lnode|
-            cost = lnode.min_cost + costmat[lnode.right_id][rnode.left_id] + rnode.cost
+            cost = lnode.min_cost + connect_cost(lnode.right_id, rnode.left_id) + rnode.cost
             if cost < rnode.min_cost
               rnode.min_cost = cost
               rnode.min_prev = lnode
@@ -120,7 +122,7 @@ module Suika
         end
       end
 
-      eos = lattice.begin_nodes[-1].first
+      eos = lattice.begin_nodes[-1][0]
       prev_node = eos.min_prev
       res = []
       until prev_node.nil?

data/lib/suika/version.rb

@@ -3,5 +3,5 @@
 # Suika is a Japanese morphological analyzer written in pure Ruby.
 module Suika
   # The version of Suika you are using.
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/sig/suika.rbs

@@ -0,0 +1,3 @@
+module Suika
+  VERSION: String
+end

data/sig/suika/char_def.rbs

@@ -0,0 +1,25 @@
+module Suika
+  class CharDef
+    def self.char_type: (String ch) -> String
+    def self.char_category: (String ch) -> { invoke: bool, group: bool, length: Integer }
+
+    MAX_GROUPING_SIZE: Integer
+
+    private
+
+    #CHAR_CATEGORY: Hash[String, { invoke: bool, group: bool, length: Integer }]
+    CHAR_CATEGORY: Hash[String, untyped]
+    CHAR_TYPES: Array[String]
+
+    SPACE: Array[Range[Integer]]
+    NUMERIC: Array[Range[Integer]]
+    SYMBOL: Array[Range[Integer]]
+    ALPHA: Array[Range[Integer]]
+    CYRILLIC: Array[Range[Integer]]
+    GREEK: Array[Range[Integer]]
+    HIRAGANA: Array[Range[Integer]]
+    KATAKANA: Array[Range[Integer]]
+    KANJI: Array[Range[Integer]]
+    KANJINUMERIC: Array[Range[Integer]]
+  end
+end

data/sig/suika/lattice.rbs

@@ -0,0 +1,11 @@
+module Suika
+  class Lattice
+    attr_reader begin_nodes: Array[Array[::Suika::Node]]
+    attr_reader end_nodes: Array[Array[::Suika::Node]]
+    attr_reader length: Integer
+
+    def initialize: (Integer length) -> void
+    def insert: (Integer begin_id, Integer end_id, String surface, bool unknown,
+                 Integer left_id, Integer right_id, Integer cost, Array[String] attrs) -> void
+  end
+end

data/sig/suika/node.rbs

@@ -0,0 +1,18 @@
+module Suika
+  class Node
+    attr_accessor surface: String
+    attr_accessor unknown: bool
+    attr_accessor min_cost: Integer
+    # attr_accessor min_prev: ::Suika::Node?
+    attr_accessor min_prev: untyped
+    attr_accessor left_id: Integer
+    attr_accessor right_id: Integer
+    attr_accessor cost: Integer
+    attr_accessor attrs: Array[String]
+
+    def initialize: (?surface: String surface, ?unknown: bool unknown,
+                     ?min_cost: Integer min_cost, ?min_prev: ::Suika::Node? min_prev,
+                     ?left_id: ::Integer left_id, ?right_id: ::Integer right_id,
+                     ?cost: ::Integer cost, ?attrs: Array[String] attrs) -> void
+  end
+end

data/sig/suika/tagger.rbs

@@ -0,0 +1,23 @@
+module Suika
+  class Tagger
+    def initialize: () -> void
+    def parse: (String sentence) -> Array[String]
+
+    private
+
+    DICTIONARY_PATH: String
+    DICTIONARY_KEY: String
+    INT_MAX: untyped
+
+    attr_reader trie: ::DartsClone::DoubleArray
+
+    # type feature = [Integer, Integer, Integer, String, String, String, String, String, String, String]
+
+    # def features: () -> Array[Array[feature]]
+    def features: () -> Array[Array[untyped]]
+    # def unknowns: () -> Hash[String, Array[feature]]
+    def unknowns: () -> Hash[String, Array[untyped]]
+    def connect_cost: (Integer r_id, Integer l_id) -> Integer
+    def viterbi: (::Suika::Lattice lattice) -> Array[String]
+  end
+end

data/suika.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/suika/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/suika/blob/main/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://rubydoc.info/gems/suika'
 
   # Specify which files should be added to the gem when it is released.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suika
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - yoshoku
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-11 00:00:00.000000000 Z
+date: 2021-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dartsclone
@@ -31,18 +31,20 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
+- ".github/workflows/build.yml"
+- ".github/workflows/coverage.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - NOTICE.txt
 - README.md
 - Rakefile
+- Steepfile
 - bin/console
 - bin/setup
 - dict/sysdic.gz
@@ -52,6 +54,11 @@ files:
 - lib/suika/node.rb
 - lib/suika/tagger.rb
 - lib/suika/version.rb
+- sig/suika.rbs
+- sig/suika/char_def.rbs
+- sig/suika/lattice.rbs
+- sig/suika/node.rbs
+- sig/suika/tagger.rbs
 - suika.gemspec
 homepage: https://github.com/yoshoku/suika
 licenses:
@@ -59,9 +66,9 @@ licenses:
 metadata:
   homepage_uri: https://github.com/yoshoku/suika
   source_code_uri: https://github.com/yoshoku/suika
-  changelog_uri: https://github.com/yoshoku/suika/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/yoshoku/suika/blob/main/CHANGELOG.md
   documentation_uri: https://rubydoc.info/gems/suika
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -76,8 +83,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: Suika is a Japanese morphological analyzer written in pure Ruby.
 test_files: []

data/.travis.yml

@@ -1,12 +0,0 @@
----
-os: linux
-dist: xenial
-language: ruby
-cache: bundler
-rvm:
-  - '2.5'
-  - '2.6'
-  - '2.7'
-
-before_install:
-  - gem install bundler -v 2.1.4

data/Gemfile.lock

@@ -1,36 +0,0 @@
-PATH
-  remote: .
-  specs:
-    suika (0.2.0)
-      dartsclone (>= 0.2.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    dartsclone (0.2.0)
-    diff-lcs (1.4.4)
-    rake (12.3.3)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  rake (~> 12.0)
-  rspec (~> 3.0)
-  suika!
-
-BUNDLED WITH
-   2.1.2