suika 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1fb292a3b36ec1dde8f93da624092876220ec33fa49ff635cbaff93fa9337137
-  data.tar.gz: 0e2c0cc53c6f25099dfa455f85d618491819dd98cc33047a6aebd4fb5858d13a
+  metadata.gz: 108d59930a3a546de3a603d4c7de5851249264a4eddd76c8a350407b2d0b353a
+  data.tar.gz: d879db93ee8593466c415ba9064b50221ee2355fa0caedcd301fe07de9ab4880
 SHA512:
-  metadata.gz: 1f54af2d9955f7c562df25d0d724c0b8f1460959b4d35d835c9412a7897e329a95016a7bc151c7125a8b4be9788a18dc7265cc4b4355ad7c1a5b1c828af0f622
-  data.tar.gz: a7b1a9a484b51d03ec92a09a2e4ee1a24e1cfcbaafe27f8cd7cbc0b7b02989c0960ebc1799974816adf6399e2e21fa02e06cad1fb8a584db4238e08f93baf93b
+  metadata.gz: aceff27ac13d1c7b03ae286e2a24c02f5fe75d987e5ea91879777e66adf8155b8d92d185399e2fe31bf6dd7e6448fb175d3ed0b864c55e2df2ff30b6dcaad5a3
+  data.tar.gz: a92d9161f73a761c9621fa720725dbebdfa74da1d8b892ab3ecab8aa9ee835d98bf42341818219b0fdf1dd3a164a1206f306f0e8dcad9030b7137776ed5142e0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.1.3
+- Fix unknown word processing.
+- Remove redundant spaces from output.
+
 # 0.1.2
 - Fix local variable typo in Tagger.parse.
 

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    suika (0.1.2)
+      rambling-trie (~> 2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    rake (12.3.3)
+    rambling-trie (2.1.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  suika!
+
+BUNDLED WITH
+   2.1.2

data/README.md

@@ -30,13 +30,13 @@ require 'suika'
 tagger = Suika::Tagger.new
 tagger.parse('すもももももももものうち').each { |token| puts token }
 
-# すもも  名詞, 一般, *, *, *, *, すもも, スモモ, スモモ
-# も      助詞, 係助詞, *, *, *, *, も, モ, モ
-# もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-# も      助詞, 係助詞, *, *, *, *, も, モ, モ
-# もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-# の      助詞, 連体化, *, *, *, *, の, ノ, ノ
-# うち    名詞, 非自立, 副詞可能, *, *, *, うち, ウチ, ウチ
+# すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+# も      助詞,係助詞,*,*,*,*,も,モ,モ
+# もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+# も      助詞,係助詞,*,*,*,*,も,モ,モ
+# もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+# の      助詞,連体化,*,*,*,*,の,ノ,ノ
+# うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
 ```
 
 Since the Tagger class loads the binary dictionary at initialization, it is recommended to reuse the instance.

data/lib/suika/char_def.rb

@@ -16,39 +16,41 @@ module Suika
       CHAR_CATEGORY[char_type(ch)]
     end
 
+    MAX_GROUPING_SIZE = 24
+
     CHAR_CATEGORY = {
       'DEFAULT' => {
-        invoke: 0, group: 1, length: 0
+        invoke: false, group: true, length: 0
       },
       'SPACE' => {
-        invoke: 0, group: 1, length: 0
+        invoke: false, group: true, length: 0
       },
       'KANJI' => {
-        invoke: 0, group: 0, length: 2
+        invoke: false, group: false, length: 2
       },
       'SYMBOL' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'NUMERIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'ALPHA' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'HIRAGANA' => {
-        invoke: 0, group: 1, length: 2
+        invoke: false, group: true, length: 2
       },
       'KATAKANA' => {
-        invoke: 1, group: 1, length: 2
+        invoke: true, group: true, length: 2
       },
       'KANJINUMERIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'GREEK' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       },
       'CYRILLIC' => {
-        invoke: 1, group: 1, length: 0
+        invoke: true, group: true, length: 0
       }
     }.freeze
 

data/lib/suika/lattice.rb

@@ -4,7 +4,7 @@ module Suika
   # @!visibility private
   class Lattice
     # @!visibility private
-    Node = Struct.new(:surface, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs, keyword_init: true)
+    Node = Struct.new(:surface, :unknown, :min_cost, :min_prev, :left_id, :right_id, :cost, :attrs, keyword_init: true)
 
     attr_reader :begin_nodes, :end_nodes, :length
 
@@ -13,15 +13,15 @@ module Suika
       @length = length
       @begin_nodes = Array.new(length + 1) { [] }
       @end_nodes = Array.new(length + 1) { [] }
-      bos = Node.new(surface: 'BOS', left_id: 0, right_id: 0, cost: 0, attrs: [])
+      bos = Node.new(surface: 'BOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
       @end_nodes[0].append(bos)
-      eos = Node.new(surface: 'EOS', left_id: 0, right_id: 0, cost: 0, attrs: [])
+      eos = Node.new(surface: 'EOS', unknown: false, left_id: 0, right_id: 0, cost: 0, attrs: [])
       @begin_nodes[length].append(eos)
     end
 
     # @!visibility private
-    def insert(begin_id, end_id, surface, left_id, right_id, cost, attrs)
-      node = Node.new(surface: surface, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
+    def insert(begin_id, end_id, surface, unknown, left_id, right_id, cost, attrs)
+      node = Node.new(surface: surface, unknown: unknown, left_id: left_id, right_id: right_id, cost: cost, attrs: attrs)
       @begin_nodes[begin_id].append(node)
       @end_nodes[end_id].append(node)
     end

data/lib/suika/tagger.rb

@@ -12,13 +12,13 @@ module Suika
   #   tagger = Suika::Tagger.new
   #   tagger.parse('すもももももももものうち').each { |token| puts token }
   #
-  #   # すもも  名詞, 一般, *, *, *, *, すもも, スモモ, スモモ
-  #   # も      助詞, 係助詞, *, *, *, *, も, モ, モ
-  #   # もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-  #   # も      助詞, 係助詞, *, *, *, *, も, モ, モ
-  #   # もも    名詞, 一般, *, *, *, *, もも, モモ, モモ
-  #   # の      助詞, 連体化, *, *, *, *, の, ノ, ノ
-  #   # うち    名詞, 非自立, 副詞可能, *, *, *, うち, ウチ, ウチ
+  #   # すもも  名詞,一般,*,*,*,*,すもも,スモモ,スモモ
+  #   # も      助詞,係助詞,*,*,*,*,も,モ,モ
+  #   # もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+  #   # も      助詞,係助詞,*,*,*,*,も,モ,モ
+  #   # もも    名詞,一般,*,*,*,*,もも,モモ,モモ
+  #   # の      助詞,連体化,*,*,*,*,の,ノ,ノ
+  #   # うち    名詞,非自立,副詞可能,*,*,*,うち,ウチ,ウチ
   #
   class Tagger
     # Create a new tagger by loading the built-in binary dictionary.
@@ -41,41 +41,36 @@ module Suika
       while start < terminal
         word = sentence[start]
         pos = start
-        is_unknown = true
+        matched = false
         while @trie.match?(word) && pos < terminal
           if @dictionary.key?(word)
+            matched = true
             @dictionary[word].each do |el|
-              lattice.insert(start, start + word.length,
-                             word, el[0].to_i, el[1].to_i, el[2].to_i,
-                             el[3..-1])
+              lattice.insert(start, start + word.length, word, false,
+                             el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
             end
-            is_unknown = false
           end
           pos += 1
           word = sentence[start..pos]
         end
 
-        unless is_unknown
-          start += 1
-          next
-        end
-
         word = sentence[start]
-        char_type = CharDef.char_type(sentence[start])
         char_cate = CharDef.char_category(sentence[start])
-        if char_cate[:group] == 1
-          unk_terminal = char_cate[:length].zero? ? terminal : start + char_cate[:length]
+        unless !char_cate[:invoke] && matched
+          char_length = char_cate[:group] ? CharDef::MAX_GROUPING_SIZE : char_cate[:length]
+          unk_terminal = [start + char_length, terminal].min
           pos = start + 1
+          char_type = CharDef.char_type(sentence[start])
           while pos < unk_terminal && char_type == CharDef.char_type(sentence[pos])
             word << sentence[pos]
             pos += 1
           end
+          @unknown_dictionary[char_type].each do |el|
+            lattice.insert(start, start + word.length, word, true,
+                           el[0].to_i, el[1].to_i, el[2].to_i, el[3..-1])
+          end
         end
-        @unknown_dictionary[char_type].each do |el|
-          lattice.insert(start, start + word.length,
-                         word, el[0].to_i, el[1].to_i, el[2].to_i,
-                         el[3..-1])
-        end
+
         start += 1
       end
 
@@ -111,9 +106,10 @@ module Suika
       prev_node = eos.min_prev
       res = []
       until prev_node.nil?
-        res.append("#{prev_node.surface}\t#{prev_node.attrs.join(', ')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
+        res.append("#{prev_node.surface}\t#{prev_node.attrs.join(',')}") if prev_node.surface != 'BOS' && prev_node.surface != 'EOS'
         prev_node = prev_node.min_prev
       end
+
       res.reverse
     end
   end

data/lib/suika/version.rb

@@ -3,5 +3,5 @@
 # Suika is a Japanese morphological analyzer written in pure Ruby.
 module Suika
   # The version of Suika you are using.
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

data/suika.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/magro/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/yoshoku/suika/blob/master/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://rubydoc.info/gems/suika'
 
   # Specify which files should be added to the gem when it is released.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suika
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-05 00:00:00.000000000 Z
+date: 2020-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rambling-trie
@@ -38,6 +38,7 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - NOTICE.txt
 - README.md
@@ -57,7 +58,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/yoshoku/suika
   source_code_uri: https://github.com/yoshoku/suika
-  changelog_uri: https://github.com/yoshoku/magro/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/yoshoku/suika/blob/master/CHANGELOG.md
   documentation_uri: https://rubydoc.info/gems/suika
 post_install_message:
 rdoc_options: []