okura 0.0.0 → 0.0.1

data/bin/okura

@@ -0,0 +1,83 @@
+#!/bin/env ruby
+# -*- filetype:ruby -*-
+
+require 'rubygems'
+
+commands={
+  'console' => lambda {|args|
+	require 'okura/console'
+	raise IllegalOption.new('Usage: console dict_dir') unless args.length==1
+	dict_dir=args[0]
+	Okura::Console.new.run_console dict_dir
+  },
+  'compile' => lambda {|args|
+	require 'okura/serializer'
+    require 'optparse'
+    format_info=Okura::Serializer::FormatInfo.new
+    opt=OptionParser.new
+    dry=false
+    opt.on('--word_dic=TYPE'){|t| format_info.word_dic=t.to_sym}
+    opt.on('--dry'){|d|dry=d}
+    opt.on('--encoding=ENCODING'){|encoding| format_info.encoding=encoding}
+    opt.parse! args
+	raise IllegalOption.new('Usage: compile dict_dir dest_dir') unless args.length==2
+    dict_dir=args[0]
+    bin_dir=args[1]
+    puts <<-EOS
+Settings
+- File Encoding: #{format_info.encoding}
+- Format:
+  - word_dic: #{format_info.word_dic}
+  - unk_dic: #{format_info.unk_dic}
+  - features: #{format_info.features}
+  - char_types: #{format_info.char_types}
+  - matrix: #{format_info.matrix}
+    EOS
+    next if dry
+    Dir.mkdir bin_dir unless File.exists? bin_dir
+    format_info.compile_dict(dict_dir,bin_dir)
+  },
+  'help' => lambda {|args|
+	puts <<-EOS
+USAGE:
+  okura command [args]
+  okura --version
+command:
+  compile src_dir dest_dir
+  console dict_dir
+  help
+	EOS
+  }
+}
+
+class IllegalOption < StandardError
+  def initialize msg
+	@message=msg
+  end
+  attr_reader :message
+end
+
+if $*.empty?
+  commands['help'].call []
+else
+  command=$*.first
+  if command == '--version'
+    require 'okura/version'
+    puts "Okura #{Okura::Version}"
+    exit 0
+  end
+  unless commands.has_key? command
+    puts "unknown command: #{command}"
+	commands[:help].call []
+	exit 1
+  end
+  begin
+	commands[command].call $*[1..-1]
+    exit 0
+  rescue IllegalOption
+	$stderr.puts $!.message
+	exit 1
+  end
+end
+
+

data/lib/okura.rb

@@ -113,8 +113,8 @@ module Okura
       def get id
         raise 'bad id' unless id < @indices.length
         from=@indices[id]
-        to=(id+1 < @indices.length) ? @indices[id+1] : @str.length
-        @str[from...to]
+        to=(id+1 < @indices.length) ? @indices[id+1] : @str.bytesize
+        (from...to).map{|i|@str.getbyte(i)}.pack('C*').force_encoding 'UTF-8'
       end
       def [](id)
         get id
@@ -132,7 +132,7 @@ module Okura
           id=@indices.length
           @indices.push @size
           @surfaces.push surface
-          @size+=surface.size
+          @size+=surface.bytesize
           id
         end
       end
@@ -258,7 +258,7 @@ module Okura
     end
     # Integer -> Feature
     def from_id id
-      @map_id[id]
+      @map_id[id] || (raise "Features: ID undefined (#{id})")
     end
     def [](id)
       from_id id

data/lib/okura/serializer.rb

@@ -30,11 +30,17 @@ module Okura
           }
         }
 
+        features_r=open_src(src_dir,'right-id.def'){|src|
+          open_dest(bin_dir,'right-id.bin'){|dest|
+            serializer_for('Features',features).compile(src,dest)
+          }
+        }
+
         word_src_files=
           Dir.chdir(src_dir){ Dir.glob('*.csv') }.
           map{|file|File.join(src_dir,file)}
         open_dest(bin_dir,'word_dic.bin'){|dest|
-          serializer_for('WordDic',word_dic).compile(features_l,word_src_files,encoding,dest)
+          serializer_for('WordDic',word_dic).compile(features_l,features_r,word_src_files,encoding,dest)
         }
 
         char_types=open_src(src_dir,'char.def'){|src|
@@ -45,7 +51,7 @@ module Okura
 
         open_src(src_dir,'unk.def'){|src|
           open_dest(bin_dir,'unk_dic.bin'){|dest|
-            serializer_for('UnkDic',unk_dic).compile(char_types,features_l,src,dest)
+            serializer_for('UnkDic',unk_dic).compile(char_types,features_l,features_r,src,dest)
           }
         }
 
@@ -64,6 +70,9 @@ module Okura
         features_l=open_bin(bin_dir,'left-id.bin'){|bin|
           serializer_for('Features',features).load(bin)
         }
+        features_r=open_bin(bin_dir,'right-id.bin'){|bin|
+          serializer_for('Features',features).load(bin)
+        }
         wd=open_bin(bin_dir,'word_dic.bin'){|f|
           serializer_for('WordDic',word_dic).load(f)
         }
@@ -142,15 +151,15 @@ module Okura
         }
       end
       class Naive
-        def compile(features,inputs,encoding,output)
+        def compile(features_l,features_r,inputs,encoding,output)
           dic=Okura::WordDic::Naive.new
           Okura::Serializer::WordDic.each_input(inputs,encoding){|input|
             parser=Okura::Parser::Word.new(input)
             parser.each{|surface,lid,rid,cost|
               word=Okura::Word.new(
                 surface,
-                features.from_id(lid),
-                features.from_id(rid),
+                features_l.from_id(lid),
+                features_r.from_id(rid),
                 cost
               )
               dic.define word
@@ -163,16 +172,16 @@ module Okura
         end
       end
       class DoubleArray
-        def compile(features,inputs,encoding,output)
-          puts 'loading'
+        def compile(features_l,features_r,inputs,encoding,output)
+          puts 'loading...'
           dic=Okura::WordDic::DoubleArray::Builder.new
           Okura::Serializer::WordDic.each_input(inputs,encoding){|input|
             parser=Okura::Parser::Word.new(input)
             parser.each{|surface,lid,rid,cost|
               word=Okura::Word.new(
                 surface,
-                features.from_id(lid),
-                features.from_id(rid),
+                features_l.from_id(lid),
+                features_r.from_id(rid),
                 cost
               )
               dic.define word
@@ -182,7 +191,7 @@ module Okura
           writer=Okura::Serializer::BinaryWriter.new output
           words,base,check=dic.data_for_serialize
           raise 'base.length!=check.length' if base.length!=check.length
-          puts 'serialize words'
+          puts 'writing words...'
           words.instance_eval do
             writer.write_object @groups
             writer.write_object @left_features
@@ -191,13 +200,12 @@ module Okura
             writer.write_int32_array @right_ids
             writer.write_int32_array @costs
             writer.write_int32_array @surface_ids
-            puts 'serialize surfaces'
             @surfaces.instance_eval do
               writer.write_object @str
               writer.write_int32_array @indices
             end
           end
-          puts 'serialize DAT indices'
+          puts 'writing word index...'
           writer.write_int32_array base
           writer.write_int32_array check
         end
@@ -255,11 +263,11 @@ module Okura
     end
     module UnkDic
       class Marshal
-        def compile(char_types,features,input,output)
+        def compile(char_types,features_l,features_r,input,output)
           unk=Okura::UnkDic.new char_types
           parser=Okura::Parser::UnkDic.new input
           parser.each{|type_name,lid,rid,cost|
-            unk.define type_name,features.from_id(lid),features.from_id(rid),cost
+            unk.define type_name,features_l.from_id(lid),features_r.from_id(rid),cost
           }
           ::Marshal.dump(unk,output)
         end

data/test/okura_spec.rb

@@ -191,16 +191,12 @@ Z,9,10,5244,記号,空白,*,*,*,*,*
         tagger.dic.word_dic.word_size.should == 3
         tagger.mat.cost(0,1).should == 5
 
-        pending {
-          w2=tagger.dic.word_dic.possible_words('w2',0)[0]
-          w2.left.name.should == 'F5'
-          w2.right.name.should == 'F6'
-        }
-        penging {
-          u1=tagger.dic.unk_dic.word_templates_for('A')
-          u1.left.name.should == 'F5'
-          u1.right.name.should == 'F6'
-        }
+        w2=tagger.dic.word_dic.possible_words('w2',0)[0]
+        w2.left.text.should == 'F5'
+        w2.right.text.should == 'F6'
+        u1=tagger.dic.unk_dic.word_templates_for('A')[0]
+        u1.left.text.should == 'F5'
+        u1.right.text.should == 'F6'
       }
     end
   end
@@ -272,17 +268,18 @@ TYPE3   0 1 3
     # subject : Serializer class
     it 'コンパイルして復元できる' do
       serializer=subject.new
-      features=Okura::Features.new
-      features.add 854,f(854)
-      features.add 458,f(458)
-      features.add 645,f(645)
-      features.add 546,f(546)
+      features_l=Okura::Features.new
+      features_l.add 854,f(854)
+      features_l.add 645,f(645)
+      features_r=Okura::Features.new
+      features_r.add 458,f(458)
+      features_r.add 546,f(546)
       out=StringIO.new
       src=<<-EOS
 あがなう,854,458,6636,動詞,自立,*,*,五段・ワ行促音便,基本形,あがなう,アガナウ,アガナウ,あがなう/購う/贖う,
 あがめる,645,546,1234,動詞,自立,*,*,一段,基本形,あがめる,アガメル,アガメル,あがめる/崇める,
       EOS
-      serializer.compile(features,[as_io(src)],'UTF-8',out)
+      serializer.compile(features_l,features_r,[as_io(src)],'UTF-8',out)
       out.rewind
       wd=serializer.load(out)
 
@@ -307,13 +304,14 @@ TYPE3   0 1 3
       cts.define_type 'Z',false,true,0
       cts.define_map 0x0001,cts.named('A'),[]
       cts.define_map 0x0002,cts.named('Z'),[]
-      features=Okura::Features.new
-      features.add 5,'F5'
-      features.add 6,'F6'
-      features.add 9,'F9'
-      features.add 10,'F10'
+      features_l=Okura::Features.new
+      features_l.add 5,'F5'
+      features_l.add 9,'F9'
+      features_r=Okura::Features.new
+      features_r.add 6,'F6'
+      features_r.add 10,'F10'
       out=StringIO.new
-      serializer.compile(cts,features,as_io(<<-EOS),out)
+      serializer.compile(cts,features_l,features_r,as_io(<<-EOS),out)
 A,5,6,3274,記号,一般,*,*,*,*,*
 Z,9,10,5244,記号,空白,*,*,*,*,*
       EOS

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: okura
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -45,7 +45,8 @@ dependencies:
         version: 0.5.4
 description: Pure ruby morpheme analyzer, using MeCab format dic
 email: discommunicative@gmail.com
-executables: []
+executables:
+- okura
 extensions: []
 extra_rdoc_files: []
 files:
@@ -58,6 +59,7 @@ files:
 - test/okura_spec.rb
 - test/spec_helper.rb
 - test/words_spec.rb
+- bin/okura
 homepage: https://github.com/todesking/okura
 licenses: []
 post_install_message: