okura 0.0.0

data/lib/okura/console.rb

@@ -0,0 +1,23 @@
+require 'okura'
+require 'okura/serializer'
+
+module Okura
+  class Console
+	def run_console dict_dir
+	  tagger=Okura::Serializer::FormatInfo.create_tagger(dict_dir)
+	  print 'okura> '
+	  while $stdin.gets
+		nodes=tagger.parse($_.strip)
+		(0...nodes.length).each{|i|
+		  puts nodes[i].map{|n|"#{n.word.surface}\t#{n.word.right.text} #{n.word.cost}"}
+		  puts
+		}
+		nodes.mincost_path.each{|n|
+		  puts "#{n.word.surface}\t#{n.word.right.text}"
+		}
+		print 'okura> '
+	  end
+	  return 0
+	end
+  end
+end

data/lib/okura/parser.rb

@@ -0,0 +1,131 @@
+module Okura
+  module Parser
+
+    def parse_error line
+      raise 'parse error: '+line
+    end
+
+    module Base
+
+      def initialize io
+        @io=io
+      end
+
+      include Enumerable
+
+      def each &b
+        return Enumerator.new(self) unless b
+
+        @io.each_line {|line|
+          b.call *parse(line)
+        }
+      end
+    end
+
+    class Matrix
+      include Base
+
+      def initialize io
+        @io=io
+        @rid_size,@lid_size=io.readline.split(/\s/).map(&:to_i)
+      end
+
+      attr_reader :rid_size
+      attr_reader :lid_size
+
+      def parse line
+        rid,lid,cost=line.split(/\s/).map(&:to_i)
+        [rid,lid,cost]
+      end
+    end
+
+    class Word
+      include Base
+      def parse line
+        ti,ts=:to_i,:to_s
+        cols=line.split /,/
+        cols[0..3].zip([ts,ti,ti,ti]).map{|v,f|f.to_proc.call v}
+      end
+    end
+
+    class Feature
+      include Base
+      def parse line
+        id_s,name=line.strip.split(/ /,2)
+        id=id_s.to_i
+        [id,name]
+      end
+    end
+
+    class UnkDic
+      include Base
+      def parse line
+        type_s,lid_s,rid_s,cost_s,additional=line.split(/,/,5)
+        lid,rid,cost=[lid_s,rid_s,cost_s].map(&:to_i)
+        [type_s,lid,rid,cost]
+      end
+    end
+
+    class CharType
+      def initialize
+        @callbacks={
+          :mapping_single=>[],
+          :mapping_range=>[],
+          :define_type=>[]
+        }
+      end
+
+      def on_mapping_single &b
+        @callbacks[:mapping_single] << b
+      end
+
+      def on_mapping_range &b
+        @callbacks[:mapping_range] << b
+      end
+
+      def on_chartype_def &b
+        @callbacks[:define_type] << b
+      end
+
+      def parse_all io
+        io.each_line {|line|
+          parse line
+        }
+      end
+
+      def parse line
+        cols=line.gsub(/\s*#.*$/,'').split(/\s+/)
+          return if cols.empty?
+        case cols[0]
+        when /^0x([0-9a-fA-F]{4})(?:\.\.0x([0-9a-fA-F]{4}))?$/
+          # mapping
+          parse_error line unless cols.size >= 2
+          type=cols[1]
+          compat_types=cols[2..-1]
+          from=$1.to_i(16)
+          if $2
+            # mapping(range)
+            to=$2.to_i(16)
+            @callbacks[:mapping_range].each{|c|
+              c.call from,to,type,compat_types
+            }
+          else
+            # mapping(single)
+            @callbacks[:mapping_single].each{|c|
+              c.call from,type,compat_types
+            }
+          end
+        when /^\w+$/
+          parse_error line unless cols.size == 4
+          # typedef
+          @callbacks[:define_type].each{|c|
+            c.call cols[0],(cols[1]=='1'),(cols[2]=='1'),Integer(cols[3])
+          }
+        else
+          # error
+          parse_error line
+        end
+      end
+    end
+  end
+end

data/lib/okura/serializer.rb

@@ -0,0 +1,317 @@
+require 'yaml'
+require 'okura'
+require 'okura/parser'
+
+module Okura
+  module Serializer
+    # 辞書ファイルのコンパイル形式を表現し､コンパイルとロードの制御を担当する
+    class FormatInfo
+      def initialize
+        @word_dic=:DoubleArray
+        @unk_dic=:Marshal
+        @features=:Marshal
+        @char_types=:Marshal
+        @matrix=:Marshal
+        @encoding='EUC-JP'
+      end
+      attr_accessor :word_dic
+      attr_accessor :unk_dic
+      attr_accessor :features
+      attr_accessor :char_types
+      attr_accessor :matrix
+      attr_accessor :encoding
+
+      # 指定されたディレクトリにあるソースをコンパイルする
+      def compile_dict src_dir,bin_dir
+        open_dest(bin_dir,'format-info'){|dest| self.compile dest}
+        features_l=open_src(src_dir,'left-id.def'){|src|
+          open_dest(bin_dir,'left-id.bin'){|dest|
+            serializer_for('Features',features).compile(src,dest)
+          }
+        }
+
+        word_src_files=
+          Dir.chdir(src_dir){ Dir.glob('*.csv') }.
+          map{|file|File.join(src_dir,file)}
+        open_dest(bin_dir,'word_dic.bin'){|dest|
+          serializer_for('WordDic',word_dic).compile(features_l,word_src_files,encoding,dest)
+        }
+
+        char_types=open_src(src_dir,'char.def'){|src|
+          open_dest(bin_dir,'char_types.bin'){|dest|
+            serializer_for('CharTypes',@char_types).compile(src,dest)
+          }
+        }
+
+        open_src(src_dir,'unk.def'){|src|
+          open_dest(bin_dir,'unk_dic.bin'){|dest|
+            serializer_for('UnkDic',unk_dic).compile(char_types,features_l,src,dest)
+          }
+        }
+
+        open_src(src_dir,'matrix.def'){|src|
+          open_dest(bin_dir,'matrix.bin'){|dest|
+            serializer_for('Matrix',matrix).compile(src,dest)
+          }
+        }
+      end
+      # 指定されたディレクトリにあるコンパイル済み辞書をロードし､Taggerを作成する
+      def self.create_tagger bin_dir
+        format_info=File.open(File.join(bin_dir,'format-info')){|f| self.load f }
+        format_info.create_tagger bin_dir
+      end
+      def create_tagger bin_dir
+        features_l=open_bin(bin_dir,'left-id.bin'){|bin|
+          serializer_for('Features',features).load(bin)
+        }
+        wd=open_bin(bin_dir,'word_dic.bin'){|f|
+          serializer_for('WordDic',word_dic).load(f)
+        }
+        ud=open_bin(bin_dir,'unk_dic.bin'){|f|
+          serializer_for('UnkDic',unk_dic).load(f)
+        }
+        mat=open_bin(bin_dir,'matrix.bin'){|f|
+          serializer_for('Matrix',matrix).load(f)
+        }
+        dic=Okura::Dic.new wd,ud
+        tagger=Okura::Tagger.new dic,mat
+        tagger
+      end
+      # このFormatInfoオブジェクトをシリアライズする
+      def compile io
+        YAML.dump({
+          word_dic: word_dic,
+          unk_dic: unk_dic,
+          features: features,
+          char_types: char_types,
+          matrix: matrix
+        },io)
+      end
+      # シリアライズされたFormatInfoオブジェクトを復元する
+      def self.load io
+        data=YAML.load(io)
+        fi=FormatInfo.new
+        fi.word_dic=data[:word_dic]
+        fi.unk_dic=data[:unk_dic]
+        fi.features=data[:features]
+        fi.char_types=data[:char_types]
+        fi.matrix=data[:matrix]
+        fi
+      end
+      private
+      def open_src dir,filename,&block
+        File.open(File.join(dir,filename),"r:#{encoding}:UTF-8",&block)
+      end
+      def open_dest dir,filename,&block
+        File.open(File.join(dir,filename),'wb:ASCII-8BIT',&block)
+      end
+      def open_bin dir,filename,&block
+        File.open(File.join(dir,filename),'rb:ASCII-8BIT',&block)
+      end
+      def serializer_for data_type_name,format_type_name
+        data_type=Okura::Serializer.const_get data_type_name
+        format_type=data_type.const_get format_type_name
+        format_type.new
+      end
+    end
+    module Features
+      class Marshal
+        def compile(input,output)
+          parser=Okura::Parser::Feature.new input
+          features=Okura::Features.new
+          parser.each{|id,text|
+            features.add id,text
+          }
+          ::Marshal.dump(features,output)
+          features
+        end
+        def load(io)
+          ::Marshal.load(io)
+        end
+      end
+    end
+    module WordDic
+      def self.each_input inputs,encoding,&block
+        inputs.each{|input|
+          case input
+          when String
+            File.open(input,"r:#{encoding}:UTF-8",&block)
+          else
+            block.call input
+          end
+        }
+      end
+      class Naive
+        def compile(features,inputs,encoding,output)
+          dic=Okura::WordDic::Naive.new
+          Okura::Serializer::WordDic.each_input(inputs,encoding){|input|
+            parser=Okura::Parser::Word.new(input)
+            parser.each{|surface,lid,rid,cost|
+              word=Okura::Word.new(
+                surface,
+                features.from_id(lid),
+                features.from_id(rid),
+                cost
+              )
+              dic.define word
+            }
+          }
+          Marshal.dump(dic,output)
+        end
+        def load(io)
+          Marshal.load(io)
+        end
+      end
+      class DoubleArray
+        def compile(features,inputs,encoding,output)
+          puts 'loading'
+          dic=Okura::WordDic::DoubleArray::Builder.new
+          Okura::Serializer::WordDic.each_input(inputs,encoding){|input|
+            parser=Okura::Parser::Word.new(input)
+            parser.each{|surface,lid,rid,cost|
+              word=Okura::Word.new(
+                surface,
+                features.from_id(lid),
+                features.from_id(rid),
+                cost
+              )
+              dic.define word
+            }
+          }
+
+          writer=Okura::Serializer::BinaryWriter.new output
+          words,base,check=dic.data_for_serialize
+          raise 'base.length!=check.length' if base.length!=check.length
+          puts 'serialize words'
+          words.instance_eval do
+            writer.write_object @groups
+            writer.write_object @left_features
+            writer.write_object @right_features
+            writer.write_int32_array @left_ids
+            writer.write_int32_array @right_ids
+            writer.write_int32_array @costs
+            writer.write_int32_array @surface_ids
+            puts 'serialize surfaces'
+            @surfaces.instance_eval do
+              writer.write_object @str
+              writer.write_int32_array @indices
+            end
+          end
+          puts 'serialize DAT indices'
+          writer.write_int32_array base
+          writer.write_int32_array check
+        end
+        def load(io)
+          reader=Okura::Serializer::BinaryReader.new io
+          words=begin
+                  groups=reader.read_object
+                  left_features=reader.read_object
+                  right_features=reader.read_object
+                  left_ids=reader.read_int32_array
+                  right_ids=reader.read_int32_array
+                  costs=reader.read_int32_array
+                  surface_ids=reader.read_int32_array
+                  surfaces=begin
+                             str=reader.read_object
+                             indices=reader.read_int32_array
+                             Okura::Words::CompactStringArray.new str,indices
+                           end
+                  Okura::Words.new(
+                    groups,surfaces,left_features,right_features,surface_ids,left_ids,right_ids,costs
+                  )
+                end
+          base=reader.read_int32_array
+          check=reader.read_int32_array
+          Okura::WordDic::DoubleArray::Builder.build_from_serialized [words,base,check]
+        end
+      end
+    end
+    module CharTypes
+      class Marshal
+        def compile(input,output)
+          cts=Okura::CharTypes.new
+
+          parser=Okura::Parser::CharType.new
+          parser.on_chartype_def{|name,invoke,group,length|
+            cts.define_type(name,invoke,group,length)
+          }
+          parser.on_mapping_single{|char,type,ctypes|
+            cts.define_map char,cts.named(type),ctypes.map{|ct|cts.named(ct)}
+          }
+          parser.on_mapping_range{|from,to,type,ctypes|
+            (from..to).each{|char|
+              cts.define_map char,cts.named(type),ctypes.map{|ct|cts.named(ct)}
+            }
+          }
+          parser.parse_all input
+
+          ::Marshal.dump(cts,output)
+          cts
+        end
+        def load(io)
+          ::Marshal.load(io)
+        end
+      end
+    end
+    module UnkDic
+      class Marshal
+        def compile(char_types,features,input,output)
+          unk=Okura::UnkDic.new char_types
+          parser=Okura::Parser::UnkDic.new input
+          parser.each{|type_name,lid,rid,cost|
+            unk.define type_name,features.from_id(lid),features.from_id(rid),cost
+          }
+          ::Marshal.dump(unk,output)
+        end
+        def load(io)
+          ::Marshal.load(io)
+        end
+      end
+    end
+    module Matrix
+      class Marshal
+        def compile(input,output)
+          parser=Okura::Parser::Matrix.new input
+          mat=Okura::Matrix.new parser.rid_size,parser.lid_size
+          parser.each{|rid,lid,cost|
+            mat.set(rid,lid,cost)
+          }
+          ::Marshal.dump(mat,output)
+        end
+        def load(io)
+          ::Marshal.load(io)
+        end
+      end
+    end
+    class BinaryReader
+      def initialize io
+        @io=io
+      end
+      def read_int32
+        @io.read(4).unpack('l').first
+      end
+      def read_int32_array
+        size=read_int32
+        @io.read(4*size).unpack('l*')
+      end
+      def read_object
+        Marshal.load @io
+      end
+    end
+    class BinaryWriter
+      def initialize io
+        @io=io
+      end
+      def write_int32 value
+        @io.write [value].pack('l')
+      end
+      def write_int32_array value
+        write_int32 value.length
+        @io.write value.pack('l*')
+      end
+      def write_object obj
+        Marshal.dump obj,@io
+      end
+    end
+  end
+end