okura 0.0.0

data/lib/okura/version.rb

@@ -0,0 +1,3 @@
+module Okura
+  Version = '0.0.0'
+end

data/lib/okura/word_dic.rb

@@ -0,0 +1,216 @@
+module Okura
+  module WordDic
+    class Naive
+      def initialize
+        @size=0
+        @root=TrieNode.new
+      end
+      class TrieNode
+        def initialize
+          @nodes={}
+          @leafs=[]
+        end
+        def add word,i=0
+          if i==word.surface.length
+            @leafs.push word
+          else
+            fst=word.surface[i]
+            node=@nodes[fst]
+            @nodes[fst]=node=TrieNode.new if node.nil?
+            node.add word,i+1
+          end
+        end
+        def find_all str,i,res=Array.new
+          res.concat @leafs
+          return res unless i < str.length
+          node=@nodes[str[i]]
+          return res if node.nil?
+          node.find_all(str,i+1,res)
+          res
+        end
+      end
+      attr_reader :size
+      def define word
+        @size+=1
+        @root.add word
+      end
+      # -> [Word]
+      def possible_words str,i
+        @root.find_all str,i
+      end
+      def word_size
+        @size
+      end
+    end
+    class DoubleArray
+      # Words -> [Integer] -> [Integer]
+      def initialize words,base,check
+        @words,@base,@check=words,base,check
+      end
+      def possible_words str,i
+        ret=[]
+        prev=nil
+        cur=1
+        str[i..-1].bytes.each{|c|
+          next_index=@base[cur]+c+1
+          break unless @check[next_index]==cur
+          prev,cur=cur,next_index
+          # check EOS node
+          eos_index=@base[cur]
+          if @check[eos_index]==cur
+            raise "@base[#{eos_index}] should < 0 but #{@base[eos_index]}" unless @base[eos_index] < 0
+            ret.push -@base[eos_index]-1
+          end
+        }
+        return ret.map{|x|@words.group(x)}.flatten(1)
+      end
+      def word_size
+        @words.word_size
+      end
+      class Builder
+        class DAData
+          def initialize root
+            # offset | +0         | +1       | +2       | ...
+            # data   | -data_id-1 | child(0) | child(1) | ...
+            #
+            # base[0] = -last free cell
+            # check[0] = -first free cell
+            # 1 = root node id
+            @base=[0,0]
+            @check=[0,0]
+            @length=2
+            b,node_id=construct! root
+            @base[1]=b
+          end
+          attr_reader :base
+          attr_reader :check
+          attr_reader :length
+
+          def construct! node,parent=1
+            # base[parent_node_id] should == s
+            # -base[s+0] : data id
+            # s+1+c : child node id for char c
+            # check[m] : parent node id for node m
+            s=find_free_space_for node
+            if node.has_data?
+              alloc! s,parent
+              @base[s]=-node.data_id-1
+            end
+            node.children.each{|c,cn| alloc! child_index(s,c),parent }
+            node.children.each{|c,cn|
+              idx=child_index(s,c)
+              @base[idx]=construct! cn,idx
+            }
+            s
+          end
+          def child_index base,c
+            base+c+1
+          end
+          def alloc! index,parent
+            assert index>0
+            assert free?(index)
+            if length <= index
+              expand!(index+1)
+            end
+            assert has_free_cell?
+
+            prev_free=-@base[index]
+            next_free=-@check[index]
+            @base[next_free]=-prev_free
+            @check[prev_free]=-next_free
+            @base[index]=0 # dummy value
+            @check[index]=parent
+            assert !free?(index)
+          end
+          def expand! size
+            if size <= length
+              return
+            end
+            (length...size).each{|i|
+              if has_free_cell?
+                @base[i]=@base[0]
+                @check[i]=0
+                @check[-@base[0]]=-i
+                @base[0]=-i
+              else
+                @base[i]=0
+                @check[i]=0
+                @base[0]=-i
+                @check[0]=-i
+              end
+            }
+            @length=size
+          end
+          def free? index
+            length <= index || @check[index] <= 0
+          end
+          def find_free_space_for node
+            alloc_indexes=node.children.keys.map{|c|c+1}
+            alloc_indexes+=[0] if node.has_data?
+            return 0 if alloc_indexes.empty?
+            min=alloc_indexes.min
+            i=-@check[0]
+            while i!=0
+              assert free?(i)
+              if 0 < i-min && alloc_indexes.all?{|idx|free?(idx+i-min)}
+                return i-min
+              end
+              i=-@check[i]
+            end
+            # free space not found
+            return [length-min,1].max
+          end
+          def has_free_cell?
+            @base[0]!=0
+          end
+          def assert cond
+            raise unless cond
+          end
+        end
+        class Node
+          def initialize
+            @data_id=nil
+            @children={}
+          end
+          attr_reader :data_id
+          attr_reader :children
+          def has_data?
+            !!data_id
+          end
+          def add bytes,idx,data_id
+            if idx==bytes.length
+              @data_id=data_id
+            else
+              c=bytes[idx]
+              (@children[c]||=Node.new).add(bytes,idx+1,data_id)
+            end
+          end
+        end
+        def initialize
+          @root=Node.new
+          @words=Okura::Words::Builder.new
+        end
+        def define word
+          word_group_id=@words.add word
+          key=word.surface.bytes.to_a
+          @root.add key,0,word_group_id
+        end
+        def build
+          da=DAData.new @root
+          DoubleArray.new *data_for_serialize
+        end
+        # -> [ Words, [Integer], [Integer] ]
+        def data_for_serialize
+          da=DAData.new @root
+          [@words.build,da.base,da.check]
+        end
+        # [ Words, [Integer], [Integer] ] -> WordDic::DoubleArray
+        def self.build_from_serialized data
+          words,base,check=data
+          puts base.length
+          DoubleArray.new words,base,check
+        end
+      end
+    end
+  end
+end

data/lib/okura.rb

@@ -0,0 +1,388 @@
+# coding: utf-8
+require 'okura/word_dic'
+
+module Okura
+  class Tagger
+    def initialize dic,mat
+      @dic,@mat=dic,mat
+    end
+    attr_reader :dic
+    attr_reader :mat
+    # -> [String]
+    def wakati str,mat
+      mincost_path=parse(str).mincost_path
+      return nil if mincost_path.nil?
+      return mincost_path.map{|node|node.word.surface}
+    end
+    # -> Nodes
+    def parse str
+      chars=str.split(//)
+      nodes=Nodes.new(chars.length+2,@mat)
+      nodes.add(0,Node.mk_bos_eos)
+      nodes.add(chars.length+1,Node.mk_bos_eos)
+      str.length.times{|i|
+        @dic.possible_words(str,i).each{|w|
+          nodes.add(i+1,Node.new(w))
+        }
+      }
+      nodes
+    end
+  end
+  class Nodes
+    def initialize len,mat
+      @mat=mat
+      @begins=(0...len).map{[]}
+      @ends=(0...len).map{[]}
+    end
+    def [](i)
+      @begins[i]
+    end
+    def length
+      @begins.length
+    end
+    # Matrix -> [Node] | nil
+    def mincost_path
+      return [] if length==0
+      # calc cost
+      self[0].each{|n|
+        n.total_cost=n.word.cost
+        n.nearest_prev=nil
+      }
+      (1...length).each{|i|
+        prevs=@ends[i-1]
+        curs=@begins[i]
+        prevs.each{|prev|
+          # 途中で行き止まりのNodeはtotal_costが設定されない
+          next if prev.total_cost.nil?
+          curs.each{|cur|
+            join_cost=@mat.cost(prev.word.right.id,cur.word.left.id)
+            next if join_cost.nil?
+            cost=prev.total_cost+join_cost+cur.word.cost
+            if !cur.total_cost || cost < cur.total_cost
+              cur.total_cost=cost
+              cur.nearest_prev=prev
+            end
+          }
+        }
+      }
+      # calc mincost path
+      ret=[]
+      cur=self[-1][0]
+      until cur.nil?
+        ret.push cur
+        cur=cur.nearest_prev
+      end
+      # TODO: disconnected
+      #  return nil unless ...
+      # success
+      return ret.reverse
+    end
+    def add i,node
+      @begins[i].push node
+      @ends[i+node.length-1].push node
+    end
+  end
+  class Node
+    def initialize word
+      @word=word
+      @nearest_prev=nil
+      @total_cost=nil
+    end
+    attr_reader :word
+    attr_accessor :nearest_prev
+    attr_accessor :total_cost
+    def length
+      word.surface.length
+    end
+    def to_s
+      "Node(#{word},#{total_cost})"
+    end
+    def self.mk_bos_eos
+      f=Features::BOS_EOS
+      node=Node.new Word.new('BOS/EOS',f,f,0)
+      def node.length; 1; end
+      node
+    end
+  end
+  class Words
+    class CompactStringArray
+      def initialize str,indices
+        @str=str
+        @indices=indices
+      end
+      def get id
+        raise 'bad id' unless id < @indices.length
+        from=@indices[id]
+        to=(id+1 < @indices.length) ? @indices[id+1] : @str.length
+        @str[from...to]
+      end
+      def [](id)
+        get id
+      end
+      class Builder
+        def initialize
+          @indices=[]
+          @surfaces=[]
+          @size=0
+        end
+        def build
+          Okura::Words::CompactStringArray.new @surfaces.join(''),@indices
+        end
+        def add surface
+          id=@indices.length
+          @indices.push @size
+          @surfaces.push surface
+          @size+=surface.size
+          id
+        end
+      end
+    end
+    class Builder
+      def initialize
+        # group id -> [Word]
+        @groups=[]
+        @next_group_id=0
+        # surface -> id
+        @group_ids={}
+        @surfaces=Okura::Words::CompactStringArray::Builder.new
+        @left_features=Features.new
+        @right_features=Features.new
+        @surface_ids=[]
+        @left_ids=[]
+        @right_ids=[]
+        @costs=[]
+      end
+      def add word
+        unless @group_ids.has_key? word.surface
+          gid=add_group! word.surface
+          wid=add_word! gid,word
+          @group_ids[word.surface]=gid
+          @groups[gid]=[wid]
+          gid
+        else
+          gid=@group_ids[word.surface]
+          wid=add_word! gid,word
+          @groups[gid].push wid
+          gid
+        end
+      end
+      def build
+        Okura::Words.new(
+          @groups,@surfaces.build,@left_features,@right_features,@surface_ids,@left_ids,@right_ids,@costs
+        )
+      end
+      private
+      def add_group! surface
+        group_id=@surfaces.add surface
+        group_id
+      end
+      def add_word! group_id,word
+        wid=@surface_ids.length
+        @surface_ids.push group_id
+        @left_ids.push word.left.id
+        @right_ids.push word.right.id
+        @left_features.add word.left.id,word.left.text
+        @right_features.add word.right.id,word.right.text
+        @costs.push word.cost
+        wid
+      end
+    end
+    def initialize groups,surfaces,left_features,right_features,surface_ids,left_ids,right_ids,costs
+      # group id -> [word id]
+      @groups=groups
+      @surfaces=surfaces
+      @left_features=left_features
+      @right_features=right_features
+      @surface_ids=surface_ids
+      @left_ids=left_ids
+      @right_ids=right_ids
+      @costs=costs
+    end
+    def group group_id
+      @groups[group_id].map{|wid|
+        Word.new(
+          @surfaces[@surface_ids[wid]],
+          @left_features[@left_ids[wid]],
+          @right_features[@right_ids[wid]],
+          @costs[wid]
+        )
+      }
+    end
+    def word_size
+      @groups.inject(0){|a,x|a+x.size}
+    end
+  end
+  class Word
+    def initialize surface,left,right,cost
+      raise "bad feature: #{left.inspect}" unless left.respond_to? :text
+      @surface,@left,@right,@cost=surface,left,right,cost
+    end
+    # String
+    attr_reader :surface
+    # Feature
+    attr_reader :left
+    # Feature
+    attr_reader :right
+    # Integer
+    attr_reader :cost
+    def == other
+      return [surface,left,right,cost] ==
+        [other.surface,other.left,other.right,other.cost]
+    end
+    def hash
+      [surface,left,right,cost].hash
+    end
+    def to_s
+      "Word(#{surface},#{left.id},#{right.id},#{cost})"
+    end
+  end
+  class Feature
+    def initialize id,text
+      @id,@text=id,text
+    end
+    attr_reader :id
+    attr_reader :text
+    def to_s
+      "Feature(#{id},#{text})"
+    end
+    def == other
+      return self.id==other.id
+    end
+    def hash
+      self.id.hash
+    end
+  end
+  class Features
+    def initialize
+      @map_id={}
+    end
+    # Integer -> Feature
+    def from_id id
+      @map_id[id]
+    end
+    def [](id)
+      from_id id
+    end
+    def add id,text
+      @map_id[id]=Feature.new id,text
+    end
+    def size
+      @map_id.size
+    end
+    BOS_EOS=Feature.new 0,'BOS/EOS'
+  end
+  class Dic
+    def initialize word_dic,unk_dic
+      @word_dic,@unk_dic=word_dic,unk_dic
+    end
+    attr_reader :word_dic
+    attr_reader :unk_dic
+    # -> [Word]
+    def possible_words str,i
+      ret=@word_dic.possible_words str,i
+      ret.concat(@unk_dic.possible_words(str,i,!ret.empty?))
+      ret
+    end
+  end
+  class UnkDic
+    # CharTypes -> Features ->
+    def initialize char_types
+      @char_types=char_types
+      # CharType.name => [Word]
+      @templates={}
+    end
+    # -> [Word]
+    def possible_words str,i,found_in_normal_dic
+      ret=[]
+      first_char_type=@char_types.type_for str[i].ord
+      return [] if found_in_normal_dic && !first_char_type.invoke?
+
+      collect_result ret,first_char_type,str[i..i] if first_char_type.length > 0
+
+      l=1
+      str[(i+1)..-1].each_codepoint{|cp|
+        break unless first_char_type.accept? cp
+        l+=1
+        collect_result ret,first_char_type,str[i...(i+l)] if first_char_type.length >= l
+      }
+      collect_result ret,first_char_type,str[i...(i+l)] if first_char_type.group? && first_char_type.length < l
+
+      ret
+    end
+    private
+    def collect_result ret,type,surface
+      (@templates[type.name]||[]).each{|tp|
+        ret.push Word.new surface,tp.left,tp.right,tp.cost
+      }
+    end
+    public
+    # String -> Feature -> Feature -> Integer ->
+    def define type_name,left,right,cost
+      type=@char_types.named type_name
+      (@templates[type_name]||=[]).push Word.new '',left,right,cost
+    end
+    def word_templates_for type_name
+      @templates[type_name].dup
+    end
+    def rule_size
+      @templates.values.inject(0){|sum,t|sum+t.size}
+    end
+  end
+  class CharTypes
+    def initialize
+      @types={}
+      @mapping={}
+      @compat_mapping={}
+    end
+    def type_for charcode
+      @mapping[charcode]||default_type||
+        (raise "Char type for 0x#{charcode.to_s(16)} is not defined,"+
+         " and DEFAULT type is not defined too")
+    end
+    def define_type name,invoke,group,length
+      @types[name]=CharType.new(name,invoke,group,length)
+    end
+    def define_map charcode,type,compat_types
+      @mapping[charcode]=type
+      type.add charcode
+      compat_types.each{|ct|ct.add charcode}
+    end
+    def named name
+      @types[name] || (raise "Undefined char type: #{name}")
+    end
+    def default_type
+      named 'DEFAULT'
+    end
+  end
+  class CharType
+    def initialize name,invoke,group,length
+      @name,@invoke,@group,@length=name,invoke,group,length
+      @accept_charcodes={}
+    end
+    def add charcode
+      @accept_charcodes[charcode]=true
+    end
+    attr_reader :name
+    attr_reader :length
+    def group?; @group; end
+    def invoke?; @invoke; end
+    def accept? charcode
+      @accept_charcodes[charcode]
+    end
+  end
+  class Matrix
+    def initialize rsize,lsize
+      @mat=[nil]*(lsize*rsize)
+      @lsize,@rsize=lsize,rsize
+    end
+    # Feature.id -> Feature.id -> Int
+    def cost rid,lid
+      @mat[rid*lsize+lid]
+    end
+    def set(rid,lid,cost)
+      @mat[rid*lsize+lid]=cost
+    end
+    attr_reader :rsize
+    attr_reader :lsize
+  end
+end