migemo-lib 0.4.3

data/doc/migemo.ja.rd

@@ -0,0 +1,66 @@
+=begin
+= Ruby/Migemo: ローマ字のまま日本語をインクリメンタル検索する Ruby用のライブラリ
+Ruby/Migemo はローマ字のまま日本語をインクリメンタル検索する
+ためのライブラリです。
+
+=== 使用例
+
+  % cat sample.rb
+  require 'migemo'
+
+  dict = MigemoStaticDict.new("migemo-dict")
+  dict_cache  = MigemoDictCache.new("migemo-dict" + ".cache")
+  user_dict = MigemoUserDict.new("user-dict")
+
+  while line = gets
+    pattern = line.chomp
+    migemo = Migemo.new(pattern,dict)
+    migemo.optimization = 3
+    migemo.dict_cache = dict_cache
+    migemo.user_dict = user_dict
+    migemo.type = "ruby"
+    puts migemo.regex
+  end
+
+== API
+
+--- MigemoStaticDict#new(filename)
+    静的な辞書のオブジェクトを生成する
+
+--- MigemoDictCache#new(filename)
+    静的な辞書のキャッシュのオブジェクトを生成する
+
+--- MigemoUserDict#new(filename)
+    ユーザ辞書のオブジェクトを生成する
+
+--- MigemoRegexDict#new(filename)
+    正規表現辞書のオブジェクトを生成する
+
+--- Migemo#new(pattern, dict)
+    Migemoオブジェクトを生成する。dict には
+    MigemoStaticDict オブジェクトかStringを、pattern には検索パター
+    ンを与える
+
+--- Migemo#regex
+    検索パターンを展開した正規表現の文字列を返す。
+
+--- Migemo#type
+    正規表現の種類 (emacs, ruby, perl) を設定する accessor。[ruby]
+
+--- Migemo#dict_cache
+    静的辞書のキャッシュを設定する accessor。
+
+--- Migemo#usr_dict
+    ユーザ辞書のオブジェクトを設定する accessor。
+
+--- Migemo#regex_dict
+    正規表現辞書のオブジェクトを設定する accessor。
+
+--- Migemo#insertion
+    1文字ごとに挟む文字列を設定する accessor。
+
+--- Migemo#optimization
+    正規表現のコンパクト化のレベル (0-3) を設定する accessor。[3]
+
+satoru@namazu.org
+=end

data/lib/migemo-dict.rb

@@ -0,0 +1,126 @@
+#
+# Ruby/Migemo - a library for Japanese incremental search.
+#
+# Copyright (C) 2001 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+
+require 'bsearch'
+require 'migemo/core_ext/string'
+
+class MigemoDictItem
+  def initialize(key, values)
+    @key = key
+    @values = values
+    raise if @key == nil
+    raise if @values == nil
+  end
+
+  attr_reader :key
+  attr_reader :values
+end
+
+class MigemoDict
+  def initialize (filename)
+    @dict  = File.new(filename)
+  end
+
+  def lookup (pattern)
+    pattern = pattern.downcase
+    raise "nil pattern" if pattern == nil
+  end
+
+  private
+  def decompose (line)
+    array = line.chomp.split("\t").delete_if do |x| x == nil end
+    key = array.shift
+    values = array
+    raise if key == nil
+    raise if values == nil
+    return key, values
+  end
+end
+
+class MigemoStaticDict < MigemoDict
+  def initialize (filename)
+    super(filename)
+    @index = File.new(filename + ".idx").read.unpack "N*"
+  end
+
+  def lookup (pattern)
+    range = @index.bsearch_range do |idx| 
+      key, values = decompose(get_line(idx))
+      key.prefix_match(pattern)
+    end
+    if range 
+      range.each do |i| 
+        key, values = decompose(get_line(@index[i]))
+        yield(MigemoDictItem.new(key, values)) 
+      end
+    end
+  end
+
+  private
+  def get_line (index)
+    @dict.seek(index)
+    @dict.gets
+  end
+end
+
+class MigemoUserDict < MigemoDict
+  def initialize (filename)
+    super(filename)
+    @lines = @dict.readlines.delete_if {|x| /^;/ =~ x}.sort
+  end
+
+  def lookup (pattern)
+    range = @lines.bsearch_range do |line| 
+      key, values = decompose(line)
+      key.prefix_match(pattern)
+    end
+    if range 
+      range.each do |i| 
+        key, values = decompose(@lines[i])
+        yield(MigemoDictItem.new(key, values)) 
+      end
+    end
+  end
+end
+
+class MigemoRegexDict < MigemoUserDict
+end
+
+class MigemoDictCache
+  def initialize (filename)
+    @dict  = File.new(filename)
+    @index = File.new(filename + ".idx").read.unpack "N*"
+  end
+
+  def decompose (idx)
+    @dict.seek(idx)
+    keylen = @dict.read(4).unpack("N").first
+    key = @dict.read(keylen).unpack("a*").first
+    datalen  = @dict.read(4).unpack("N").first
+    data     = Marshal.load(@dict.read(datalen))
+    return key, data
+  end
+  private :decompose
+
+  def lookup (pattern)
+    raise if pattern == nil
+    pattern = pattern.downcase
+    idx = @index.bsearch_first do |_idx|
+      key, data = decompose(_idx)
+      key <=> pattern
+    end
+    if idx
+      key, data = decompose(@index[idx])
+      return data
+    else
+      nil
+    end
+  end
+end

data/lib/migemo-regex.rb

@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*-
+#
+# Ruby/Migemo - a library for Japanese incremental search.
+#
+# Copyright (C) 2001 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+
+module MigemoRegex
+  class RegexAlternation < Array
+    def sort
+      self.clone.replace(super)
+    end
+
+    def uniq
+      self.clone.replace(super)
+    end
+
+    def map
+      self.clone.replace(super {|x| yield(x)})
+    end
+
+    def delete_if
+      self.clone.replace(super {|x| yield(x)})
+    end
+
+    def select
+      self.clone.replace(super {|x| yield(x)})
+    end
+  end
+
+  class RegexConcatnation < Array
+    def map
+      self.clone.replace(super {|x| yield(x)})
+    end
+  end
+
+  class RegexCharClass < Array
+  end
+
+  class RegexCompiler
+    def initialize
+      @regex = RegexAlternation.new
+    end
+    attr_reader :regex
+
+    def push (item)
+      if item and item != ""
+        @regex.push(item)
+      end
+    end
+
+    def uniq
+      @regex.uniq
+    end
+
+    def optimize (level)
+      @regex = optimize1(@regex) if level >= 1
+      @regex = optimize2(@regex) if level >= 2
+      @regex = optimize3(@regex) if level >= 3
+    end
+
+    private
+    # ["運", "運動", "運転", "日本", "日本語"] => ["安" "運" "日本"]
+    # (運|運動|運転|日本|日本語) => (安|運|日本)
+    def optimize1 (regex)
+      prefixpat = nil
+      sorted = (defined?(Encoding)) ? regex.sort_by{|s| s.encode("EUC-JP") } : regex.sort
+      sorted.select do |word|
+        if prefixpat && prefixpat.match(word) then
+          false # excluded
+        else
+          prefixpat = Regexp.new("^" + Regexp.quote(word))
+          true # included
+        end
+      end
+    end
+
+    # (あああ|ああい|ああう)
+    # => (あ(あ(あ|い|う)))
+    def optimize2 (regex)
+      tmpregex = (defined?(Encoding)) ? regex.sort_by{|s| s.encode("EUC-JP") }.clone : regex.sort.clone # I wish Array#cdr were available...
+      optimized = RegexAlternation.new
+      until tmpregex.empty?
+        head = tmpregex.shift
+        initial = head.first
+        friends = RegexAlternation.new
+        while item = tmpregex.first
+          if initial == item.first
+            friends.push(item.rest)
+            tmpregex.shift
+          else
+            break
+          end
+        end
+        if friends.empty?
+          optimized.push head
+        else
+          concat = RegexConcatnation.new
+          concat.push(initial)
+          friends.unshift(head.rest) 
+          concat.push(optimize2(friends))
+          optimized.push(concat)
+        end
+      end
+      return optimized
+    end
+
+    # (あ|い|う|え|お)
+    # => [あいうえお]
+    def optimize3 (regex)
+      charclass = RegexCharClass.new
+      if regex.instance_of?(RegexAlternation)
+        regex.delete_if do |x|
+          if x.instance_of?(String) && x =~ /^.$/ then
+            charclass.push(x)
+            true
+          end
+        end
+      end
+
+      if charclass.length == 1
+        regex.unshift charclass.first
+      elsif charclass.length > 1
+        regex.unshift charclass
+      end
+
+      regex.map do |x|
+        if x.instance_of?(RegexAlternation) || x.instance_of?(RegexConcatnation)
+          optimize3(x)
+        else
+          x
+        end
+      end
+    end
+  end
+
+  class RegexMetachars
+    def initialize
+      @bar = '|'
+      @lparen = '('
+      @rparen = ')'
+    end
+    attr_accessor :bar
+    attr_accessor :lparen
+    attr_accessor :rparen
+  end
+
+  class RegexEgrepMetachars < RegexMetachars
+  end
+
+  class RegexPerlMetachars < RegexMetachars
+    def initialize
+      @bar = '|'
+      @lparen = '(?:'
+      @rparen = ')'
+    end
+  end
+
+  class RegexRubyMetachars < RegexMetachars
+  end
+
+  class RegexEmacsMetachars < RegexMetachars
+    def initialize
+      @bar = '\\|'
+      @lparen = '\\('
+      @rparen = '\\)'
+    end
+  end
+
+  class RegexRenderer
+    def initialize (regex, insertion)
+      raise if regex == nil
+      @regex = regex
+      @meta = RegexMetachars.new
+      @insertion = insertion
+      @with_paren = false
+    end
+    attr_accessor :with_paren
+
+    def render
+      if @with_paren  # e.g. "(a|b|c)"
+        render0(@regex)
+      else            # e.g. "a|b|c"
+        @regex.map do |x|
+          render0(x)
+        end.join @meta.bar
+      end
+    end
+
+    def join_regexes (string, regexes)
+      ([string] + regexes).join @meta.bar
+    end
+
+    private
+    def render_alternation (regex)
+      if regex.length == 0
+        raise
+      elsif regex.length == 1
+        render0(regex[0])
+      else
+        @meta.lparen + 
+          regex.map {|x| render0(x) }.join(@meta.bar) + 
+          @meta.rparen
+      end
+    end
+
+    def render_concatnation (regex)
+      regex.map {|x| render0(x) }.join(@insertion)
+    end
+
+    # We don't use Regexp.quote because the following regex
+    # is more general (not ruby-specific) and safe to use.
+    def escape_string (string)
+      string.gsub(/([\x00-\x1f\x21-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f])/, '\\\\\\1')
+    end
+
+    def escape_charclass (string)
+      string.gsub(/\\/, '\\\\\\')
+    end
+
+    def render_charclass (regex)
+      if regex.delete("-")
+        regex.push("-")  # move "-" to the end of Array.
+      end
+      if regex.delete("]")
+        regex.unshift("]")  # move "]" to the beginning of Array.
+      end
+      escape_charclass("[" + regex.join + "]")
+    end
+
+    def insert (string)
+      if @insertion != ""
+        tmp = string.gsub(/(\\.|.)/, "\\1#{@insertion}")
+        tmp = tmp.sub(/#{Regexp.quote(@insertion)}$/, "")
+      else
+        string
+      end
+    end
+
+    def render_string (regex)
+      insert(escape_string(regex))
+    end
+
+    def render0 (x)
+      if x.instance_of?(RegexAlternation)
+        render_alternation(x)
+      elsif x.instance_of?(RegexConcatnation)
+        render_concatnation(x)
+      elsif x.instance_of?(RegexCharClass)
+        render_charclass(x)
+      elsif x.instance_of?(String)
+        render_string(x)
+      else
+        raise "unexpected type: #{x} of #{x.class}"
+      end
+    end
+  end
+
+  class RegexPerlRenderer < RegexRenderer
+    def initialize (regex, insertion)
+      super(regex, insertion)
+      @meta = RegexPerlMetachars.new
+    end
+  end
+
+  class RegexRubyRenderer < RegexPerlRenderer
+  end
+
+  class RegexEgrepRenderer < RegexRenderer
+  end
+
+  class RegexEmacsRenderer < RegexRenderer
+    def initialize (regex, insertion)
+      super(regex, insertion)
+      @meta = RegexEmacsMetachars.new
+    end
+
+    def escape_string (string)
+      str = Regexp.quote(string)
+      str.gsub!(/\\\(/, "(")
+      str.gsub!(/\\\)/, ")")
+      str.gsub!(/\\\|/, "|")
+      str.gsub!(/\\\</, "<")
+      str.gsub!(/\\\>/, ">")
+      str.gsub!(/\\\=/, "=")
+      str.gsub!(/\\\'/, "'")
+      str.gsub!(/\\\`/, "`")
+      str.gsub!(/\\\{/, "{")
+      str
+    end
+
+    def escape_charclass (string)
+      string
+    end
+  end
+
+  module RegexMetacharsFactory
+    def new (type)
+      case type
+      when nil
+        RegexRubyMetachars.new
+      when "emacs"
+        RegexEmacsMetachars.new
+      when "perl"
+        RegexPerlMetachars.new
+      when "ruby"
+        RegexRubyMetachars.new
+      when "egrep"
+        RegexEgrepMetachars.new
+      else
+        raise "Unknown type: #{type}"
+      end
+    end
+    module_function :new
+  end
+
+  module RegexRendererFactory
+    def new (regex, type, insertion)
+      case type
+      when nil
+        RegexRubyRenderer.new(regex, insertion)
+      when "emacs"
+        RegexEmacsRenderer.new(regex, insertion)
+      when "perl"
+        RegexPerlRenderer.new(regex, insertion)
+      when "ruby"
+        RegexRubyRenderer.new(regex, insertion)
+      when "egrep"
+        RegexEgrepRenderer.new(regex, insertion)
+      else
+        raise "Unknown type: #{regex}"
+      end
+    end
+    module_function :new
+  end
+end