migemo-lib 0.4.3

data/lib/migemo.rb

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+#
+# Ruby/Migemo - a library for Japanese incremental search.
+#
+# Copyright (C) 2001 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+
+$LOAD_PATH << File.dirname(File.expand_path(__FILE__))
+require 'migemo/core_ext/string'
+require 'migemo-dict'
+require 'migemo-regex'
+require 'migemo/version'
+require 'romkan'
+include MigemoRegex
+
+
+class Migemo
+  def initialize (pattern, dict=nil)
+    @static_dict = if dict.nil?
+                     MigemoStaticDict.new(File.dirname(File.expand_path(__FILE__)) + '/../data/migemo-dict')
+                   elsif dict.is_a?(String)
+                     MigemoStaticDict.new(dict)
+                   else
+                     dict
+                   end
+    @type = "ruby"
+    @pattern = pattern
+    @insertion = ""
+    @optimization = 3
+
+    @dict_cache = nil
+    @user_dict = nil
+    @regex_dict = nil
+    @with_paren = false
+  end
+  attr_accessor :optimization
+  attr_accessor :type
+  attr_accessor :insertion
+  attr_accessor :dict_cache
+  attr_accessor :user_dict
+  attr_accessor :regex_dict
+  attr_accessor :with_paren
+
+  def lookup
+    if @pattern == ""
+      return RegexAlternation.new
+    end
+    result = if @dict_cache
+               lookup_cache || lookup0
+             else
+               lookup0
+             end
+    if @user_dict
+      lookup_user_dict.each{|x| result.push(x) }
+    end
+    result
+  end
+
+  def regex_tree
+    lookup
+  end
+
+  def regex
+    regex = lookup
+    renderer = RegexRendererFactory.new(regex, @type, @insertion)
+    renderer.with_paren = @with_paren
+    string = renderer.render
+    string = renderer.join_regexes(string, lookup_regex_dict) if @regex_dict
+    string
+  end
+
+  private
+  # `do'   => (ど)
+  # `d'    => (っ だ ぢ づ で ど)
+  # `sh'   => (しゃ し しゅ しぇ しょ)
+  # `don'  => (どん どな どに どぬ どね どの どっ)  # special case 1
+  # `nodd' => (のっ)                                # special case 2
+  # `doc'  => (どっ どち)                           # special case 3
+  # `dox'  => (どっ どゃ どゅ どょ)                 # special case 4
+  # `essy' => (えっしゃ えっしゅ えっしょ)          # special case 5
+  # `ny'   => (にゃ にゅ にょ)                      # special case 6
+  def expand_kanas
+    kana = @pattern.downcase.to_kana
+    /^(.*)(.)$/ =~ kana ;
+    head = $1;
+    last = $2;
+
+    cand = Array.new;
+    return [] if last == nil
+    if last.consonant?
+      if /^(.*)(.)$/ =~ head && $2.consonant?
+        head2 = $1;
+        beforelast = $2;
+        if last == beforelast # special case 2
+          cand.push head2 + "っ"
+        elsif /^(.*)(.)$/ =~ head2 && beforelast == $2 && last.consonant?
+          # special case 5
+          cand += (beforelast + last).expand_consonant.map do |x|
+            $1 + "っ" + x.to_kana
+          end
+        else
+          cand += (beforelast + last).expand_consonant.map do |x|
+            head2 + x.to_kana
+          end
+        end
+      elsif /^(.*?)(n?)ny$/ =~ @pattern && $2 == "" # special case 6
+        head2 = $1
+        cand += "ny".expand_consonant.map do |x|
+          head2 + x.to_kana
+        end
+      else
+        deriv = last.expand_consonant
+        deriv.push "xtsu";
+        if last == "c" # special case 3
+          deriv.push "chi";
+        elsif last == "x" # special case 4
+          deriv.push "xya", "xyu", "xyo", "xwa"
+        end
+        cand += deriv.map do |x| head + x.to_kana end
+      end
+    elsif last == "ん" # speacial case 1
+      cand.push kana;
+      cand += ("n".expand_consonant + ["っ"]).map do |x|
+        head + x.to_kana
+      end
+    else
+      cand.push kana
+    end
+    return cand.sort
+  end
+
+  # `めし' => (飯 飯合 雌蘂 雌蕊 飯櫃 目下 飯粒 召使 飯屋)
+  def expand_words (dict, pattern)
+    raise if pattern == nil
+    words = Array.new
+    dict.lookup(pattern) do |item|
+      words += item.values
+    end
+    return words
+  end
+
+  def lookup_cache
+    @dict_cache.lookup(@pattern)
+  end
+
+  def lookup0
+    compiler = RegexCompiler.new
+    compiler.push(@pattern)
+    compiler.push(@pattern.to_fullwidth)
+    expand_kanas.each do |x| 
+      compiler.push(x)
+      compiler.push(x.to_katakana)
+      expand_words(@static_dict, x).each do |_x| compiler.push(_x) end
+    end
+    expand_words(@static_dict, @pattern).each do |x| compiler.push(x) end
+    compiler.uniq
+    compiler.optimize(@optimization) if @optimization
+    compiler.regex
+  end
+
+  def lookup_user_dict
+    compiler = RegexCompiler.new
+    expand_kanas.each do |x| 
+      expand_words(@user_dict, x).each do |_x| compiler.push(_x) end
+    end
+    expand_words(@user_dict, @pattern).each do |x| compiler.push(x) end
+    compiler.uniq
+    compiler.optimize(@optimization) if @optimization
+    compiler.regex
+  end
+
+  def lookup_regex_dict
+    regexes = []
+    @regex_dict.lookup(@pattern) do |item|
+      regexes += item.values
+    end
+    regexes
+  end
+end

data/lib/migemo/core_ext/string.rb

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+class String
+  # Hiragana to Katakana
+  def to_katakana
+    self.gsub(/う゛/, 'ヴ').tr('ぁ-ん', 'ァ-ン')
+  end
+
+  def first
+    /^(\\.|.)/ =~ self
+    $1
+  end
+
+  def last
+    /(\\.|.)$/ =~ self
+    $1
+  end
+
+  def rest
+    /^(\\.|.)(.*)/ =~ self
+    $2
+  end
+
+  HANZEN_TAB = {
+    " " => "　", "!" => "！", '"' => "”", "#" => "＃", 
+    "\$" => "＄", "%" => "％", "&" => "＆", "'" => "’",
+    "(" => "（", ")" => "）", "*" => "＊", "+" => "＋",
+    "," => "，", "-" => "−", "." => "．", "/" => "／",
+    "0" => "０", "1" => "１", "2" => "２", "3" => "３",
+    "4" => "４", "5" => "５", "6" => "６", "7" => "７",
+    "8" => "８", "9" => "９", ":" => "：", ";" => "；",
+    "<" => "＜", "=" => "＝", ">" => "＞", "?" => "？",
+    '@' => "＠", "A" => "Ａ", "B" => "Ｂ", "C" => "Ｃ",
+    "D" => "Ｄ", "E" => "Ｅ", "F" => "Ｆ", "G" => "Ｇ",
+    "H" => "Ｈ", "I" => "Ｉ", "J" => "Ｊ", "K" => "Ｋ",
+    "L" => "Ｌ", "M" => "Ｍ", "N" => "Ｎ", "O" => "Ｏ",
+    "P" => "Ｐ", "Q" => "Ｑ", "R" => "Ｒ", "S" => "Ｓ",
+    "T" => "Ｔ", "U" => "Ｕ", "V" => "Ｖ", "W" => "Ｗ",
+    "X" => "Ｘ", "Y" => "Ｙ", "Z" => "Ｚ", "[" => "［", 
+    "\\" => "＼", "]" => "］", "^" => "＾", "_" => "＿",
+    "`" => "‘", "a" => "ａ", "b" => "ｂ", "c" => "ｃ",
+    "d" => "ｄ", "e" => "ｅ", "f" => "ｆ", "g" => "ｇ",
+    "h" => "ｈ", "i" => "ｉ", "j" => "ｊ", "k" => "ｋ",
+    "l" => "ｌ", "m" => "ｍ", "n" => "ｎ", "o" => "ｏ",
+    "p" => "ｐ", "q" => "ｑ", "r" => "ｒ", "s" => "ｓ",
+    "t" => "ｔ", "u" => "ｕ", "v" => "ｖ", "w" => "ｗ",
+    "x" => "ｘ", "y" => "ｙ", "z" => "ｚ", "{" => "｛",
+    "|" => "｜", "}" => "｝", "~" => "‾"} #'
+
+  HANZEN_RE = Regexp.new(HANZEN_TAB.keys.sort.map {|x| 
+                           Regexp.quote(x)
+                         }.join('|'))
+
+  def to_fullwidth
+    self.gsub(HANZEN_RE) {|s| HANZEN_TAB[s]}
+  end
+
+  def prefix_match (string)
+    self[0, string.length] <=> string
+  end
+end

data/lib/migemo/version.rb

@@ -0,0 +1,8 @@
+class Migemo
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 4
+    TINY  = 3
+    STRING = [MAJOR, MINOR, TINY].compact.join('.')
+  end
+end

data/test/cache_test.rb

@@ -0,0 +1,28 @@
+require 'test_helper'
+class CacheTest < Test::Unit::TestCase
+  def setup
+    base_dir = File.dirname(File.expand_path(__FILE__))
+    dict_file =  base_dir+ '/../data/test-dict'
+    words = base_dir + '/../data/migemo-chars'
+    File.open(dict_file) do |f|
+      @words = f.readlines.map{|l| l =~ /^(\w).*?	/; $1 }.compact.uniq
+    end
+    unless File.exists?(dict_file + '.cache')
+      Migemo::Cache.new(dict_file, words).generate.save(dict_file + '.cache')
+    end
+    @cache_dict = MigemoDictCache.new(dict_file + '.cache')
+  end
+
+  def test_caches
+    @words.each do |w|
+      [:normal, :cache].each do |ivar|
+        migemo = Migemo.new(w, migemo_dict)
+        migemo.user_dict = user_dict
+        migemo.regex_dict = regex_dict
+        instance_variable_set("@#{ivar}", migemo)
+      end
+      @cache.dict_cache = @cache_dict
+      assert_equal @normal.regex, @cache.regex
+    end
+  end
+end

data/test/charclass_test.rb

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+require 'test_helper'
+
+class CharclassTest < Test::Unit::TestCase
+  def test_ruby_type
+    migemo = Migemo.new('sym', migemo_dict)
+    assert_equal "[]$%@\\\\-]|sym|ｓｙｍ", migemo.regex
+  end
+
+  def test_emacs_type
+    migemo = Migemo.new('sym', migemo_dict)
+    migemo.type = 'emacs'
+    assert_equal "[]$%@\\-]\\|sym\\|ｓｙｍ", migemo.regex
+  end
+end

data/test/convert_test.rb

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+require 'test_helper'
+
+class ConvertTest < Test::Unit::TestCase
+  def test_convert
+    input =<<-EOF
+;;
+;; This is a comment line.
+;;
+りかい /理解/
+りかいs /理解/
+motion /モーション/
+りくとく /六徳;人が守るべき六つの徳。「ろくとく」とも/
+EOF
+    expects =<<-EOF
+;;
+;; This is Migemo's dictionary generated from SKK's.
+;;
+;;
+;; This is a comment line.
+;;
+motion	モーション
+りかい	理解
+りくとく	六徳
+EOF
+    convert = Migemo::Convert.new(input)
+    assert_equal expects,  (convert.header + convert.transfer).join("\n") + "\n"
+  end
+end

data/test/emacs_type_test.rb

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+require 'test_helper'
+
+class TestEmacsType < Test::Unit::TestCase
+  def test_mot
+    migemo = Migemo.new('mot', migemo_dict)
+    migemo.type = 'emacs'
+    assert_equal 'mot\|ｍｏｔ\|も[たちっつてと]\|モ\([タチッツテト]\|ー\(ション\|ター\)\|スラ\)', migemo.regex
+  end
+end

data/test/insertion_test.rb

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+require 'test_helper'
+class InsertionTest < Test::Unit::TestCase
+  def test_mot
+    migemo = Migemo.new('mot', migemo_dict)
+    migemo.insertion = "\\s *"
+    assert_equal 'm\s *o\s *t|ｍ\s *ｏ\s *ｔ|も\s *[たちっつてと]|モ\s *(?:[タチッツテト]|ー\s *(?:シ\s *ョ\s *ン|タ\s *ー)|ス\s *ラ)', migemo.regex
+  end
+
+  def test_mot_as_emacs
+    migemo = Migemo.new('mot', migemo_dict)
+    migemo.type = 'emacs'
+    migemo.insertion = "\\s *"
+    assert_equal 'm\s *o\s *t\|ｍ\s *ｏ\s *ｔ\|も\s *[たちっつてと]\|モ\s *\([タチッツテト]\|ー\s *\(シ\s *ョ\s *ン\|タ\s *ー\)\|ス\s *ラ\)', migemo.regex
+  end
+end

data/test/migemo_test.rb

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+require 'test_helper'
+
+class MigemoTest < Test::Unit::TestCase
+  def test_empty_string
+    migemo = Migemo.new("", migemo_dict)
+    assert_equal "", migemo.regex
+  end
+
+  def test_k
+    migemo = Migemo.new("k", migemo_dict)
+    assert_equal "[kｋかきくけこっカキクケコッ機帰気]", migemo.regex
+  end
+
+  def test_ki_with_no_optimize
+    migemo = Migemo.new("ki", migemo_dict)
+    migemo.optimization = 0
+    assert_equal "ki|ｋｉ|き|キ|気|機|帰|機能|帰納|帰農|機能主義|機能的|帰納的", migemo.regex
+  end
+
+  def test_ki_with_optimize1
+    migemo = Migemo.new("ki", migemo_dict)
+    migemo.optimization = 1
+    assert_equal "ki|ｋｉ|き|キ|機|帰|気", migemo.regex
+  end
+
+  def test_ki_with_optimize3
+    migemo = Migemo.new("ki", migemo_dict)
+    migemo.optimization = 3
+    assert_equal "[きキ機帰気]|ki|ｋｉ", migemo.regex
+  end
+
+  def test_kin
+    migemo = Migemo.new("kin", migemo_dict)
+    migemo.optimization = 3
+    assert_equal "kin|ｋｉｎ|き[っなにぬねのん]|キ[ッナニヌネノン]|機能|帰[納農]", migemo.regex 
+  end
+
+  def test_mot_with_optimze2
+    migemo = Migemo.new("mot", migemo_dict)
+    migemo.optimization = 2
+    assert_equal "mot|ｍｏｔ|も(?:た|ち|っ|つ|て|と)|モ(?:ー(?:ション|ター)|スラ|タ|チ|ッ|ツ|テ|ト)", migemo.regex
+  end
+
+  def test_mot_with_optimze3
+    migemo = Migemo.new("mot", migemo_dict)
+    migemo.optimization = 3
+    assert_equal "mot|ｍｏｔ|も[たちっつてと]|モ(?:[タチッツテト]|ー(?:ション|ター)|スラ)", migemo.regex
+  end
+end

data/test/regex_dict_test.rb

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+require 'test_helper'
+
+class RegexDictTest < Test::Unit::TestCase
+  def test_m
+    migemo = Migemo.new('m', migemo_dict)
+    migemo.regex_dict = regex_dict
+    assert_equal '[mｍっまみむめもッマミムメモ]|\([-0-9a-zA-Z_.]+@[-0-9a-zA-Z_.]+\)', migemo.regex
+  end
+
+
+  def test_ur
+    migemo = Migemo.new('ur', migemo_dict)
+    migemo.regex_dict = regex_dict
+    assert_equal 'ur|ｕｒ|う[っらりるれろ]|ウ[ッラリルレロ]|\(\(http\|https\|ftp\|afs\|wais\|telnet\|ldap\|gopher\|news\|nntp\|rsync\|mailto\)://[-_.!~*\'()a-zA-Z0-9;/?:@&=+$,%#]+\)', migemo.regex
+  end
+
+  def test_m_with_userdict
+    migemo = Migemo.new('m', migemo_dict)
+    migemo.regex_dict = regex_dict
+    migemo.user_dict = user_dict
+    assert_equal '[mｍっまみむめもッマミムメモ]|Message Of The Day|\([-0-9a-zA-Z_.]+@[-0-9a-zA-Z_.]+\)', migemo.regex
+  end
+end

data/test/regex_test.rb

@@ -0,0 +1,16 @@
+require 'test_helper'
+
+class RegexTest < Test::Unit::TestCase
+  def test_compile
+    patterns = []
+    File.open(File.dirname(File.expand_path(__FILE__)) + '/../data/migemo-dict') do |f|
+      lines = f.readlines.map(&:chomp)
+      10.times{ patterns << lines.slice!(rand(lines.length)) }
+    end
+    patterns.each do |pattern|
+      migemo = Migemo.new(pattern, migemo_dict)
+      assert Regexp.compile(migemo.regex)
+    end
+  end
+end
+