ve 0.0.2

data/lib/providers/japanese_transliterators.rb

@@ -0,0 +1,293 @@
+# Encoding: UTF-8
+
+class Ve
+  class Provider
+    class JapaneseTransliterators < Ve::Provider
+
+      def initialize(config = {})
+      end
+
+      def works?
+        true
+      end
+
+      def parse(text, options = {})
+        Ve::Parse::JapaneseTransliterators.new(text)
+      end
+
+    end
+  end
+end
+
+class Ve
+  class Parse
+    class JapaneseTransliterators < Ve::Parse
+
+      H_SYLLABIC_N   = 'ん'
+      H_SMALL_TSU    = 'っ'
+      
+      HIRA_TO_LATN = {
+        "あ"=>"a", "い"=>"i", "う"=>"u", "え"=>"e", "お"=>"o",
+        "か"=>"ka", "き"=>"ki", "く"=>"ku", "け"=>"ke", "こ"=>"ko",
+        "が"=>"ga", "ぎ"=>"gi", "ぐ"=>"gu", "げ"=>"ge", "ご"=>"go",
+        "さ"=>"sa", "し"=>"shi", "す"=>"su", "せ"=>"se", "そ"=>"so",
+        "ざ"=>"za", "じ"=>"ji", "ず"=>"zu", "ぜ"=>"ze", "ぞ"=>"zo",
+        "た"=>"ta", "ち"=>"chi", "つ"=>"tsu", "て"=>"te", "と"=>"to",
+        "だ"=>"da", "ぢ"=>"ji", "づ"=>"zu", "で"=>"de", "ど"=>"do",
+        "な"=>"na", "に"=>"ni", "ぬ"=>"nu", "ね"=>"ne", "の"=>"no",
+        "は"=>"ha", "ひ"=>"hi", "ふ"=>"fu", "へ"=>"he", "ほ"=>"ho",
+        "ば"=>"ba", "び"=>"bi", "ぶ"=>"bu", "べ"=>"be", "ぼ"=>"bo",
+        "ぱ"=>"pa", "ぴ"=>"pi", "ぷ"=>"pu", "ぺ"=>"pe", "ぽ"=>"po",
+        "ま"=>"ma", "み"=>"mi", "む"=>"mu", "め"=>"me", "も"=>"mo",
+        "や"=>"ya", "ゆ"=>"yu", "よ"=>"yo",
+        "ら"=>"ra", "り"=>"ri", "る"=>"ru", "れ"=>"re", "ろ"=>"ro",
+        "わ"=>"wa", "うぃ"=>"whi", "うぇ"=>"whe", "を"=>"wo",
+        "ゑ"=>"wye", "ゐ"=>"wyi", "ー"=>"-", "ん"=>"n",
+
+        "きゃ"=>"kya", "きゅ"=>"kyu", "きょ"=>"kyo", "きぇ"=>"kye", "きぃ"=>"kyi",
+        "ぎゃ"=>"gya", "ぎゅ"=>"gyu", "ぎょ"=>"gyo", "ぎぇ"=>"gye", "ぎぃ"=>"gyi",
+        "くぁ"=>"kwa", "くぃ"=>"kwi", "くぅ"=>"kwu", "くぇ"=>"kwe", "くぉ"=>"kwo",
+        "ぐぁ"=>"qwa", "ぐぃ"=>"gwi", "ぐぅ"=>"gwu", "ぐぇ"=>"gwe", "ぐぉ"=>"gwo",
+        "しゃ"=>"sha", "しぃ"=>"syi", "しゅ"=>"shu", "しぇ"=>"she", "しょ"=>"sho",
+        "じゃ"=>"jya", "じゅ"=>"zyu", "じぇ"=>"zye", "じょ"=>"zyo", "じぃ"=>"zyi",
+        "すぁ"=>"swa", "すぃ"=>"swi", "すぅ"=>"swu", "すぇ"=>"swe", "すぉ"=>"swo",
+        "ちゃ"=>"tya", "ちゅ"=>"tyu", "ちぇ"=>"tye", "ちょ"=>"tyo", "ちぃ"=>"tyi",
+        "ぢゃ"=>"dya", "ぢぃ"=>"dyi", "ぢゅ"=>"dyu", "ぢぇ"=>"dye", "ぢょ"=>"dyo",
+        "つぁ"=>"tsa", "つぃ"=>"tsi", "つぇ"=>"tse", "つぉ"=>"tso", "てゃ"=>"tha",
+        "てぃ"=>"thi", "てゅ"=>"thu", "てぇ"=>"the", "てょ"=>"tho", "とぁ"=>"twa",
+        "とぃ"=>"twi", "とぅ"=>"twu", "とぇ"=>"twe", "とぉ"=>"two", "でゃ"=>"dha",
+        "でぃ"=>"dhi", "でゅ"=>"dhu", "でぇ"=>"dhe", "でょ"=>"dho", "どぁ"=>"dwa",
+        "どぃ"=>"dwi", "どぅ"=>"dwu", "どぇ"=>"dwe", "どぉ"=>"dwo", "にゃ"=>"nya",
+        "にゅ"=>"nyu", "にょ"=>"nyo", "にぇ"=>"nye", "にぃ"=>"nyi", "ひゃ"=>"hya",
+        "ひぃ"=>"hyi", "ひゅ"=>"hyu", "ひぇ"=>"hye", "ひょ"=>"hyo", "びゃ"=>"bya",
+        "びぃ"=>"byi", "びゅ"=>"byu", "びぇ"=>"bye", "びょ"=>"byo", "ぴゃ"=>"pya",
+        "ぴぃ"=>"pyi", "ぴゅ"=>"pyu", "ぴぇ"=>"pye", "ぴょ"=>"pyo", "ふぁ"=>"fwa",
+        "ふぃ"=>"fyi", "ふぇ"=>"fye", "ふぉ"=>"fwo", "ふぅ"=>"fwu", "ふゃ"=>"fya",
+        "ふゅ"=>"fyu", "ふょ"=>"fyo", "みゃ"=>"mya", "みぃ"=>"myi", "みゅ"=>"myu",
+        "みぇ"=>"mye", "みょ"=>"myo", "りゃ"=>"rya", "りぃ"=>"ryi", "りゅ"=>"ryu",
+        "りぇ"=>"rye", "りょ"=>"ryo",
+        "ゔぁ"=>"va", "ゔぃ"=>"vyi", "ゔ"=>"vu", "ゔぇ"=>"vye", "ゔぉ"=>"vo",
+        "ゔゃ"=>"vya", "ゔゅ"=>"vyu", "ゔょ"=>"vyo",
+        "うぁ"=>"wha", "いぇ"=>"ye", "うぉ"=>"who",
+        "ぁ"=>"xa", "ぃ"=>"xi", "ぅ"=>"xu", "ぇ"=>"xe", "ぉ"=>"xo",
+        "ゕ"=>"xka", "ゖ"=>"xke", "ゎ"=>"xwa"
+      }
+      
+      LATN_TO_HIRA = {
+        'a'   => 'あ', 'i'   => 'い',                'u'  => 'う',               'e'  => 'え',   'o'  => 'お',
+        'ka'  => 'か', 'ki'  => 'き',                'ku' => 'く',               'ke' => 'け',   'ko' => 'こ',
+        'ga'  => 'が', 'gi'  => 'ぎ',                'gu' => 'ぐ',               'ge' => 'げ',   'go' => 'ご',
+        'sa'  => 'さ', 'si'  => 'し', 'shi' => 'し', 'su' => 'す',               'se' => 'せ',   'so' => 'そ',
+        'za'  => 'ざ', 'zi'  => 'じ', 'ji'  => 'じ', 'zu' => 'ず',               'ze' => 'ぜ',   'zo' => 'ぞ',
+        'ta'  => 'た', 'ti'  => 'ち', 'chi' => 'ち', 'tu' => 'つ', 'tsu'=> 'つ', 'te' => 'て',   'to' => 'と',
+        'da'  => 'だ', 'di'  => 'ぢ',                'du' => 'づ', 'dzu'=> 'づ', 'de' => 'で',   'do' => 'ど',
+        'na'  => 'な', 'ni'  => 'に',                'nu' => 'ぬ',               'ne' => 'ね',   'no' => 'の',
+        'ha'  => 'は', 'hi'  => 'ひ',                'hu' => 'ふ', 'fu' => 'ふ', 'he' => 'へ',   'ho' => 'ほ',
+        'ba'  => 'ば', 'bi'  => 'び',                'bu' => 'ぶ',               'be' => 'べ',   'bo' => 'ぼ',
+        'pa'  => 'ぱ', 'pi'  => 'ぴ',                'pu' => 'ぷ',               'pe' => 'ぺ',   'po' => 'ぽ',
+        'ma'  => 'ま', 'mi'  => 'み',                'mu' => 'む',               'me' => 'め',   'mo' => 'も',
+        'ya'  => 'や',                               'yu' => 'ゆ',                               'yo' => 'よ',
+        'ra'  => 'ら', 'ri'  => 'り',                'ru' => 'る',               're' => 'れ',   'ro' => 'ろ',
+        'la'  => 'ら', 'li'  => 'り',                'lu' => 'る',               'le' => 'れ',   'lo' => 'ろ',
+        'wa'  => 'わ', 'wi'  => 'うぃ',                                          'we' => 'うぇ', 'wo' => 'を',
+        'wye' => 'ゑ', 'wyi' => 'ゐ', '-' => 'ー',
+
+        'n'   => 'ん', 'nn'  => 'ん', "n'"=> 'ん',
+
+        'kya' => 'きゃ', 'kyu' => 'きゅ', 'kyo' => 'きょ', 'kye' => 'きぇ', 'kyi' => 'きぃ',
+        'gya' => 'ぎゃ', 'gyu' => 'ぎゅ', 'gyo' => 'ぎょ', 'gye' => 'ぎぇ', 'gyi' => 'ぎぃ',
+        'kwa' => 'くぁ', 'kwi' => 'くぃ', 'kwu' => 'くぅ', 'kwe' => 'くぇ', 'kwo' => 'くぉ',
+        'gwa' => 'ぐぁ', 'gwi' => 'ぐぃ', 'gwu' => 'ぐぅ', 'gwe' => 'ぐぇ', 'gwo' => 'ぐぉ',
+        'qwa' => 'ぐぁ', 'gwi' => 'ぐぃ', 'gwu' => 'ぐぅ', 'gwe' => 'ぐぇ', 'gwo' => 'ぐぉ',
+
+        'sya' => 'しゃ', 'syi' => 'しぃ', 'syu' => 'しゅ', 'sye' => 'しぇ', 'syo' => 'しょ',
+        'sha' => 'しゃ',                  'shu' => 'しゅ', 'she' => 'しぇ', 'sho' => 'しょ',
+        'ja'  => 'じゃ',                  'ju'  => 'じゅ', 'je'  => 'じぇ', 'jo'  => 'じょ',
+        'jya' => 'じゃ', 'jyi' => 'じぃ', 'jyu' => 'じゅ', 'jye' => 'じぇ', 'jyo' => 'じょ',
+        'zya' => 'じゃ', 'zyu' => 'じゅ', 'zyo' => 'じょ', 'zye' => 'じぇ', 'zyi' => 'じぃ',
+        'swa' => 'すぁ', 'swi' => 'すぃ', 'swu' => 'すぅ', 'swe' => 'すぇ', 'swo' => 'すぉ',
+
+        'cha' => 'ちゃ',                  'chu' => 'ちゅ', 'che' => 'ちぇ', 'cho' => 'ちょ',
+        'cya' => 'ちゃ', 'cyi' => 'ちぃ', 'cyu' => 'ちゅ', 'cye' => 'ちぇ', 'cyo' => 'ちょ',
+        'tya' => 'ちゃ', 'tyi' => 'ちぃ', 'tyu' => 'ちゅ', 'tye' => 'ちぇ', 'tyo' => 'ちょ',
+        'dya' => 'ぢゃ', 'dyi' => 'ぢぃ', 'dyu' => 'ぢゅ', 'dye' => 'ぢぇ', 'dyo' => 'ぢょ',
+        'tsa' => 'つぁ', 'tsi' => 'つぃ',                  'tse' => 'つぇ', 'tso' => 'つぉ',
+        'tha' => 'てゃ', 'thi' => 'てぃ', 'thu' => 'てゅ', 'the' => 'てぇ', 'tho' => 'てょ',
+        'twa' => 'とぁ', 'twi' => 'とぃ', 'twu' => 'とぅ', 'twe' => 'とぇ', 'two' => 'とぉ',
+        'dha' => 'でゃ', 'dhi' => 'でぃ', 'dhu' => 'でゅ', 'dhe' => 'でぇ', 'dho' => 'でょ',
+        'dwa' => 'どぁ', 'dwi' => 'どぃ', 'dwu' => 'どぅ', 'dwe' => 'どぇ', 'dwo' => 'どぉ',
+
+        'nya' => 'にゃ', 'nyu' => 'にゅ', 'nyo' => 'にょ', 'nye' => 'にぇ', 'nyi' => 'にぃ',
+
+        'hya' => 'ひゃ', 'hyi' => 'ひぃ', 'hyu' => 'ひゅ', 'hye' => 'ひぇ', 'hyo' => 'ひょ',
+        'bya' => 'びゃ', 'byi' => 'びぃ', 'byu' => 'びゅ', 'bye' => 'びぇ', 'byo' => 'びょ',
+        'pya' => 'ぴゃ', 'pyi' => 'ぴぃ', 'pyu' => 'ぴゅ', 'pye' => 'ぴぇ', 'pyo' => 'ぴょ',
+        'fa'  => 'ふぁ', 'fi'  => 'ふぃ',                  'fe'  => 'ふぇ', 'fo'  => 'ふぉ',
+        'fwa' => 'ふぁ', 'fwi' => 'ふぃ', 'fwu' => 'ふぅ', 'fwe' => 'ふぇ', 'fwo' => 'ふぉ',
+        'fya' => 'ふゃ', 'fyi' => 'ふぃ', 'fyu' => 'ふゅ', 'fye' => 'ふぇ', 'fyo' => 'ふょ',
+
+        'mya' => 'みゃ', 'myi' => 'みぃ', 'myu' => 'みゅ', 'mye' => 'みぇ', 'myo' => 'みょ',
+
+        'rya' => 'りゃ', 'ryi' => 'りぃ', 'ryu' => 'りゅ', 'rye' => 'りぇ', 'ryo' => 'りょ',
+        'lya' => 'りゃ', 'lyu' => 'りゅ', 'lyo' => 'りょ', 'lye' => 'りぇ', 'lyi' => 'りぃ',
+
+        'va'  => 'ゔぁ', 'vi'  => 'ゔぃ', 'vu'  => 'ゔ',   've'  => 'ゔぇ',  'vo' => 'ゔぉ',
+        'vya' => 'ゔゃ', 'vyi' => 'ゔぃ', 'vyu' => 'ゔゅ', 'vye' => 'ゔぇ', 'vyo' => 'ゔょ',
+        'wha' => 'うぁ', 'whi' => 'うぃ', 'ye'  => 'いぇ', 'whe' => 'うぇ', 'who' => 'うぉ',
+
+        'xa'  => 'ぁ', 'xi'   => 'ぃ', 'xu'  => 'ぅ', 'xe'  => 'ぇ', 'xo'   => 'ぉ',
+        'xya' => 'ゃ', 'xyu'  => 'ゅ', 'xyo' => 'ょ',
+        'xtu' => 'っ', 'xtsu' => 'っ',
+        'xka' => 'ゕ', 'xke'  => 'ゖ', 'xwa' => 'ゎ',
+
+        '@@' => '　', '#[' => '「', '#]' => '」', '#,' => '、', '#.' => '。', '#/' => '・',
+      }
+
+      attr_reader :tokens, :text
+
+      def initialize(text)
+        @tokens = []
+        @text = text
+      end
+
+      def transliterate_from_hrkt_to_latn
+        @text = transliterate_from_kana_to_hira
+        transliterate_from_hira_to_latn
+      end
+
+      def transliterate_from_hira_to_latn
+        # Hepburn style romaji
+        kana = @text.dup
+        romaji = ''
+        geminate = false
+
+        while kana.length > 0
+          [2, 1].each do |length|
+            mora = ''
+            for_conversion = kana[0, length]
+
+            if for_conversion == H_SMALL_TSU
+              geminate = true
+              kana[0, length] = ''
+              break
+            elsif for_conversion == H_SYLLABIC_N && kana[1, 1].match(/[やゆよ]/)
+              # Syllabic N before ya, yu or yo
+              mora = "n'"
+            elsif HIRA_TO_LATN[for_conversion]
+              # Generic cases
+              mora = HIRA_TO_LATN[for_conversion]
+            end
+
+            if mora.length > 0
+              if geminate
+                geminate = false
+                romaji << mora[0, 1]
+              end
+              romaji << mora
+              kana[0, length] = ''
+              break
+            elsif length == 1
+              # Nothing found
+              romaji << for_conversion
+              kana[0, length] = ''
+            end
+          end
+        end
+
+        return romaji
+      end
+      
+      def transliterate_from_latn_to_hrkt
+        romaji = @text.dup
+        kana = ''
+
+        romaji.gsub!(/m([BbPp])/, 'n\1')
+        romaji.gsub!(/M([BbPp])/, 'N\1')
+
+        while romaji.length > 0
+          [3, 2, 1].each do |length|
+            mora = ''
+            for_removal = length
+            for_conversion = romaji[0, length]
+            is_upper = !!(for_conversion.match(/^\p{Upper}/))
+            for_conversion.downcase!
+
+            if for_conversion.match(/nn[aiueo]/)
+              # nna should kanafy to んな instead of んあ
+              # This is what people expect for words like konna, anna, zannen
+              mora = H_SYLLABIC_N
+              for_removal = 1
+            elsif LATN_TO_HIRA[for_conversion]
+              # Generic cases
+              mora = LATN_TO_HIRA[for_conversion]
+            elsif for_conversion == 'tch' || ( length == 2 && for_conversion.match(/([kgsztdnbpmyrlwc])\1/))
+              # tch and double-consonants for small tsu 
+              mora = H_SMALL_TSU
+              for_removal = 1
+            end
+
+            if mora.length > 0
+              if is_upper
+                # Dance so we can call transliterate_from_hira_to_kana on internal data
+                # TODO: Need a better way for this
+                temp_text = @text
+                @text = mora.dup
+                kana << transliterate_from_hira_to_kana
+                @text = temp_text
+              else
+                kana << mora
+              end
+              
+              romaji[0, for_removal] = ''
+              break
+            elsif length == 1
+              # Nothing found
+              kana << for_conversion
+              romaji[0, 1] = ''
+            end
+          end
+        end
+
+        return kana
+      end
+      
+      def transliterate_from_kana_to_hira
+        transpose_codepoints_in_range(@text, -96, 12449..12534)
+      end
+
+      def transliterate_from_hira_to_kana
+        transpose_codepoints_in_range(@text, 96, 12353..12438)
+      end
+
+      def transliterate_from_fullwidth_to_halfwidth
+        res = transpose_codepoints_in_range(@text, -65248, 65281..65374)
+        transpose_codepoints_in_range(res, -12256, 12288..12288)
+      end
+
+      def transliterate_from_halfwidth_to_fullwidth
+        res = transpose_codepoints_in_range(@text, 65248, 33..126)
+        transpose_codepoints_in_range(res, 12256, 32..32)
+      end
+      
+      private
+
+      def transpose_codepoints_in_range(text, distance, range)
+        result = ''
+
+        text.each_codepoint do |c|
+          if c >= range.first and c <= range.last
+            result << (c + distance).chr(Encoding::UTF_8)
+          else
+            result << c.chr(Encoding::UTF_8)
+          end
+        end
+
+        return result
+      end
+      
+    end
+  end
+end
+
+Ve::Manager.register(Ve::Provider::JapaneseTransliterators, :ja)
+

data/lib/providers/mecab_ipadic.rb

@@ -0,0 +1,362 @@
+# Encoding: UTF-8
+
+require 'open3'
+
+class Ve
+  class Provider
+    class MecabIpadic < Ve::Provider
+
+      BIT_STOP = 'VeEnd'
+  
+      def initialize(config = {})
+        # TODO: Make config handling better
+        @config = {:app => 'mecab',
+                   :path => '',
+                   :flags => ''}.merge(config)
+    
+        @config[:app] = `which #{@config[:app]}`
+        
+        start!
+      end
+  
+      def works?
+        (["だっ\t助動詞,*,*,*,特殊・ダ,連用タ接続,だ,ダッ,ダッ",
+          "た\t助動詞,*,*,*,特殊・タ,基本形,た,タ,タ",
+          "EOS"] == parse('だった').tokens.collect { |t| t[:raw] } )
+      end
+  
+      # Talks to the app and returns a parse object
+      def parse(text, options = {})
+        start! if @stdin.nil? # Restart if the provider crashed
+        
+        @stdin.puts "#{text} #{BIT_STOP}"
+        output = []
+        
+        while line = @stdout.readline.force_encoding('UTF-8')
+          if line =~ /#{BIT_STOP}/x
+            output << @stdout.readline # Catch the EOS
+            break
+          end
+          output << line
+        end
+        
+        Ve::Parse::MecabIpadic.new(text, output)
+      rescue
+        # TODO: No good to catch all errors like this
+        # I need a backtrace when something unexpected fails
+        Ve::Parse::MecabIpadic.new(text, [])
+      end
+
+      private
+  
+      # TODO: Use Process.spawn/kill for process control?
+      def start!
+        @stdin, @stdout, @stderr = Open3.popen3(@config[:app])
+        @stdin.set_encoding('UTF-8')
+        @stdout.set_encoding('UTF-8')
+      rescue Errno::ENOENT
+        # The parser couldn't be started. Probably not installed on this system
+      end
+  
+    end
+  end
+end
+
+class Ve
+  class Parse
+    class MecabIpadic < Ve::Parse
+      
+      PARSER = %r{^ (.+?) \t (.+) }x
+      attr_reader :tokens, :text
+      
+      def initialize(text, output)
+        @tokens = []
+        @text = text
+        position = 0
+        
+        output.each_with_index do |line, index|
+          line.rstrip!
+          token = {:raw => line}
+          # Anything unparsed at the end of the text
+          # This must happen before sentence splits are detected to avoid funny ordering
+          if output.length > 1 && output.length == index + 1
+            unparsed_md = %r{(.*? \Z\n?)}mx.match(text, position)
+            if unparsed_md[1].length > 0
+              unparsed_token = {:type => :unparsed, :literal => unparsed_md[1], :raw => ''}
+              unparsed_token[:characters] = (position..(position+unparsed_token[:literal].length-1))
+              @tokens << unparsed_token
+            end
+          end
+          
+          if line =~ %r{^ EOS $}x
+            token[:type] = :sentence_split
+            token[:literal] = ''
+          elsif md = PARSER.match(line)
+            # The parsed token
+            token[:type] = :parsed
+            token[:literal] = md[1]
+            info = md[2].split(',')
+            [:pos, :pos2, :pos3, :pos4, :inflection_type, :inflection_form, :lemma, :reading, :hatsuon].each_with_index do |attr, i|
+              token[attr] = info[i]
+            end
+            
+            # Anything unparsed preceding this token
+            unparsed_md = %r{(.*?) #{Regexp.quote(token[:literal])}}mx.match(text, position)
+            if unparsed_md[1].length > 0
+              unparsed_token = {:type => :unparsed, :literal => unparsed_md[1]}
+              unparsed_token[:characters] = (position..(position+unparsed_token[:literal].length-1))
+              @tokens << unparsed_token
+              position += unparsed_token[:literal].length
+            end
+            
+            token[:characters] = (position..(position+token[:literal].length-1))
+            position += token[:literal].length
+          else
+            # C'est une catastrophe
+          end
+
+          @tokens << token
+        end
+      end
+      
+      # PoS
+      MEISHI = '名詞'
+      KOYUUMEISHI = '固有名詞'
+      DAIMEISHI = '代名詞'
+      JODOUSHI = '助動詞'
+      KAZU = '数'
+      JOSHI = '助詞'
+      SETTOUSHI = '接頭詞'
+      DOUSHI = '動詞'
+      KIGOU = '記号'
+      FIRAA = 'フィラー'
+      SONOTA = 'その他'
+      KANDOUSHI = '感動詞'
+      RENTAISHI = '連体詞'
+      SETSUZOKUSHI = '接続詞'
+      FUKUSHI = '副詞'
+      SETSUZOKUJOSHI = '接続助詞'
+      KEIYOUSHI = '形容詞'
+
+      # Pos2 and Inflection types
+      HIJIRITSU = '非自立'
+      FUKUSHIKANOU = '副詞可能'
+      SAHENSETSUZOKU = 'サ変接続'
+      KEIYOUDOUSHIGOKAN = '形容動詞語幹'
+      NAIKEIYOUSHIGOKAN = 'ナイ形容詞語幹'
+      JODOUSHIGOKAN = '助動詞語幹'
+      FUKUSHIKA = '副詞化'
+      TAIGENSETSUZOKU = '体言接続'
+      RENTAIKA = '連体化'
+      TOKUSHU = '特殊'
+      SETSUBI = '接尾'
+      SETSUZOKUSHITEKI = '接続詞的'
+      DOUSHIHIJIRITSUTEKI = '動詞非自立的'
+      SAHEN_SURU = 'サ変・スル'
+      TOKUSHU_TA = '特殊・タ'
+      TOKUSHU_NAI = '特殊・ナイ'
+      TOKUSHU_TAI = '特殊・タイ'
+      TOKUSHU_DESU = '特殊・デス'
+      TOKUSHU_DA = '特殊・ダ'
+      TOKUSHU_MASU = '特殊・マス'
+
+      # Etc
+      NA = 'な'
+      NI = 'に'
+      TE = 'て'
+      DE = 'で'
+      BA = 'ば'
+
+      def words
+        words = []
+        tokens = @tokens.find_all { |t| t[:type] == :parsed }
+        tokens = tokens.to_enum
+
+        # This is becoming very big
+        begin
+          while token = tokens.next
+            pos = nil
+            grammar = nil
+            eat_next = false
+            eat_lemma = true
+            attach_to_previous = false
+            also_attach_to_lemma = false
+
+            case token[:pos]
+            when MEISHI
+              pos = Ve::PartOfSpeech::Noun
+
+              case token[:pos2]
+              when KOYUUMEISHI
+                pos = Ve::PartOfSpeech::ProperNoun
+              when DAIMEISHI
+                pos = Ve::PartOfSpeech::Pronoun
+              when FUKUSHIKANOU, SAHENSETSUZOKU, KEIYOUDOUSHIGOKAN, NAIKEIYOUSHIGOKAN
+                if tokens.more?
+                  following = tokens.peek
+                  if following[:inflection_type] == SAHEN_SURU
+                    pos = Ve::PartOfSpeech::Verb
+                    eat_next = true
+                  elsif following[:inflection_type] == TOKUSHU_DA
+                    pos = Ve::PartOfSpeech::Adjective
+                    if following[:inflection_form] == TAIGENSETSUZOKU
+                      eat_next = true
+                      eat_lemma = false
+                    end
+                  elsif following[:inflection_type] == TOKUSHU_NAI
+                    pos = Ve::PartOfSpeech::Adjective
+                    eat_next = true
+                  elsif following[:pos] == JOSHI && following[:literal] == NI
+                    pos = Ve::PartOfSpeech::Adverb
+                    eat_next = true
+                  end
+                end
+              when HIJIRITSU, TOKUSHU
+                if tokens.more?
+                  following = tokens.peek
+                  case token[:pos3]
+                  when FUKUSHIKANOU
+                    if following[:pos] == JOSHI && following[:literal] == NI
+                      pos = Ve::PartOfSpeech::Adverb
+                      eat_next = true
+                    end
+                  when JODOUSHIGOKAN
+                    if following[:inflection_type] == TOKUSHU_DA
+                      pos = Ve::PartOfSpeech::Verb
+                      grammar = :auxillary
+                      if following[:inflection_form] == TAIGENSETSUZOKU
+                        eat_next = true
+                      end
+                    elsif following[:pos] == JOSHI && following[:pos2] == FUKUSHIKA
+                      pos = Ve::PartOfSpeech::Adverb
+                      eat_next = true
+                    end
+                  when KEIYOUDOUSHIGOKAN
+                    pos = Ve::PartOfSpeech::Adjective
+                    if (following[:inflection_type] == TOKUSHU_DA && following[:inflection_form] == TAIGENSETSUZOKU) || following[:pos2] == RENTAIKA
+                      eat_next = true
+                    end
+                  end
+                end
+              when KAZU
+                # TODO: recurse and find following numbers and add to this word. Except non-numbers like 幾
+                pos = Ve::PartOfSpeech::Number
+                if words.length > 0 && words[-1].part_of_speech == Ve::PartOfSpeech::Number
+                  attach_to_previous = true
+                  also_attach_to_lemma = true
+                end
+              when SETSUBI
+                # TODO: elaborate a bit?
+                pos = Ve::PartOfSpeech::Suffix
+              when SETSUZOKUSHITEKI
+                pos = Ve::PartOfSpeech::Conjunction
+              when DOUSHIHIJIRITSUTEKI
+                pos = Ve::PartOfSpeech::Verb
+                grammar = :nominal
+              end
+            when SETTOUSHI
+              # TODO: elaborate this when we have the "main part" feature for words?
+              pos = Ve::PartOfSpeech::Prefix
+            when JODOUSHI
+              pos = Ve::PartOfSpeech::Postposition
+
+              if [TOKUSHU_TA, TOKUSHU_NAI, TOKUSHU_TAI, TOKUSHU_MASU].include?(token[:inflection_type])
+                attach_to_previous = true
+              elsif (token[:inflection_type] == TOKUSHU_DA || token[:inflection_type] == TOKUSHU_DESU) && token[:literal] != NA
+                pos = Ve::PartOfSpeech::Verb
+              end
+            when DOUSHI
+              pos = Ve::PartOfSpeech::Verb
+              if token[:pos2] == SETSUBI
+                attach_to_previous = true
+              elsif token[:pos2] == HIJIRITSU
+                grammar = :auxillary
+              end
+            when KEIYOUSHI
+              pos = Ve::PartOfSpeech::Adjective
+            when JOSHI
+              pos = Ve::PartOfSpeech::Postposition
+              if token[:pos2] == SETSUZOKUJOSHI && [TE, DE, BA].include?(token[:literal])
+                attach_to_previous = true
+              end
+            when RENTAISHI
+              pos = Ve::PartOfSpeech::Determiner
+            when SETSUZOKUSHI
+              pos = Ve::PartOfSpeech::Conjunction
+            when FUKUSHI
+              pos = Ve::PartOfSpeech::Adverb
+            when KIGOU
+              pos = Ve::PartOfSpeech::Symbol
+            when FIRAA, KANDOUSHI
+              pos = Ve::PartOfSpeech::Interjection
+            when SONOTA
+              pos = Ve::PartOfSpeech::Other
+            else
+              # C'est une catastrophe
+            end
+
+            if attach_to_previous && words.length > 0
+              words[-1].tokens << token
+              words[-1].word << token[:literal]
+              words[-1].extra[:reading] << (token[:reading] || '')
+              words[-1].extra[:transcription] << (token[:hatsuon] || '')
+              words[-1].lemma << token[:lemma] if also_attach_to_lemma
+            else
+              pos = Ve::PartOfSpeech::TBD if pos.nil?
+              word = Ve::Word.new(token[:literal], token[:lemma], pos, [token], {
+                :reading => token[:reading] || '',
+                :transcription => token[:hatsuon] || '',
+                :grammar => grammar
+              }, {
+                :reading_script => :kata,
+                :transcription_script => :kata
+              })
+
+              if eat_next
+                following = tokens.next
+                word.tokens << following
+                word.word << following[:literal]
+                word.extra[:reading] << following[:reading]
+                word.extra[:transcription] << following[:hatsuon]
+                word.lemma << following[:lemma] if eat_lemma
+              end
+
+              words << word
+            end
+          end
+        rescue StopIteration
+        end
+
+        return words
+      end
+      
+      def sentences
+        # TODO: Sentence objects that keep track of the sentence's tokens
+        sentences = []
+        current = ''
+        
+        @tokens.each do |token|
+          if token[:type] == :sentence_split
+            sentences << current
+            current = ''
+          elsif token[:literal] == '。'
+            current << token[:literal]
+            sentences << current
+            current = ''
+          else
+            current << token[:literal]
+          end
+        end
+        
+        # In case there is no :sentence_split at the end
+        sentences << current if current.length > 0
+        
+        sentences
+      end
+      
+    end
+  end
+end
+
+Ve::Manager.register(Ve::Provider::MecabIpadic, :ja)
+