ve 0.0.2

data/lib/ve.rb

@@ -0,0 +1,111 @@
+$: << File.expand_path(File.dirname(__FILE__))
+
+require 'misc'
+require 'word'
+require 'part_of_speech'
+require 'languages/english'
+require 'languages/japanese'
+require 'pp'
+
+class Ve
+  
+  class Manager
+    def self.provider_for(language, function)
+      @@provider_for[language.to_sym][function.to_sym]
+    end
+
+    # TODO: Make a difference between what features are available locally
+    # and what requires contacting external Ves
+    def self.register(klass, language)
+      @@provider_for ||= {}
+      provider = klass.new
+      # This won't work if people start monkey patching the providers with public methods that arent abilities
+      # It's also not pretty, but kinda nifty
+      provider_name = provider.class.to_s.split('::').last
+      parse_class = Kernel.class_eval("Ve::Parse::#{provider_name}")
+      abilities = parse_class.public_instance_methods - Object.public_instance_methods
+      abilities.each do |a|
+        @@provider_for[language.to_sym] ||= {}
+        @@provider_for[language.to_sym][a] = provider
+      end
+    end
+  end
+  
+  # TODO: Put into separate files
+  class LocalInterface    
+    def initialize(language, config = {})
+      @language = language
+    end
+
+    def method_missing(function, *args)
+      provider = Ve::Manager.provider_for(@language, function)
+      parse = provider.parse(args[0])
+      parse.send(function.to_sym)
+    end
+  end
+  
+  class HTTPInterface
+    require 'net/http'
+    require 'uri'
+    require 'json'
+    
+    def initialize(language, config = {})
+      @language = language
+      @base_url = config[:url]
+    end
+
+    def method_missing(function, *args)
+      url = "#{@base_url}/#{@language}/#{function}"
+      uri = URI.parse(url)
+      response = Net::HTTP.post_form(uri, {:text => args[0]})
+      data = JSON.parse(response.body)
+      result = []
+      
+      data.each do |obj|
+        # TODO: Support transliterations
+        case obj['_class']
+        when 'Word'
+          result << Ve::Word.new(obj['word'], obj['lemma'], obj['part_of_speech'], obj['tokens'], obj['extra'], obj['info'])
+        end
+      end
+      
+      result
+    end
+  end
+  
+  @@interface = Ve::LocalInterface
+  @@interface_for = {}
+  @@config = {}
+  
+  # End-users only interact with this class, so it must provide a sexy interface
+  # to all functionality in the providers and parse objects
+  
+  # Basic, non-sexy, local interface only
+  def self.get(text, language, function, *args)
+    provider = Ve::Manager.provider_for(language, function, *args)
+    parse = provider.parse(text, args)
+    parse.send(function.to_sym)
+  end
+  
+  # Early sexy verision
+  def self.in(language)
+    unless @@interface_for[language]
+      @@interface_for[language] = @@interface.new(language, @@config)
+    end
+    
+    @@interface_for[language]
+  end
+
+  def self.config(interface, config)
+    @@interface = interface
+    @@config = config
+  end
+  
+end
+
+# TODO: Autoload this shit
+require 'providers/fallbacks'
+require 'providers/mecab_ipadic'
+require 'providers/freeling_en'
+require 'providers/japanese_transliterators'
+

data/lib/word.rb

@@ -0,0 +1,43 @@
+class Ve
+  class Word
+    
+    attr_accessor :word, :lemma, :part_of_speech, :tokens, :extra, :info
+    
+    # TODO: More elegance
+    def initialize(word, lemma, part_of_speech, tokens, extra = {}, info = {})
+      @word = word.dup
+      @lemma = lemma.dup
+      @part_of_speech = part_of_speech
+      @tokens = tokens
+      
+      # TODO: I don't like this, it's too unstructured
+      @extra = extra
+      @info = info
+    end
+    
+    # TODO: the main part of a word, for example 重要 in 重要な
+    def main_part
+    end
+
+    def base_form
+      @lemma
+    end
+    
+    def inflected?
+      @word != @lemma
+    end
+
+    def as_json
+      {
+        :_class => 'Word',
+        :word => @word,
+        :lemma => @lemma,
+        :part_of_speech => @part_of_speech.name,
+        :tokens => @tokens,
+        :extra => @extra,
+        :info => @info
+      }
+    end
+    
+  end
+end

data/sinatra/server.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'sinatra'
+require 'json'
+require 'rack/cors'
+
+require File.expand_path(File.dirname(__FILE__) + "/../lib/ve")
+
+use Rack::Cors do
+  allow do
+    origins '*'
+  end
+end
+
+get '/:language/:function' do
+  run
+end
+
+post '/:language/:function' do
+  run
+end
+
+private
+
+def run
+#  Ve.source = Ve::Local # Default
+#  Ve.source = Ve::Remote.new(:url => 'http://ve.kimtaro.com/', :access_token => 'XYZ')
+#  result = Ve.get(params[:text], params[:language], params[:function].to_sym)
+  result = Ve.in(params[:language]).words(params[:text])
+
+  case params[:function].to_sym
+  when :words
+    json = JSON.generate(result.collect(&:as_json))
+  else
+    json = result
+  end
+
+  if params[:callback]
+    json = "#{params[:callback]}(#{json})"
+    content_type 'application/javascript', :charset => 'utf-8'
+  else
+    content_type 'application/json', :charset => 'utf-8'
+  end
+
+  json
+end

data/tests/freeling_en_test.rb

@@ -0,0 +1,135 @@
+# Encoding: UTF-8
+
+require_relative 'test_helper'
+
+class FreelingEnTest < Test::Unit::TestCase
+
+  def test_should_be_able_to_start
+    freeling = Ve::Provider::FreelingEn.new
+    assert freeling.works?
+  end
+
+  def test_doesnt_die_on_japanese
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse('これは日本語です')
+    assert_equal Ve::Parse::FreelingEn, parse.class
+  end
+
+  # TODO: UTF-8 handling
+  def test_can_handle_utf8
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse('I’m')
+    assert_equal ['I\'m'], parse.tokens.collect { |t| t[:literal] }
+  end
+
+  def test_can_parse
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse('')
+    assert_equal Ve::Parse::FreelingEn, parse.class
+  end
+
+  def test_all_literals_should_equal_the_input_text
+    text = <<-EOS
+    There once was a man from X
+    Who took it upon himself to Y
+    Z
+
+    EOS
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse(text)
+    assert_equal text, parse.tokens.collect { |t| t[:literal] }.join
+  end
+
+  def test_creates_tokens_from_data_that_is_ignored_in_parsing
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse('A   B  ')
+    assert_equal [:parsed, :unparsed, :parsed, :unparsed, :sentence_split], parse.tokens.collect { |t| t[:type] }
+    assert_equal ['A', '   ', 'B', '  ', ''], parse.tokens.collect { |t| t[:literal] }
+  end
+
+  def test_can_give_sentences
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse('This is a sentence. And this was another one')
+    assert_equal ['This is a sentence.', 'And this was another one'], parse.sentences
+  end
+
+  def test_can_give_words
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse('This was a sentence.')
+    words = parse.words
+    tokens = parse.tokens
+
+    assert_equal ['This', 'was', 'a', 'sentence', '.'], words.collect(&:word)
+    assert_equal ['this', 'be', 'a', 'sentence', '.'], words.collect(&:lemma)
+    assert_equal [Ve::PartOfSpeech::Pronoun, Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Determiner, Ve::PartOfSpeech::Noun, Ve::PartOfSpeech::Symbol], words.collect(&:part_of_speech)
+    assert_equal [{:grammar => :personal}, {:grammar => :past}, {:grammar => nil}, {:grammar => nil}, {:grammar => nil}], words.collect(&:extra)
+
+    assert_equal [[tokens[0]], [tokens[2]], [tokens[4]], [tokens[6]], [tokens[7]]], words.collect(&:tokens)
+  end
+
+  def test_possessive_endings_must_be_reattached
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse("This is Jane's sentence.")
+    words = parse.words
+    tokens = parse.tokens
+
+    assert_equal ['This', 'is', "Jane's", 'sentence', '.'], words.collect(&:word)
+    assert_equal ['this', 'be', "jane", 'sentence', '.'], words.collect(&:lemma)
+    assert_equal [Ve::PartOfSpeech::Pronoun, Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::ProperNoun, Ve::PartOfSpeech::Noun, Ve::PartOfSpeech::Symbol], words.collect(&:part_of_speech)
+    assert_equal [{:grammar => :personal}, {:grammar => nil}, {:gramamr => nil}, {:grammar => nil}, {:grammar => nil}], words.collect(&:extra)
+    assert_equal [[tokens[0]], [tokens[2]], tokens[4..5], [tokens[7]], [tokens[8]]], words.collect(&:tokens)
+  end
+
+  def test_date_parsing
+    # Should be turned off. At least for now
+    freeling = Ve::Provider::FreelingEn.new
+
+    assert_parses_into_words(freeling,
+                             {:words => ['January'],
+                              :lemmas => ['january'],
+                              :pos => [Ve::PartOfSpeech::Noun],
+                              :extra => [{:grammar => nil}],
+                              :tokens => [0..0]},
+                             'January')
+  end
+
+  def test_symbol_parsing
+    freeling = Ve::Provider::FreelingEn.new
+
+    assert_parses_into_words(freeling,
+                             {:words => ['.', ',', '$'],
+                              :lemmas => ['.', ',', '$'],
+                              :pos => [Ve::PartOfSpeech::Symbol, Ve::PartOfSpeech::Symbol, Ve::PartOfSpeech::Symbol],
+                              :extra => [{:grammar => nil}, {:grammar => nil}, {:grammar => nil}],
+                              :tokens => [0..0, 1..1, 2..2]},
+                             '.,$')
+  end
+
+  def test_can_handle_underscores_properly
+    # Should restore them
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse("In New York")
+    words = parse.words
+    tokens = parse.tokens
+
+    assert_equal ['In', 'New York'], words.collect(&:word)
+    assert_equal ['in', 'new york'], words.collect(&:lemma)
+    assert_equal [Ve::PartOfSpeech::Preposition, Ve::PartOfSpeech::ProperNoun], words.collect(&:part_of_speech)
+    assert_equal [{:grammar => nil}, {:grammar => nil}], words.collect(&:extra)
+    assert_equal [tokens[0..0], tokens[2..2]], words.collect(&:tokens)
+
+    # Should keep them
+    # TODO
+    freeling = Ve::Provider::FreelingEn.new
+    parse = freeling.parse("In New_York")
+    words = parse.words
+    tokens = parse.tokens
+
+    assert_equal ['In', 'New_York'], words.collect(&:word)
+    assert_equal ['in', 'new_york'], words.collect(&:lemma)
+    assert_equal [Ve::PartOfSpeech::Preposition, Ve::PartOfSpeech::ProperNoun], words.collect(&:part_of_speech)
+    assert_equal [{:grammar => nil}, {:grammarl => nil}], words.collect(&:extra)
+    assert_equal [tokens[0..1], tokens[2..2], tokens[3..11]], words.collect(&:tokens)
+  end
+
+end

data/tests/japanese_transliterators_test.rb

@@ -0,0 +1,79 @@
+# Encoding: UTF-8
+
+require_relative 'test_helper'
+
+class JapaneseTransliteratorsTest < Test::Unit::TestCase
+
+  KATAKANA = "ァアィイゥウェエォオカガキギクグケゲコゴサザシジスズセゼソゾタダチヂッツヅテデトドナニヌネノハバパヒビピフブプヘベペホボポマミムメモャヤュユョヨラリルレロヮワヰヱヲンヴヵヶ"
+  HIRAGANA = "ぁあぃいぅうぇえぉおかがきぎくぐけげこごさざしじすずせぜそぞただちぢっつづてでとどなにぬねのはばぱひびぴふぶぷへべぺほぼぽまみむめもゃやゅゆょよらりるれろゎわゐゑをんゔゕゖ"
+  HALFWIDTH = "!\"\#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~ "
+  FULLWIDTH = "！＂＃＄％＆＇（）＊＋，－．／０１２３４５６７８９：；＜＝＞？＠ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ［＼］＾＿｀ａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ｛｜｝～　"
+  
+  def setup
+    @trans = Ve::Provider::JapaneseTransliterators.new
+  end
+  
+  def test_should_be_able_to_start
+    assert @trans.works?
+  end
+
+  def test_transliterate_from_hira_to_latn
+    assert_equal 'kosoado',   @trans.parse('こそあど').transliterate_from_hira_to_latn
+    assert_equal 'konna',     @trans.parse('こんな').transliterate_from_hira_to_latn
+    assert_equal 'konyaku',   @trans.parse('こにゃく').transliterate_from_hira_to_latn
+    assert_equal 'kon\'yaku', @trans.parse('こんやく').transliterate_from_hira_to_latn
+    assert_equal 'shinbun',   @trans.parse('しんぶん').transliterate_from_hira_to_latn
+    assert_equal 'appa',      @trans.parse('あっぱ').transliterate_from_hira_to_latn
+  end
+  
+  def test_transliterate_from_latn_to_hrkt
+    assert_equal('かなです',   @trans.parse('kanadesu').transliterate_from_latn_to_hrkt)
+    assert_equal('こそあど',   @trans.parse('kosoado').transliterate_from_latn_to_hrkt)
+    assert_equal('こんな',    @trans.parse('konna').transliterate_from_latn_to_hrkt)
+    assert_equal('しんぶん',   @trans.parse('shimbun').transliterate_from_latn_to_hrkt)
+    assert_equal('しんぱい',   @trans.parse('simpai').transliterate_from_latn_to_hrkt)
+    assert_equal('うぁ',     @trans.parse('wha').transliterate_from_latn_to_hrkt)
+    assert_equal('かっちゃった', @trans.parse('katchatta').transliterate_from_latn_to_hrkt)
+    assert_equal('かっわいいぃ', @trans.parse('kawwaiixi').transliterate_from_latn_to_hrkt)
+    assert_equal('おっとせい',  @trans.parse('ottosei').transliterate_from_latn_to_hrkt)
+    assert_equal('あっち',    @trans.parse('acchi').transliterate_from_latn_to_hrkt)
+
+    # Katakana
+    assert_equal('カナデス',    @trans.parse('KANADESU').transliterate_from_latn_to_hrkt)
+    assert_equal('コソアド',    @trans.parse('KOSOADO').transliterate_from_latn_to_hrkt)
+    assert_equal('コンナ',     @trans.parse('KONNA').transliterate_from_latn_to_hrkt)
+    assert_equal('シンブン',    @trans.parse('SHIMBUN').transliterate_from_latn_to_hrkt)
+    assert_equal('シンパイ',    @trans.parse('SIMPAI').transliterate_from_latn_to_hrkt)
+    assert_equal('ウァ',      @trans.parse('WHA').transliterate_from_latn_to_hrkt)
+    assert_equal('カッチャッタ',  @trans.parse('KATCHATTA').transliterate_from_latn_to_hrkt)
+    assert_equal('カッワイイィ',  @trans.parse('KAWWAIIXI').transliterate_from_latn_to_hrkt)
+    assert_equal('オットセイ',   @trans.parse('OTTOSEI').transliterate_from_latn_to_hrkt)
+    assert_equal('アッチ',     @trans.parse('ACCHI').transliterate_from_latn_to_hrkt)
+    assert_equal('カタカナ です', @trans.parse('KATAKANA desu').transliterate_from_latn_to_hrkt)
+
+    # Non-Japanese
+    assert_equal('てぃs いs そめ えんgりsh', @trans.parse('this is some english').transliterate_from_latn_to_hrkt)
+  end
+
+  def test_transliterate_from_hira_to_kana
+    assert_equal KATAKANA, @trans.parse(HIRAGANA).transliterate_from_hira_to_kana
+  end
+
+  def test_transliterate_from_kata_to_hina
+    assert_equal HIRAGANA, @trans.parse(KATAKANA).transliterate_from_kana_to_hira
+  end
+
+  def test_transliterate_from_hrkt_to_latn
+    assert_equal 'hiraganakatakana', @trans.parse('ひらがなカタカナ').transliterate_from_hrkt_to_latn
+  end
+  
+  def test_transliterate_from_fullwidth_to_halfwidth
+    assert_equal HALFWIDTH, @trans.parse(FULLWIDTH).transliterate_from_fullwidth_to_halfwidth
+  end
+  
+  def test_transliterate_from_halfwidth_to_fullwidth
+    assert_equal FULLWIDTH, @trans.parse(HALFWIDTH).transliterate_from_halfwidth_to_fullwidth
+  end
+
+
+end

data/tests/mecab_ipadic_test.rb

@@ -0,0 +1,452 @@
+# Encoding: UTF-8
+
+require_relative 'test_helper'
+
+class MecabIpadicTest < Test::Unit::TestCase
+
+  def test_should_be_able_to_start
+    mecab = Ve::Provider::MecabIpadic.new
+    assert mecab.works?
+  end
+
+  def test_can_parse
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse('')
+    assert_equal Ve::Parse::MecabIpadic, parse.class
+  end
+
+  def test_all_literals_should_equal_the_input_text
+    text = <<-EOS
+    古池や
+    蛙飛び込む
+    水の音
+
+    EOS
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse(text)
+    assert_equal text, parse.tokens.collect { |t| t[:literal] }.join
+  end
+
+  def test_tokens_must_be_created_for_parsed_and_unparsed_text
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse(' A   B  ')
+    assert_equal [:unparsed, :parsed, :unparsed, :parsed, :unparsed, :sentence_split], parse.tokens.collect { |t| t[:type] }
+    assert_equal [' ', 'A', '   ', 'B', '  ', ''], parse.tokens.collect { |t| t[:literal] }
+    assert_equal [0..0, 1..1, 2..4, 5..5, 6..7, nil], parse.tokens.collect { |t| t[:characters] }
+  end
+
+  def test_tokens_should_not_be_modified_when_attached_to_words
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse('悪化する')
+    tokens = parse.tokens
+    assert_equal '悪化', tokens[0][:literal]
+    assert_equal '悪化', tokens[0][:lemma]
+  end
+
+  def test_sentences
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse('これは文章である。で、also containing some Englishですね')
+    assert_equal ['これは文章である。', 'で、also containing some Englishですね'], parse.sentences
+  end
+
+  def test_this_shouldnt_crash
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse('チューたろうは田中さんの犬です。')
+    assert_equal 11, parse.tokens.size
+  end
+
+  def test_this_shouldnt_crash_either
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse('三十年式歩兵銃')
+    assert_equal 7, parse.tokens.size
+  end
+
+  def test_words
+    mecab = Ve::Provider::MecabIpadic.new
+
+    # Meishi
+    assert_parses_into_words(mecab, {:words => ['車'],
+                              :lemmas => ['車'],
+                              :pos => [Ve::PartOfSpeech::Noun],
+                              :extra => [{:reading => 'クルマ', :transcription => 'クルマ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '車')
+
+    # Koyuumeishi
+    assert_parses_into_words(mecab, {:words => ['太郎'],
+                              :lemmas => ['太郎'],
+                              :pos => [Ve::PartOfSpeech::ProperNoun],
+                              :extra => [{:reading => 'タロウ', :transcription => 'タロー', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '太郎')
+
+    # Daimeishi
+    assert_parses_into_words(mecab, {:words => ['彼'],
+                              :lemmas => ['彼'],
+                              :pos => [Ve::PartOfSpeech::Pronoun],
+                              :extra => [{:reading => 'カレ', :transcription => 'カレ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '彼')
+
+    # Fukushikanou
+    assert_parses_into_words(mecab, {:words => ['午後に'],
+                              :lemmas => ['午後に'],
+                              :pos => [Ve::PartOfSpeech::Adverb],
+                              :extra => [{:reading => 'ゴゴニ', :transcription => 'ゴゴニ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '午後に')
+
+    # Kazu
+    assert_parses_into_words(mecab, {:words => ['一'],
+                              :lemmas => ['一'],
+                              :pos => [Ve::PartOfSpeech::Number],
+                              :extra => [{:reading => 'イチ', :transcription => 'イチ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '一')
+
+    assert_parses_into_words(mecab, {:words => ['１２３'],
+                              :lemmas => ['１２３'],
+                              :pos => [Ve::PartOfSpeech::Number],
+                              :extra => [{:reading => 'イチニサン', :transcription => 'イチニサン', :grammar => nil}],
+                              :tokens => [0..2]},
+                             '１２３')
+
+    # Sahensetsuzoku + tokumi ta
+    assert_parses_into_words(mecab, {:words => ['悪化した'],
+                              :lemmas => ['悪化する'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'アッカシタ', :transcription => 'アッカシタ', :grammar => nil}],
+                              :tokens => [0..2]},
+                             '悪化した')
+
+    # Keiyoudoushigokan
+    assert_parses_into_words(mecab, {:words => ['重要な'],
+                              :lemmas => ['重要'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'ジュウヨウナ', :transcription => 'ジューヨーナ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '重要な')
+
+    # Naikeiyoushigokan
+    assert_parses_into_words(mecab, {:words => ['とんでもない'],
+                              :lemmas => ['とんでもない'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'トンデモナイ', :transcription => 'トンデモナイ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             'とんでもない')
+
+    # Meishi hijiritsu fukushikanou
+    assert_parses_into_words(mecab, {:words => ['の', 'うちに'],
+                              :lemmas => ['の', 'うちに'],
+                              :pos => [Ve::PartOfSpeech::Postposition, Ve::PartOfSpeech::Adverb],
+                              :extra => [{:reading => 'ノ', :transcription => 'ノ', :grammar => nil},
+                                         {:reading => 'ウチニ', :transcription => 'ウチニ', :grammar => nil}],
+                              :tokens => [0..0, 1..2]},
+                             'のうちに')
+
+    # Meishi hijiritsu jodoushigokan
+    assert_parses_into_words(mecab, {:words => ['の', 'ような'],
+                              :lemmas => ['の', 'ようだ'],
+                              :pos => [Ve::PartOfSpeech::Postposition, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'ノ', :transcription => 'ノ', :grammar => nil},
+                                         {:reading => 'ヨウナ', :transcription => 'ヨーナ', :grammar => :auxillary}],
+                              :tokens => [0..0, 1..2]},
+                             'のような')
+
+    assert_parses_into_words(mecab, {:words => ['の', 'ように'],
+                              :lemmas => ['の', 'ように'],
+                              :pos => [Ve::PartOfSpeech::Postposition, Ve::PartOfSpeech::Adverb],
+                              :extra => [{:reading => 'ノ', :transcription => 'ノ', :grammar => nil},
+                                         {:reading => 'ヨウニ', :transcription => 'ヨーニ', :grammar => nil}],
+                              :tokens => [0..0, 1..2]},
+                             'のように')
+
+    # Meishi hijiritsu keiyoudoushigokan
+    assert_parses_into_words(mecab, {:words => ['みたいな'],
+                              :lemmas => ['みたいだ'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'ミタイナ', :transcription => 'ミタイナ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             'みたいな')
+
+    assert_parses_into_words(mecab, {:words => ['みたいの'],
+                              :lemmas => ['みたいの'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'ミタイノ', :transcription => 'ミタイノ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             'みたいの')
+
+    assert_parses_into_words(mecab, {:words => ['みたい', 'だ'],
+                              :lemmas => ['みたい', 'だ'],
+                              :pos => [Ve::PartOfSpeech::Adjective, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'ミタイ', :transcription => 'ミタイ', :grammar => nil},
+                                         {:reading => 'ダ', :transcription => 'ダ', :grammar => nil}],
+                              :tokens => [0..0, 1..1]},
+                             'みたいだ')
+
+    # Meishi tokushu jodoushigokan
+    assert_parses_into_words(mecab, {:words => ['行く', 'そう', 'だ'],
+                              :lemmas => ['行く', 'そう', 'だ'],
+                              :pos => [Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'イク', :transcription => 'イク', :grammar => nil},
+                                         {:reading => 'ソウ', :transcription => 'ソー', :grammar => :auxillary},
+                                         {:reading => 'ダ', :transcription => 'ダ', :grammar => nil}],
+                              :tokens => [0..0, 1..1, 2..2]},
+                             '行くそうだ')
+
+    # Meishi setsubi
+    # TODO: This should maybe be parsed as one noun instead
+    assert_parses_into_words(mecab, {:words => ['楽し', 'さ'],
+                              :lemmas => ['楽しい', 'さ'],
+                              :pos => [Ve::PartOfSpeech::Adjective, Ve::PartOfSpeech::Suffix],
+                              :extra => [{:reading => 'タノシ', :transcription => 'タノシ', :grammar => nil},
+                                         {:reading => 'サ', :transcription => 'サ', :grammar => nil}],
+                              :tokens => [0..0, 1..1]},
+                             '楽しさ')
+
+    # Meishi setsuzokushiteki
+    assert_parses_into_words(mecab, {:words => ['日本', '対', 'アメリカ'],
+                              :lemmas => ['日本', '対', 'アメリカ'],
+                              :pos => [Ve::PartOfSpeech::ProperNoun, Ve::PartOfSpeech::Conjunction, Ve::PartOfSpeech::ProperNoun],
+                              :extra => [{:reading => 'ニッポン', :transcription => 'ニッポン', :grammar => nil},
+                                         {:reading => 'タイ', :transcription => 'タイ', :grammar => nil},
+                                         {:reading => 'アメリカ', :transcription => 'アメリカ', :grammar => nil}],
+                              :tokens => [0..0, 1..1, 2..2]},
+                             '日本対アメリカ')
+
+    # Meishi doushihijiritsuteki
+    assert_parses_into_words(mecab, {:words => ['見て', 'ごらん'],
+                              :lemmas => ['見る', 'ごらん'],
+                              :pos => [Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'ミテ', :transcription => 'ミテ', :grammar => nil},
+                                         {:reading => 'ゴラン', :transcription => 'ゴラン', :grammar => :nominal}],
+                              :tokens => [0..1, 2..2]},
+                             '見てごらん')
+
+    # Settoushi
+    assert_parses_into_words(mecab, {:words => ['お', '座り'],
+                              :lemmas => ['お', '座り'],
+                              :pos => [Ve::PartOfSpeech::Prefix, Ve::PartOfSpeech::Noun],
+                              :extra => [{:reading => 'オ', :transcription => 'オ', :grammar => nil},
+                                         {:reading => 'スワリ', :transcription => 'スワリ', :grammar => nil}],
+                              :tokens => [0..0, 1..1]},
+                             'お座り')
+
+    # Kigou
+    assert_parses_into_words(mecab, {:words => ['。'],
+                              :lemmas => ['。'],
+                              :pos => [Ve::PartOfSpeech::Symbol],
+                              :extra => [{:reading => '。', :transcription => '。', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '。')
+
+    # Firaa
+    assert_parses_into_words(mecab, {:words => ['えと'],
+                              :lemmas => ['えと'],
+                              :pos => [Ve::PartOfSpeech::Interjection],
+                              :extra => [{:reading => 'エト', :transcription => 'エト', :grammar => nil}],
+                              :tokens => [0..0]},
+                             'えと')
+
+    # Sonota
+    assert_parses_into_words(mecab, {:words => ['だ', 'ァ'],
+                              :lemmas => ['だ', 'ァ'],
+                              :pos => [Ve::PartOfSpeech::Postposition, Ve::PartOfSpeech::Other],
+                              :extra => [{:reading => 'ダ', :transcription => 'ダ', :grammar => nil},
+                                         {:reading => 'ァ', :transcription => 'ア', :grammar => nil}],
+                              :tokens => [0..0, 1..1]},
+                             'だァ')
+
+    # Kandoushi
+    assert_parses_into_words(mecab, {:words => ['おはよう'],
+                              :lemmas => ['おはよう'],
+                              :pos => [Ve::PartOfSpeech::Interjection],
+                              :extra => [{:reading => 'オハヨウ', :transcription => 'オハヨー', :grammar => nil}],
+                              :tokens => [0..0]},
+                             'おはよう')
+
+    # Rentaishi
+    assert_parses_into_words(mecab, {:words => ['この'],
+                              :lemmas => ['この'],
+                              :pos => [Ve::PartOfSpeech::Determiner],
+                              :extra => [{:reading => 'コノ', :transcription => 'コノ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             'この')
+
+    # Setsuzokushi
+    assert_parses_into_words(mecab, {:words => ['そして'],
+                              :lemmas => ['そして'],
+                              :pos => [Ve::PartOfSpeech::Conjunction],
+                              :extra => [{:reading => 'ソシテ', :transcription => 'ソシテ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             'そして')
+
+    # Fukushi
+    assert_parses_into_words(mecab, {:words => ['多分'],
+                              :lemmas => ['多分'],
+                              :pos => [Ve::PartOfSpeech::Adverb],
+                              :extra => [{:reading => 'タブン', :transcription => 'タブン', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '多分')
+
+    # Doushi
+    assert_parses_into_words(mecab, {:words => ['行く'],
+                              :lemmas => ['行く'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'イク', :transcription => 'イク', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '行く')
+
+    assert_parses_into_words(mecab, {:words => ['行かない'],
+                              :lemmas => ['行く'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'イカナイ', :transcription => 'イカナイ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '行かない')
+
+    assert_parses_into_words(mecab, {:words => ['行って', 'きて'],
+                              :lemmas => ['行く', 'くる'],
+                              :pos => [Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'イッテ', :transcription => 'イッテ', :grammar => nil},
+                                         {:reading => 'キテ', :transcription => 'キテ', :grammar => :auxillary}],
+                              :tokens => [0..1, 2..3]},
+                             '行ってきて')
+
+    # Doushi setsubi
+    assert_parses_into_words(mecab, {:words => ['行かれる'],
+                              :lemmas => ['行く'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'イカレル', :transcription => 'イカレル', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '行かれる')
+
+    assert_parses_into_words(mecab, {:words => ['食べさせられた'],
+                              :lemmas => ['食べる'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'タベサセラレタ', :transcription => 'タベサセラレタ', :grammar => nil}],
+                              :tokens => [0..3]},
+                             '食べさせられた')
+
+    # Doushi + jodoushi
+    assert_parses_into_words(mecab, {:words => ['食べました'],
+                              :lemmas => ['食べる'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'タベマシタ', :transcription => 'タベマシタ', :grammar => nil}],
+                              :tokens => [0..2]},
+                             '食べました')
+
+    # Keiyoushi
+    assert_parses_into_words(mecab, {:words => ['寒い'],
+                              :lemmas => ['寒い'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'サムイ', :transcription => 'サムイ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '寒い')
+
+    assert_parses_into_words(mecab, {:words => ['寒くて'],
+                              :lemmas => ['寒い'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'サムクテ', :transcription => 'サムクテ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '寒くて')
+
+    assert_parses_into_words(mecab, {:words => ['寒かった'],
+                              :lemmas => ['寒い'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'サムカッタ', :transcription => 'サムカッタ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '寒かった')
+
+    assert_parses_into_words(mecab, {:words => ['寒ければ'],
+                              :lemmas => ['寒い'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'サムケレバ', :transcription => 'サムケレバ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '寒ければ')
+
+    assert_parses_into_words(mecab, {:words => ['寒けりゃ'],
+                              :lemmas => ['寒い'],
+                              :pos => [Ve::PartOfSpeech::Adjective],
+                              :extra => [{:reading => 'サムケリャ', :transcription => 'サムケリャ', :grammar => nil}],
+                              :tokens => [0..0]},
+                             '寒けりゃ')
+
+    assert_parses_into_words(mecab, {:words => ['食べたい'],
+                              :lemmas => ['食べる'],
+                              :pos => [Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'タベタイ', :transcription => 'タベタイ', :grammar => nil}],
+                              :tokens => [0..1]},
+                             '食べたい')
+
+    # Joshi
+    assert_parses_into_words(mecab, {:words => ['日本', 'から'],
+                              :lemmas => ['日本', 'から'],
+                              :pos => [Ve::PartOfSpeech::ProperNoun, Ve::PartOfSpeech::Postposition],
+                              :extra => [{:reading => 'ニッポン', :transcription => 'ニッポン', :grammar => nil},
+                                         {:reading => 'カラ', :transcription => 'カラ', :grammar => nil}],
+                              :tokens => [0..0, 1..1]},
+                             '日本から')
+
+    # The copula
+    assert_parses_into_words(mecab, {:words => ['日本', 'です'],
+                              :lemmas => ['日本', 'です'],
+                              :pos => [Ve::PartOfSpeech::ProperNoun, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'ニッポン', :transcription => 'ニッポン', :grammar => nil},
+                                         {:reading => 'デス', :transcription => 'デス', :grammar => nil}],
+                              :tokens => [0..0, 1..1]},
+                             '日本です')
+
+    assert_parses_into_words(mecab, {:words => ['日本', 'だった'],
+                              :lemmas => ['日本', 'だ'],
+                              :pos => [Ve::PartOfSpeech::ProperNoun, Ve::PartOfSpeech::Verb],
+                              :extra => [{:reading => 'ニッポン', :transcription => 'ニッポン', :grammar => nil},
+                                         {:reading => 'ダッタ', :transcription => 'ダッタ', :grammar => nil}],
+                              :tokens => [0..0, 1..2]},
+                             '日本だった')
+
+    # いるから
+    assert_parses_into_words(mecab, {:words => ['いる', 'から'],
+                             :lemmas => ['いる', 'から'],
+                             :pos => [Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Postposition],
+                             :extra => [{:reading => 'イル', :transcription => 'イル', :grammar => nil},
+                                        {:reading => 'カラ', :transcription => 'カラ', :grammar => nil}],
+                             :tokens => [0..0, 1..1]},
+                             'いるから')
+
+    # しているから
+    assert_parses_into_words(mecab, {:words => ['して', 'いる', 'から'],
+                             :lemmas => ['する', 'いる', 'から'],
+                             :pos => [Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Postposition],
+                             :extra => [{:reading => 'シテ', :transcription => 'シテ', :grammar => nil},
+                                        {:reading => 'イル', :transcription => 'イル', :grammar => :auxillary},
+                                        {:reading => 'カラ', :transcription => 'カラ', :grammar => nil}],
+                             :tokens => [0..0, 1..1, 2..2]},
+                             'しているから')
+
+    # 基準があるが、
+    assert_parses_into_words(mecab, {:words => ['して', 'いる', 'から'],
+                             :lemmas => ['する', 'いる', 'から'],
+                             :pos => [Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Verb, Ve::PartOfSpeech::Postposition],
+                             :extra => [{:reading => 'シテ', :transcription => 'シテ', :grammar => nil},
+                                        {:reading => 'イル', :transcription => 'イル', :grammar => :auxillary},
+                                          {:reading => 'カラ', :transcription => 'カラ', :grammar => nil}],
+                             :tokens => [0..0, 1..1, 2..2]},
+                             '基準があるが、')
+
+    # TODO: xした should parse as adjective?
+    assert_parses_into_words(mecab, {:words => [],
+                              :lemmas => [],
+                              :pos => [],
+                              :extra => [],
+                              :tokens => []},
+                             '')
+  end
+
+  def todo_test_word_transliteration
+    mecab = Ve::Provider::MecabIpadic.new
+    parse = mecab.parse('日本', :transliterate_words => :latn)
+
+    assert_equal 'nihon', parse.words.first.transliteration(:latn)
+  end
+
+end