docdiff 0.5.0

data/test/diff_test.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/ruby
+require 'test/unit'
+require "docdiff/diff"
+
+class TC_Diff < Test::Unit::TestCase
+
+  def setup()
+    #
+  end
+
+  def test_new_ses()
+    a1 = [:a, :b, :c]
+    a2 = [:a, :x, :c]
+    expected = [[:common_elt_elt, [:a], [:a]],
+                [:del_elt,        [:b], nil],
+                [:add_elt,         nil, [:x]],
+                [:common_elt_elt, [:c], [:c]]]
+    actual              = []
+    actual_speculative  = []
+    actual_shortestpath = []
+    actual_contours     = []
+    Diff.new(a1, a2).ses               .each{|e| actual              << e}
+    Diff.new(a1, a2).ses(:speculative ).each{|e| actual_speculative  << e}
+    Diff.new(a1, a2).ses(:shortestpath).each{|e| actual_shortestpath << e}
+    Diff.new(a1, a2).ses(:contours    ).each{|e| actual_contours     << e}
+    assert_equal(expected, actual)
+    assert_equal(expected, actual_speculative)
+    assert_equal(expected, actual_shortestpath)
+    assert_equal(expected, actual_contours)
+  end
+
+  def teardown()
+    #
+  end
+
+end

data/test/difference_test.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/ruby
+require 'test/unit'
+require 'docdiff/difference'
+
+class TC_Difference < Test::Unit::TestCase
+
+  def setup()
+    #
+  end
+
+  def test_new()
+    array1 = [:a, :b, :c]
+    array2 = [:a, :x, :c]
+    expected =   [[:common_elt_elt, [:a], [:a]],
+                  [:change_elt,     [:b], [:x]],
+                  [:common_elt_elt, [:c], [:c]]]
+    assert_equal(expected, Difference.new(array1, array2))
+  end
+
+  def test_raw_list()
+    array1 = [:a, :b, :c]
+    array2 = [:a, :x, :c]
+    expected =   [[:common_elt_elt, [:a], [:a]],
+                  [:del_elt,        [:b], nil],
+                  [:add_elt,         nil, [:x]],
+                  [:common_elt_elt, [:c], [:c]]]
+    assert_equal(expected, Difference.new(array1, array2).raw_list)
+  end
+
+  def test_former_only()
+    array1 = [:a, :b, :c]
+    array2 = [:a, :x, :c]
+    expected =   [[:common_elt_elt, [:a], [:a]],
+                  [:change_elt,     [:b], nil],
+                  [:common_elt_elt, [:c], [:c]]]
+    assert_equal(expected, Difference.new(array1, array2).former_only)
+    array1 = [:a, :b, :c]
+    array2 = [:a, :c, :d]
+    expected =   [[:common_elt_elt, [:a], [:a]],
+                  [:del_elt,        [:b], nil],
+                  [:common_elt_elt, [:c], [:c]]]
+    assert_equal(expected, Difference.new(array1, array2).former_only)
+  end
+
+  def test_latter_only()
+    array1 = [:a, :b, :c]
+    array2 = [:a, :x, :c]
+    expected =   [[:common_elt_elt, [:a], [:a]],
+                  [:change_elt,     nil,  [:x]],
+                  [:common_elt_elt, [:c], [:c]]]
+    assert_equal(expected, Difference.new(array1, array2).latter_only)
+    array1 = [:a, :b, :c]
+    array2 = [:a, :c, :d]
+    expected =   [[:common_elt_elt, [:a], [:a]],
+                  [:common_elt_elt, [:c], [:c]],
+                  [:add_elt,        nil,  [:d]]]
+    assert_equal(expected, Difference.new(array1, array2).latter_only)
+  end
+
+  def teardown()
+    #
+  end
+
+end

data/test/docdiff_test.rb

@@ -0,0 +1,193 @@
+#!/usr/bin/ruby
+# -*- coding: us-ascii; -*-
+require 'test/unit'
+require 'docdiff'
+require 'nkf'
+
+class TC_Document < Test::Unit::TestCase
+
+  def setup()
+    #
+  end
+
+  def test_compare_by_line()
+    doc1 = Document.new("Foo bar.\nBaz quux.", 'US-ASCII', 'LF')
+    doc2 = Document.new("Foo.\nBaz quux.", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = [[:change_elt,     ["Foo bar.\n"], ["Foo.\n"]],
+                [:common_elt_elt, ['Baz quux.'], ['Baz quux.']]]
+    assert_equal(expected, docdiff.compare_by_line(doc1, doc2))
+  end
+  def test_compare_by_line_word()
+    doc1 = Document.new("a b c d\ne f", 'US-ASCII', 'LF')
+    doc2 = Document.new("a x c d\ne f", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = [[:common_elt_elt, ["a "], ["a "]],
+                [:change_elt,     ["b "], ["x "]],
+                [:common_elt_elt, ["c ", "d", "\n"], ["c ", "d", "\n"]],
+                [:common_elt_elt, ["e f"], ["e f"]]]
+    assert_equal(expected,
+                 docdiff.compare_by_line_word(doc1, doc2))
+  end
+  def test_compare_by_line_word_char()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = [[:common_elt_elt, ['foo '], ['foo ']],
+                [:common_elt_elt, ['b'], ['b']],
+                [:change_elt,     ['a'], ['e', 'e']],
+                [:common_elt_elt, ['r'], ['r']],
+                [:common_elt_elt, ["\n"], ["\n"]],
+                [:common_elt_elt, ['baz'], ['baz']]]
+    assert_equal(expected,
+                 docdiff.compare_by_line_word_char(doc1, doc2))
+  end
+
+  def test_run_line_html()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = '<?xml version="1.0" encoding="US-ASCII"?>' + "\n" +
+     '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"' + "\n" +
+     '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">' + "\n" +
+     '<html><head>' + "\n" +
+     '<meta http-equiv="Content-Type" content="text/html; charset=US-ASCII" />' + "\n" +
+     '<title>Difference</title>' + "\n" +
+     '<style type="text/css">' + "\n" +
+     ' body {font-family: monospace;}' + "\n" +
+     ' span.del {background: hotpink; border: thin inset;}' + "\n" +
+     ' span.add {background: deepskyblue; font-weight: bolder; border: thin outset;}' + "\n" +
+     ' span.before-change {background: yellow; border: thin inset;}' + "\n" +
+     ' span.after-change {background: lime; font-weight: bolder; border: thin outset;}' + "\n" +
+     " li.entry .position {font-weight: bolder; margin-top: 0em; margin-bottom: 0em; padding-top: 0.5em; padding-bottom: 0em;}\n" +
+     " li.entry .body {margin-top: 0em; margin-bottom: 0em; padding-top: 0em; padding-bottom: 0.5em;}\n" +
+     " li.entry {border-top: thin solid gray;}\n" +
+     '</style>' + "\n" +
+     '</head><body><div>' + "\n" +
+     '<span class="before-change"><del>foo bar<br />' + "\n" + '</del></span>' +
+     '<span class="after-change"><ins>foo beer<br />' + "\n" + '</ins></span>' +
+     '<span class="common">baz' + "</span>" + "\n</div></body></html>" + "\n"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "line", :format => "html", :digest => false}))
+  end
+
+  def test_run_line_manued()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = "defparentheses [ ]\n" +
+               "defdelete      /\n" +
+               "defswap        |\n" +
+               "defcomment     ;\n" +
+               "defescape      ~\n" +
+               "deforder       newer-last\n" +
+               "defversion     0.9.5\n" + 
+               "[foo bar\n/foo beer\n]baz"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "line", :format => "manued", :digest => false}))
+  end
+  def test_run_word_manued()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = "defparentheses [ ]\n" +
+               "defdelete      /\n" +
+               "defswap        |\n" +
+               "defcomment     ;\n" +
+               "defescape      ~\n" +
+               "deforder       newer-last\n" +
+               "defversion     0.9.5\n" + 
+               "foo [bar/beer]\nbaz"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "word", :format => "manued", :digest => false}))
+  end
+  def test_run_char_manued()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    docdiff = DocDiff.new
+    expected = "defparentheses [ ]\n" +
+               "defdelete      /\n" +
+               "defswap        |\n" +
+               "defcomment     ;\n" +
+               "defescape      ~\n" +
+               "deforder       newer-last\n" +
+               "defversion     0.9.5\n" + 
+               "foo b[a/ee]r\nbaz"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "char", :format => "manued", :digest => false}))
+  end
+
+  def test_parse_config_file_content()
+    content = ["# comment line\n",
+               " # comment line with leading space\n",
+               "foo1 = bar\n",
+               "foo2 = bar baz \n",
+               " foo3  =  123 # comment\n",
+               "foo4 = no    \n",
+               "foo1 = tRue\n",
+               "\n",
+               "",
+               nil].join
+    expected = {:foo1=>true, :foo2=>"bar baz", :foo3=>123, :foo4=>false}
+    docdiff = DocDiff.new
+    assert_equal(expected,
+                 DocDiff.parse_config_file_content(content))
+  end
+
+  def test_run_line_user()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    config = {:tag_common_start          => '<=>',
+              :tag_common_end            => '</=>',
+              :tag_del_start             => '<->',
+              :tag_del_end               => '</->',
+              :tag_add_start             => '<+>',
+              :tag_add_end               => '</+>',
+              :tag_change_before_start   => '<!->',
+              :tag_change_before_end     => '</!->',
+              :tag_change_after_start    => '<!+>',
+              :tag_change_after_end      => '</!+>'}
+    docdiff = DocDiff.new
+    docdiff.config.update(config)
+    expected = "<!->foo bar\n</!-><!+>foo beer\n</!+><=>baz</=>"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "line", :format => "user", :digest => false}))
+  end
+  def test_run_word_user()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    config = {:tag_common_start          => '<=>',
+              :tag_common_end            => '</=>',
+              :tag_del_start             => '<->',
+              :tag_del_end               => '</->',
+              :tag_add_start             => '<+>',
+              :tag_add_end               => '</+>',
+              :tag_change_before_start   => '<!->',
+              :tag_change_before_end     => '</!->',
+              :tag_change_after_start    => '<!+>',
+              :tag_change_after_end      => '</!+>'}
+    docdiff = DocDiff.new
+    docdiff.config.update(config)
+    expected = "<=>foo </=><!->bar</!-><!+>beer</!+><=>\n</=><=>baz</=>"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "word", :format => "user", :digest => false}))
+  end
+  def test_run_char_user()
+    doc1 = Document.new("foo bar\nbaz", 'US-ASCII', 'LF')
+    doc2 = Document.new("foo beer\nbaz", 'US-ASCII', 'LF')
+    config = {:tag_common_start          => '<=>',
+              :tag_common_end            => '</=>',
+              :tag_del_start             => '<->',
+              :tag_del_end               => '</->',
+              :tag_add_start             => '<+>',
+              :tag_add_end               => '</+>',
+              :tag_change_before_start   => '<!->',
+              :tag_change_before_end     => '</!->',
+              :tag_change_after_start    => '<!+>',
+              :tag_change_after_end      => '</!+>'}
+    docdiff = DocDiff.new
+    docdiff.config.update(config)
+    expected = "<=>foo </=><=>b</=><!->a</!-><!+>ee</!+><=>r</=><=>\n</=><=>baz</=>"
+    assert_equal(expected, docdiff.run(doc1, doc2, {:resolution => "char", :format => "user", :digest => false}))
+  end
+
+
+  def teardown()
+    #
+  end
+
+end

data/test/document_test.rb

@@ -0,0 +1,626 @@
+#!/usr/bin/ruby
+# -*- coding: euc-jp; -*-
+require 'test/unit'
+require 'docdiff/document'
+require 'nkf'
+
+class TC_Document < Test::Unit::TestCase
+
+  def setup()
+    #
+  end
+
+  def test_encoding()
+    doc = Document.new("Foo bar.\nBaz quux.")
+    doc.encoding = 'US-ASCII'
+    doc.eol = 'LF'
+    expected = 'US-ASCII'
+    assert_equal(expected, doc.encoding)
+  end
+  def test_encoding_auto()
+    doc = if CharString.ruby_m17n?
+            Document.new("Foo bar.\nBaz quux.".encode("US-ASCII"))
+          else
+            Document.new("Foo bar.\nBaz quux.")
+          end
+    expected = 'US-ASCII'
+    assert_equal(expected, doc.encoding)
+  end
+  def test_eol()
+    doc = Document.new("Foo bar.\nBaz quux.")
+    doc.encoding = 'US-ASCII'
+    doc.eol = 'LF'
+    expected = 'LF'
+    assert_equal(expected, doc.eol)
+  end
+  def test_eol_auto_lf()
+    doc = Document.new("Foo bar.\nBaz quux.")
+    expected = 'LF'
+    assert_equal(expected, doc.eol)
+  end
+  def test_eol_auto_none()
+    doc = Document.new("Foo bar.")
+    expected = "NONE"
+    assert_equal(expected, doc.eol)
+  end
+  def test_eol_char_lf()
+    doc = Document.new("Foo bar.\nBaz quux.")
+#    doc.encoding = "US-ASCII"
+#    doc.eol = "LF"
+    expected = "\n"
+    assert_equal(expected, doc.eol_char)
+  end
+  def test_split_by_line()
+    doc = Document.new("Hello, my name is Watanabe.\nI am just another Ruby porter.\n")
+    expected = ["Hello, my name is Watanabe.\n", "I am just another Ruby porter.\n"]
+    assert_equal(expected, doc.split_to_line)
+  end
+
+  # test eol split_to_line() method
+  def test_cr_split_to_line()
+    doc = Document.new("foo\rbar\r")
+    expected = ["foo\r", "bar\r"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_cr_split_to_line_chomped_lastline()
+    doc = Document.new("foo\rbar")
+    expected = ["foo\r", "bar"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_cr_split_to_line_empty_line()
+    doc = Document.new("foo\r\rbar\r")
+    expected = ["foo\r", "\r", "bar\r"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_lf_split_to_line()
+    doc = Document.new("foo\nbar\n")
+    expected = ["foo\n", "bar\n"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_lf_split_to_line_chomped_lastline()
+    doc = Document.new("foo\nbar")
+    expected = ["foo\n", "bar"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_lf_split_to_line_empty_line()
+    doc = Document.new("foo\n\nbar\n")
+    expected = ["foo\n", "\n", "bar\n"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_crlf_split_to_line()
+    doc = Document.new("foo\r\nbar\r\n")
+    expected = ["foo\r\n", "bar\r\n"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_crlf_split_to_line_chomped_lastline()
+    doc = Document.new("foo\r\nbar")
+    expected = ["foo\r\n", "bar"]
+    assert_equal(expected, doc.split_to_line)
+  end
+  def test_crlf_split_to_line_empty_line()
+    doc = Document.new("foo\r\n\r\nbar\r\n")
+    expected = ["foo\r\n", "\r\n", "bar\r\n"]
+    assert_equal(expected, doc.split_to_line)
+  end
+
+  # test ASCII module
+  def test_ascii_split_to_word()
+    doc = Document.new("foo bar")
+    expected = ["foo ", "bar"]
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_ascii_split_to_word_withsymbol()
+    doc = Document.new("foo (bar) baz-baz")
+    expected = ["foo ", "(bar) ", "baz-baz"]
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_ascii_split_to_word_withquote()
+    doc = Document.new("foo's 'foo' \"bar\" 'baz.'")
+    expected = ["foo's ", "'foo' ", "\"bar\" ", "'baz.'"]
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_ascii_split_to_word_withlongspace()
+    doc = Document.new(" foo  bar")
+    expected = [" ", "foo ", " ", "bar"]
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_ascii_split_to_word_withdash()
+    doc = Document.new("foo -- bar, baz - quux")
+    expected = ["foo ", "-- ", "bar, ", "baz ", "- ", "quux"]
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_ascii_split_to_char()
+    doc = Document.new("foo bar")
+    expected = ["f","o","o"," ","b","a","r"]
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_ascii_split_to_char_with_eol_cr()
+    doc = Document.new("foo bar\r")
+    expected = ["f","o","o"," ","b","a","r","\r"]
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_ascii_split_to_char_with_eol_lf()
+    doc = Document.new("foo bar\n")
+    expected = ["f","o","o"," ","b","a","r","\n"]
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_ascii_split_to_char_with_eol_crlf()
+    doc = Document.new("foo bar\r\n")
+    expected = ["f","o","o"," ","b","a","r","\r\n"]
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_ascii_split_to_byte()
+    doc = Document.new("foo bar\r\n")
+    expected = ["f","o","o"," ","b","a","r","\r","\n"]
+    assert_equal(expected, doc.split_to_byte)
+  end
+  def test_ascii_count_byte()
+    doc = Document.new("foo bar\r\n")
+    expected = 9
+    assert_equal(expected, doc.count_byte)
+  end
+  def test_ascii_count_char()
+    doc = Document.new("foo bar\r\nbaz quux\r\n")
+    expected = 17
+    assert_equal(expected, doc.count_char)
+  end
+  def test_ascii_count_latin_graph_char()
+    doc = Document.new("foo bar\r\nbaz quux\r\n")
+    expected = 13
+    assert_equal(expected, doc.count_latin_graph_char)
+  end
+  def test_ascii_count_graph_char()
+    doc = Document.new("foo bar\r\nbaz quux\r\n")
+    expected = 13
+    assert_equal(expected, doc.count_graph_char)
+  end
+  def test_ascii_count_latin_blank_char()
+    doc = Document.new("foo bar\r\nbaz\tquux\r\n")
+    expected = 2
+    assert_equal(expected, doc.count_latin_blank_char)
+  end
+  def test_ascii_count_blank_char()
+    doc = Document.new("foo bar\r\nbaz\tquux\r\n")
+    expected = 2
+    assert_equal(expected, doc.count_blank_char)
+  end
+  def test_ascii_count_word()
+    doc = Document.new("foo bar   \r\nbaz quux\r\n")
+    expected = 6
+    assert_equal(expected, doc.count_word)
+  end
+  def test_ascii_count_latin_word()
+    doc = Document.new("foo bar   \r\nbaz quux\r\n")
+    expected = 5  # "  " is also counted as a word
+    assert_equal(expected, doc.count_latin_word)
+  end
+  def test_ascii_count_latin_valid_word()
+    doc = Document.new("1 foo   \r\n%%% ()\r\n")
+    expected = 2
+    assert_equal(expected, doc.count_latin_valid_word)
+  end
+  def test_ascii_count_line()
+    doc = Document.new("foo\r\nbar")
+    expected = 2
+    assert_equal(expected, doc.count_line)
+  end
+  def test_ascii_count_graph_line()
+    doc = Document.new("foo\r\n ")
+    expected = 1
+    assert_equal(expected, doc.count_graph_line)
+  end
+  def test_ascii_count_empty_line()
+    doc = Document.new("foo\r\n \r\n\t\r\n\r\n")
+    expected = 1
+    assert_equal(expected, doc.count_empty_line)
+  end
+  def test_ascii_count_blank_line()
+    doc = Document.new("\r\n \r\n\t\r\n ")
+    expected = 3
+    assert_equal(expected, doc.count_blank_line)
+  end
+
+  # test EUCJP module
+  def test_eucjp_split_to_word()
+    doc = Document.new(NKF.nkf("-e", "���ܸ��ʸ��foo bar"))
+    expected = ["���ܸ��","ʸ��","foo ","bar"].collect{|c| NKF.nkf("-e", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_eucjp_split_to_word_kanhira()
+    doc = Document.new(NKF.nkf("-e", "���ܸ��ʸ��"))
+    expected = ["���ܸ��", "ʸ��"].collect{|c| NKF.nkf("-e", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_eucjp_split_to_word_katahira()
+    doc = Document.new(NKF.nkf("-e", "�������ʤ�ʸ��"))
+    expected = ["�������ʤ�", "ʸ��"].collect{|c| NKF.nkf("-e", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_eucjp_split_to_word_kataonbiki()
+    doc = Document.new(NKF.nkf("-e", "��ӡ�������"), "EUC-JP")
+    expected = ["��ӡ�", "����", "��"].collect{|c| NKF.nkf("-e", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_eucjp_split_to_word_hiraonbiki()
+    doc = Document.new(NKF.nkf("-e", "���ӡ���"), "EUC-JP")
+    expected = (["�", "��ӡ���"]).collect{|c| NKF.nkf("-e", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_eucjp_split_to_word_latinmix()
+    doc = Document.new(NKF.nkf("-e", "���ܸ��Latin��ʸ��"))
+    expected = ["���ܸ��", "Latin", "��", "ʸ��"].collect{|c| NKF.nkf("-e", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_eucjp_split_to_char()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b"))
+    expected = ["��","��","��","a"," ","b"].collect{|c|NKF.nkf("-e",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_eucjp_split_to_char_with_cr()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\r"))
+    expected = ["��","��","��","a"," ","b","\r"].collect{|c|NKF.nkf("-e",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_eucjp_split_to_char_with_lf()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\n"))
+    expected = ["��","��","��","a"," ","b","\n"].collect{|c|NKF.nkf("-e",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_eucjp_split_to_char_with_crlf()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\r\n"))
+    expected = ["��","��","��","a"," ","b","\r\n"].collect{|c|NKF.nkf("-e",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_eucjp_count_char()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\r\n"))
+    expected = 7
+    assert_equal(expected, doc.count_char)
+  end
+  def test_eucjp_count_latin_graph_char()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_graph_char)
+  end
+  def test_eucjp_count_ja_graph_char()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_ja_graph_char)
+  end
+  def test_eucjp_count_graph_char()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�a b\r\n"))
+    expected = 5
+    assert_equal(expected, doc.count_graph_char)
+  end
+  def test_eucjp_count_latin_blank_char()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�\ta b\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_blank_char)
+  end
+  def test_eucjp_count_ja_blank_char()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���\ta b\r\n"))
+    expected = 1
+    assert_equal(expected, doc.count_ja_blank_char)
+  end
+  def test_eucjp_count_blank_char()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���\ta b\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_blank_char)
+  end
+  def test_eucjp_count_word()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���a b --\r\n"))
+    expected = 7 # "--" and "\r\n" are counted as word here (though not "valid")
+    assert_equal(expected, doc.count_word)
+  end
+  def test_eucjp_count_ja_word()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���a b --\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_ja_word)
+  end
+  def test_eucjp_count_latin_valid_word()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���a b --\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_valid_word)
+  end
+  def test_eucjp_count_ja_valid_word()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���a b --\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_ja_valid_word)
+  end
+  def test_eucjp_count_valid_word()
+    doc = Document.new(NKF.nkf("-e", "���ܡ���a b --\r\n"))
+    expected = 4
+    assert_equal(expected, doc.count_valid_word)
+  end
+  def test_eucjp_count_line()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 6
+    assert_equal(expected, doc.count_line)
+  end
+  def test_eucjp_count_graph_line()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 3
+    assert_equal(expected, doc.count_graph_line)
+  end
+  def test_eucjp_count_empty_line()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 1
+    assert_equal(expected, doc.count_empty_line)
+  end
+  def test_eucjp_count_blank_line()
+    doc = Document.new(NKF.nkf("-e", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 2
+    assert_equal(expected, doc.count_blank_line)
+  end
+
+  # test SJIS module
+  def test_sjis_split_to_word()
+    doc = Document.new(NKF.nkf("-s", "���ܸ��ʸ��foo bar"))
+    expected = ["���ܸ��", "ʸ��", "foo ", "bar"].collect{|c|NKF.nkf("-s",c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_sjisplit_s_to_word_kanhira()
+    doc = Document.new(NKF.nkf("-s", "���ܸ��ʸ��"))
+    expected = ["���ܸ��", "ʸ��"].collect{|c| NKF.nkf("-s", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_sjis_split_to_word_katahira()
+    doc = Document.new(NKF.nkf("-s", "�������ʤ�ʸ��"))
+    expected = ["�������ʤ�", "ʸ��"].collect{|c| NKF.nkf("-s", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_sjis_split_to_word_kataonbiki()
+    doc = Document.new(NKF.nkf("-s", "��ӡ��λ���"))
+    expected = ["��ӡ���", "����"].collect{|c| NKF.nkf("-s", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_sjis_split_to_word_hiraonbiki()
+    doc = Document.new(NKF.nkf("-s", "���ӡ���"))
+    expected = ["�", "��ӡ���"].collect{|c| NKF.nkf("-s", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_sjis_split_to_word_latinmix()
+    doc = Document.new(NKF.nkf("-s", "���ܸ��Latin��ʸ��"))
+    expected = ["���ܸ��","Latin","��","ʸ��"].collect{|c| NKF.nkf("-s", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_sjis_split_to_char()
+    doc = Document.new(NKF.nkf("-s", "ɽ�׻�a b"))
+    expected = ["ɽ","��","��","a"," ","b"].collect{|c|NKF.nkf("-s",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_sjis_split_to_char_with_cr()
+    doc = Document.new(NKF.nkf("-s", "ɽ�׻�a b\r"))
+    expected = ["ɽ","��","��","a"," ","b","\r"].collect{|c|NKF.nkf("-s",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_sjis_split_to_char_with_lf()
+    doc = Document.new(NKF.nkf("-s", "ɽ�׻�a b\n"))
+    expected = ["ɽ","��","��","a"," ","b","\n"].collect{|c|NKF.nkf("-s",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_sjis_split_to_char_with_crlf()
+    doc = Document.new(NKF.nkf("-s", "ɽ�׻�a b\r\n"))
+    expected = ["ɽ","��","��","a"," ","b","\r\n"].collect{|c|NKF.nkf("-s",c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_sjis_count_char()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�a b\r\n"))
+    expected = 7
+    assert_equal(expected, doc.count_char)
+  end
+  def test_sjis_count_latin_graph_char()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�a b\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_graph_char)
+  end
+  def test_sjis_count_ja_graph_char()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�a b\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_ja_graph_char)
+  end
+  def test_sjis_count_graph_char()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�a b\r\n"))
+    expected = 5
+    assert_equal(expected, doc.count_graph_char)
+  end
+  def test_sjis_count_latin_blank_char()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�\ta b\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_blank_char)
+  end
+  def test_sjis_count_ja_blank_char()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���\ta b\r\n"))
+    expected = 1
+    assert_equal(expected, doc.count_ja_blank_char)
+  end
+  def test_sjis_count_blank_char()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���\ta b\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_blank_char)
+  end
+  def test_sjis_count_word()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���a b --\r\n"))
+    expected = 7 # "--" and "\r\n" are counted as word here (though not "valid")
+    assert_equal(expected, doc.count_word)
+  end
+  def test_sjis_count_ja_word()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���a b --\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_ja_word)
+  end
+  def test_sjis_count_latin_valid_word()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���a b --\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_valid_word)
+  end
+  def test_sjis_count_ja_valid_word()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���a b --\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_ja_valid_word)
+  end
+  def test_sjis_count_valid_word()
+    doc = Document.new(NKF.nkf("-s", "���ܡ���a b --\r\n"))
+    expected = 4
+    assert_equal(expected, doc.count_valid_word)
+  end
+  def test_sjis_count_line()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 6
+    assert_equal(expected, doc.count_line)
+  end
+  def test_sjis_count_graph_line()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 3
+    assert_equal(expected, doc.count_graph_line)
+  end
+  def test_sjis_count_empty_line()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 1
+    assert_equal(expected, doc.count_empty_line)
+  end
+  def test_sjis_count_blank_line()
+    doc = Document.new(NKF.nkf("-s", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 2
+    assert_equal(expected, doc.count_blank_line)
+  end
+
+  # test UTF8 module
+  def test_utf8_split_to_word()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ��ʸ��foo bar"))
+    expected = ["���ܸ��", "ʸ��", "foo ", "bar"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_utf8_split_to_word_kanhira()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ��ʸ��"))
+    expected = ["���ܸ��", "ʸ��"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_utf8_split_to_word_katahira()
+    doc = Document.new(NKF.nkf("-E -w", "�������ʤ�ʸ��"))
+    expected = ["�������ʤ�", "ʸ��"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_utf8_split_to_word_kataonbiki()
+    doc = Document.new(NKF.nkf("-E -w", "��ӡ��λ���"))
+    expected = ["��ӡ���", "����"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_utf8_split_to_word_hiraonbiki()
+    doc = Document.new(NKF.nkf("-E -w", "���ӡ���"))
+    expected = ["�", "��ӡ���"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_utf8_split_to_word_latinmix()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ��Latin��ʸ��"))
+    expected = ["���ܸ��", "Latin", "��", "ʸ��"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_word)
+  end
+  def test_utf8_split_to_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b"), "UTF-8")
+    expected = ["��", "��", "��", "a", " ", "b"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_utf8_split_to_char_with_cr()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\r"), "UTF-8")
+    expected = ["��","��","��","a"," ","b","\r"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_utf8_split_to_char_with_lf()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\n"), "UTF-8")
+    expected = ["��","��","��","a"," ","b","\n"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_utf8_split_to_char_with_crlf()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\r\n"), "UTF-8")
+    expected = ["��","��","��","a"," ","b","\r\n"].collect{|c| NKF.nkf("-E -w", c)}
+    assert_equal(expected, doc.split_to_char)
+  end
+  def test_utf8_count_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\r\n"), "UTF-8")
+    expected = 7
+    assert_equal(expected, doc.count_char)
+  end
+  def test_utf8_count_latin_graph_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\r\n"), "UTF-8")
+    expected = 2
+    assert_equal(expected, doc.count_latin_graph_char)
+  end
+  def test_utf8_count_ja_graph_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\r\n"), "UTF-8")
+    expected = 3
+    assert_equal(expected, doc.count_ja_graph_char)
+  end
+  def test_utf8_count_graph_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�a b\r\n"), "UTF-8")
+    expected = 5
+    assert_equal(expected, doc.count_graph_char)
+  end
+  def test_utf8_count_latin_blank_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�\ta b\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_blank_char)
+  end
+  def test_utf8_count_ja_blank_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���\ta b\r\n"))
+    expected = 1
+    assert_equal(expected, doc.count_ja_blank_char)
+  end
+  def test_utf8_count_blank_char()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���\ta b\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_blank_char)
+  end
+  def test_utf8_count_word()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���a b --\r\n"))
+    expected = 7 # "--" and "\r\n" are counted as word here (though not "valid")
+    assert_equal(expected, doc.count_word)
+  end
+  def test_utf8_count_ja_word()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���a b --\r\n"))
+    expected = 3
+    assert_equal(expected, doc.count_ja_word)
+  end
+  def test_utf8_count_latin_valid_word()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���a b --\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_latin_valid_word)
+  end
+  def test_utf8_count_ja_valid_word()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���a b --\r\n"))
+    expected = 2
+    assert_equal(expected, doc.count_ja_valid_word)
+  end
+  def test_utf8_count_valid_word()
+    doc = Document.new(NKF.nkf("-E -w", "���ܡ���a b --\r\n"))
+    expected = 4
+    assert_equal(expected, doc.count_valid_word)
+  end
+  def test_utf8_count_line()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 6
+    assert_equal(expected, doc.count_line)
+  end
+  def test_utf8_count_graph_line()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 3
+    assert_equal(expected, doc.count_graph_line)
+  end
+  def test_utf8_count_empty_line()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 1
+    assert_equal(expected, doc.count_empty_line)
+  end
+  def test_utf8_count_blank_line()
+    doc = Document.new(NKF.nkf("-E -w", "���ܸ�\r\n��\r\n \r\n\r\nfoo\r\nbar"))
+    expected = 2
+    assert_equal(expected, doc.count_blank_line)
+  end
+
+
+
+
+  def teardown()
+    #
+  end
+
+end