docdiff 0.5.0

data/langfilter.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/ruby
+# language filter
+# usage: langfilter.rb --en <infile >outfile
+
+def ruby_m17n?
+  return true if "".respond_to? :encoding
+end
+
+lang_to_include = ARGV.shift.gsub(/-+/, "")
+lang_to_exclude = {"en"=>"ja", "ja"=>"en"}[lang_to_include]
+re = /<([a-z]+) +(?:(?:lang|title)="#{lang_to_exclude}").*?>.*?<\/\1>[\r\n]?/m
+
+ARGF.set_encoding("UTF-8") if ruby_m17n?
+ARGF.read.gsub(re, "").display

data/lib/doc_diff.rb

@@ -0,0 +1,170 @@
+# DocDiff: word/character-oriented text comparison utility
+# Copyright (C) 2002-2011 Hisashi MORITA
+# Requirements: Ruby (>= 1.8)
+class DocDiff
+
+  AppVersion = Docdiff::VERSION
+  Author = "Copyright (C) 2002-2011 Hisashi MORITA.\n" +
+           "diff library originates from Ruby/CVS by TANAKA Akira.\n"
+  License = "This software is licensed under so-called modified BSD license.\n" +
+            "See the document for detail.\n"
+  SystemConfigFileName = File.join(File::Separator, "etc", "docdiff", "docdiff.conf")
+  UserConfigFileName = File.join(ENV['HOME'], "etc", "docdiff", "docdiff.conf")
+  AltUserConfigFileName = File.join(ENV['HOME'], ".docdiff", "docdiff.conf")
+
+  def initialize()
+    @config = {}
+  end
+  attr_accessor :config
+
+  def DocDiff.parse_config_file_content(content)
+    result = {}
+    return result if content.size <= 0
+    lines = content.dup.split(/\r\n|\r|\n/).compact
+    lines.collect!{|line| line.sub(/#.*$/, '')}
+    lines.collect!{|line| line.strip}
+    lines.delete_if{|line| line == ""}
+    lines.each{|line|
+      raise 'line does not include " = ".' unless /[\s]+=[\s]+/.match line
+      name_src, value_src = line.split(/[\s]+=[\s]+/)
+      raise "Invalid name: #{name_src.inspect}" if (/\s/.match name_src)
+      raise "Invalid value: #{value_src.inspect}" unless value_src.kind_of?(String)
+      name  = name_src.intern
+      value = value_src
+      value = true if ['on','yes','true'].include? value_src.downcase
+      value = false if ['off','no','false'].include? value_src.downcase
+      value = value_src.to_i if /^[0-9]+$/.match value_src
+      result[name] = value
+    }
+    result
+  end
+
+  def compare_by_line(doc1, doc2)
+    Difference.new(doc1.split_to_line, doc2.split_to_line)
+  end
+
+  def compare_by_line_word(doc1, doc2)
+    lines = compare_by_line(doc1, doc2)
+    words = Difference.new
+    lines.each{|line|
+      if line.first == :change_elt
+        before_change = Document.new(line[1].join,
+                                     doc1.encoding, doc1.eol)
+        after_change  = Document.new(line[2].join,
+                                     doc2.encoding, doc2.eol)
+        Difference.new(before_change.split_to_word,
+                       after_change.split_to_word).each{|word|
+          words << word
+        }
+      else  # :common_elt_elt, :del_elt, or :add_elt
+        words << line
+      end
+    }
+    words
+  end
+
+  # i know this implementation of recursion is so lame...
+  def compare_by_line_word_char(doc1, doc2)
+    lines = compare_by_line(doc1, doc2)
+    lines_and_words = Difference.new
+    lines.each{|line|
+      if line.first == :change_elt
+        before_change = Document.new(line[1].join,
+                                     doc1.encoding, doc1.eol)
+        after_change  = Document.new(line[2].join,
+                                     doc2.encoding, doc2.eol)
+        Difference.new(before_change.split_to_word,
+                       after_change.split_to_word).each{|word|
+          lines_and_words << word
+        }
+      else  # :common_elt_elt, :del_elt, or :add_elt
+        lines_and_words << line
+      end
+    }
+    lines_words_and_chars = Difference.new
+    lines_and_words.each{|line_or_word|
+      if line_or_word.first == :change_elt
+        before_change = Document.new(line_or_word[1].join, doc1.encoding, doc1.eol)
+        after_change  = Document.new(line_or_word[2].join, doc2.encoding, doc2.eol)
+        Difference.new(before_change.split_to_char, after_change.split_to_char).each{|char|
+          lines_words_and_chars << char
+        }
+      else  # :common_elt_elt, :del_elt, or :add_elt
+        lines_words_and_chars << line_or_word
+      end
+    }
+    lines_words_and_chars
+  end
+
+  def run(doc1, doc2, option)
+    raise "option is nil" if option.nil?
+    raise "option[:resolution] is nil" if option[:resolution].nil?
+    raise "option[:format] is nil" if option[:format].nil?
+    case
+    when doc1.class == Document && doc2.class == Document # OK
+    when doc1.encoding != nil && doc2.encoding != nil     # OK
+    when doc1.encoding == doc2.encoding && doc1.eol == doc2.eol # OK
+    else
+      raise("Error!  Blame the author (doc1: #{doc1.encoding}, #{doc1.eol}, doc2: #{doc2.encoding}, #{doc2.eol}).")
+    end
+
+    case option[:resolution]
+    when "line"; then difference = compare_by_line(doc1, doc2)
+    when "word"; then difference = compare_by_line_word(doc1, doc2)
+    when "char"; then difference = compare_by_line_word_char(doc1, doc2)
+    else
+      raise "Unsupported resolution: #{option[:resolution].inspect}"
+    end
+    view = View.new(difference, doc1.encoding, doc1.eol)
+    user_tags = {:start_common        => (@config[:tag_common_start] ||= ''),
+                 :end_common          => (@config[:tag_common_end] ||= ''),
+                 :start_del           => (@config[:tag_del_start] ||= ''),
+                 :end_del             => (@config[:tag_del_end] ||= ''),
+                 :start_add           => (@config[:tag_add_start] ||= ''),
+                 :end_add             => (@config[:tag_add_end] ||= ''),
+                 :start_before_change => (@config[:tag_change_before_start] ||= ''),
+                 :end_before_change   => (@config[:tag_change_before_end] ||= ''),
+                 :start_after_change  => (@config[:tag_change_after_start] ||= ''),
+                 :end_after_change    => (@config[:tag_change_after_end] ||= '')}
+    case option[:digest]
+    when true
+      case option[:format]
+      when "tty";      then result = view.to_tty_digest(option)
+      when "html";     then result = view.to_html_digest(option)
+      when "manued";   then result = view.to_manued_digest(option)
+      when "wdiff";    then result = view.to_wdiff_digest(option)
+      when "stat";     then result = view.to_stat(option)
+      when "user";     then result = view.to_user_digest(user_tags)
+      else
+        raise "Unsupported output format: #{option[:format].inspect}."
+      end
+    when false
+      case option[:format]
+      when "tty";      then result = view.to_tty(option)
+      when "html";     then result = view.to_html(option)
+      when "manued";   then result = view.to_manued(option)
+      when "wdiff";    then result = view.to_wdiff(option)
+      when "stat";     then result = view.to_stat(option)
+      when "user";     then result = view.to_user(user_tags)
+      else
+        raise "Unsupported output format: #{option[:format].inspect}."
+      end
+    end
+    result.join
+  end
+
+  def process_config_file(filename)
+    file_content = nil
+    begin
+      File.open(filename, "r"){|f| file_content = f.read}
+    rescue Errno::ENOENT
+      message = "config file not found so not read."
+    ensure
+      if file_content != nil
+        self.config.update(DocDiff.parse_config_file_content(file_content))
+      end
+    end
+    message
+  end
+
+end  # class DocDiff

data/lib/docdiff.rb

@@ -0,0 +1,7 @@
+# DocDiff: word/character-oriented text comparison utility
+# Copyright (C) 2002-2011 Hisashi MORITA
+# Requirements: Ruby (>= 1.8)
+require 'docdiff/version'
+require 'doc_diff'
+module Docdiff
+end

data/lib/docdiff/charstring.rb

@@ -0,0 +1,579 @@
+#!/usr/bin/ruby
+# Character String module.
+# To use, include to String, or extend String.
+# 2003- Hisashi MORITA
+
+module CharString
+
+  Encodings = {}
+  EOLChars = {}  # End-of-line characters, such as CR, LF, CRLF.
+
+  def initialize(string)
+=begin unnecessary
+#    @encoding = CharString.guess_encoding(string)
+#    @eol     = CharString.guess_eol(string)
+=end unnecessary
+    super
+  end
+
+  def eol()
+    @eol
+#     if @eol
+#       @eol
+#     else
+#       @eol = CharString.guess_eol(self)
+#       # raise "eol is not set.\n"
+#     end
+  end
+
+  def eol=(e)
+    @eol = e
+    extend EOLChars[@eol]
+  end
+
+  def eol_char()
+    if @eol_char
+      @eol_char
+    else
+      nil
+#       extend EOLChars[eol]
+#       eol_char
+    end
+  end
+
+  def debug()
+    case
+    when @encoding  == nil
+      raise "@encoding is nil."
+    when Encodings[@encoding] == nil
+      raise "Encodings[@encoding(=#{@encoding})] is nil."
+    when Encodings[@encoding].class != Module
+      raise "Encodings[@encoding].class(=#{Encodings[@encoding].class}) is not a module."
+    when @eol == nil
+      raise "@eol is nil."
+    when EOLChars[@eol] == nil
+      raise "EOLChars[@eol(=#{@eol})] is nil."
+    else
+      # should I do some alert?
+    end
+    ["id: #{self.id}, class: #{self.class}, self: #{self}, ", 
+     "module: #{Encodings[@encoding]}, #{EOLChars[@eol]}"].join
+  end
+
+  def CharString.register_encoding(mod)
+    Encodings[mod::Encoding] = mod
+  end
+
+  def CharString.register_eol(mod)
+    EOLChars[mod::EOL] = mod
+  end
+
+  def CharString.guess_eol(string)
+    # returns 'CR', 'LF', 'CRLF', 'UNKNOWN'(binary), 
+    # 'NONE'(1-line), or nil
+    return nil if string == nil  #=> nil (argument missing)
+    eol_counts = {'CR'   => string.scan(/(\r)(?!\n)/o).size,
+                  'LF'   => string.scan(/(?:\A|[^\r])(\n)/o).size,
+                  'CRLF' => string.scan(/(\r\n)/o).size}
+    eol_counts.delete_if{|eol, count| count == 0}  # Remove missing EOL
+    eols = eol_counts.keys
+    eol_variety = eols.size  # numbers of flavors found
+    if eol_variety == 1          # Only one type of EOL found
+      return eols[0]         #=> 'CR', 'LF', or 'CRLF'
+    elsif eol_variety == 0       # No EOL found
+      return 'NONE'              #=> 'NONE' (might be 1-line file)
+    else                         # Multiple types of EOL found
+      return 'UNKNOWN'           #=> 'UNKNOWN' (might be binary data)
+    end
+  end
+
+  def CharString.ruby_m17n?
+    "".respond_to?(:force_encoding)
+  end
+
+  # Note that some languages (like Japanese) do not have 'word' or 'phrase', 
+  # thus some of the following methods are not 'linguistically correct'.
+
+  def count_byte()
+    split_to_byte().size
+  end
+
+  def count_char()  # eol = 1 char
+    split_to_char().size
+  end
+
+  def count_graph_char()
+    count_latin_graph_char() + count_ja_graph_char()
+  end
+
+  def count_blank_char()
+    count_latin_blank_char() + count_ja_blank_char()
+  end
+
+  def count_word()
+    split_to_word().size
+  end
+
+  def count_valid_word()
+    count_latin_valid_word() + count_ja_valid_word()
+  end
+
+  def count_line()  # this is common to all encodings.
+    split_to_line.size
+  end
+
+  def count_empty_line()
+    split_to_line.collect{|line|
+      line if /^(?:#{eol_char})|^$/m.match line
+    }.compact.size
+  end
+
+if ruby_m17n?
+  # for Ruby-1.9
+  def encoding()
+    String.new(self).encoding.to_s
+  end
+
+  def encoding=(cs)
+    force_encoding(cs) if self
+  end
+
+  def CharString.guess_encoding(string)
+    if string
+      string.encoding.to_s
+    else
+      nil
+    end
+  end
+
+  def split_to_byte()
+    encode("ASCII-8BIT").scan(/./nm)
+  end
+
+  def split_to_char()
+    if eol_char  # sometimes string has no end-of-line char
+      encode('UTF-8').scan(Regexp.new("(?:#{eol_char})|(?:.)", 
+                      Regexp::MULTILINE)
+      ).map{|e| e.encode(self.encoding)}
+    else                  # it seems that no EOL module was extended...
+      encode('UTF-8').scan(Regexp.new("(?:.)", 
+                      Regexp::MULTILINE)
+      ).map{|e| e.encode(self.encoding)}
+    end
+  end
+
+  def count_latin_graph_char()
+    encode('UTF-8').scan(Regexp.new("[#{Encodings['UTF-8']::GRAPH}]", 
+                    Regexp::MULTILINE)
+    ).size
+  end
+
+  def count_ja_graph_char()
+    encode('UTF-8').scan(Regexp.new("[#{Encodings['UTF-8']::JA_GRAPH}]", 
+                    Regexp::MULTILINE)
+    ).size
+  end
+
+  def count_latin_blank_char()
+    encode('UTF-8').scan(Regexp.new("[#{Encodings['UTF-8']::BLANK}]", 
+                    Regexp::MULTILINE)
+    ).size
+  end
+
+  def count_ja_blank_char()
+    encode('UTF-8').scan(Regexp.new("[#{Encodings['UTF-8']::JA_BLANK}]", 
+                    Regexp::MULTILINE)
+    ).size
+  end
+
+  def split_to_word()
+    encode('UTF-8').scan(Regexp.new(Encodings['UTF-8']::WORD_REGEXP_SRC, 
+                    Regexp::MULTILINE)
+    ).map{|e| e.encode(self.encoding)}
+  end
+
+  def count_latin_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings['UTF-8']::PRINT}]", 
+                         Regexp::MULTILINE).match word.encode('UTF-8')
+    }.compact.size
+  end
+
+  def count_ja_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings['UTF-8']::JA_PRINT}]", 
+                         Regexp::MULTILINE).match word.encode('UTF-8')
+    }.compact.size
+  end
+
+  def count_latin_valid_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings['UTF-8']::ALNUM}]", 
+                         Regexp::MULTILINE).match word.encode('UTF-8')
+    }.compact.size
+  end
+
+  def count_ja_valid_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings['UTF-8']::JA_GRAPH}]", 
+                         Regexp::MULTILINE).match word.encode('UTF-8')
+    }.compact.size
+  end
+
+  def split_to_line()
+    raise "EOLChars[eol] is #{EOLChars[eol].inspect}: eol not specified or auto-detection failed." unless EOLChars[eol]
+    if defined? eol_char
+      encode('UTF-8').scan(Regexp.new(".*?#{eol_char}|.+", 
+                      Regexp::MULTILINE)
+      ).map{|e| e.encode(self.encoding)}
+    else
+      encode('UTF-8').scan(Regexp.new(".+", 
+                      Regexp::MULTILINE)
+      ).map{|e| e.encode(self.encoding)}
+    end
+  end
+
+  def count_graph_line()
+    split_to_line.collect{|line|
+      line if Regexp.new("[#{Encodings['UTF-8']::GRAPH}" + 
+                         "#{Encodings['UTF-8']::JA_GRAPH}]", 
+                         Regexp::MULTILINE).match line.encode('UTF-8')
+    }.compact.size
+  end
+
+  def count_blank_line()
+    split_to_line.collect{|line|
+      line if Regexp.new("^[#{Encodings['UTF-8']::BLANK}" + 
+                         "#{Encodings['UTF-8']::JA_BLANK}]+(?:#{eol_char})?", 
+                         Regexp::MULTILINE).match line.encode('UTF-8')
+    }.compact.size
+  end
+
+  # load encoding modules
+  require 'docdiff/encoding/en_ascii'
+  require 'docdiff/encoding/ja_eucjp'
+  require 'docdiff/encoding/ja_sjis'
+  require 'docdiff/encoding/ja_utf8'
+else
+  # for Ruby-1.8
+  require 'iconv'
+
+  def encoding()
+    @encoding
+#     if @encoding
+#       @encoding
+#     else
+#       @encoding = CharString.guess_encoding(self)
+#       # raise "encoding is not set.\n"
+#     end
+  end
+
+  def encoding=(cs)
+    @encoding = cs
+    extend Encodings[@encoding]  # ; p "Hey, I extended #{Encodings[@encoding]}!"
+  end
+
+  # returns nil, 'US-ASCII', 'JIS', 'EUC-JP', 'Shift_JIS', 'UTF-8', or 'UNKNOWN'
+  def CharString.guess_encoding(string)
+    return nil if string == nil
+    result_using_pureruby = CharString.guess_encoding_using_pureruby(string)
+    result_using_iconv    = CharString.guess_encoding_using_iconv(string)
+    if result_using_pureruby == result_using_iconv
+      result_using_pureruby
+    else
+      "UNKNOWN"
+    end
+  end
+
+  # returns nil, 'US-ASCII', 'JIS', 'EUC-JP', 'Shift_JIS', 'UTF-8', or 'UNKNOWN'
+  def CharString.guess_encoding_using_pureruby(string)
+    return nil if string == nil
+
+    ascii_pat = '[\x00-\x7f]'
+    jis_pat   = ['(?:(?:\x1b\x28\x42)', 
+                 '|(?:\x1b\x28\x4a)', 
+                 '|(?:\x1b\x28\x49)', 
+                 '|(?:\x1b\x24\x40)', 
+                 '|(?:\x1b\x24\x42)', 
+                 '|(?:\x1b\x24\x44))'].join
+    eucjp_pat = ['(?:(?:[\x00-\x1f\x7f])', 
+                 '|(?:[\x20-\x7e])', 
+                 '|(?:\x8e[\xa1-\xdf])', 
+                 '|(?:[\xa1-\xfe][\xa1-\xfe])', 
+                 '|(?:\x8f[\xa1-\xfe][\xa1-\xfe]))'].join
+    sjis_pat  = ['(?:(?:[\x00-\x1f\x7f])', 
+                 '|(?:[\x20-\x7e])', 
+                 '|(?:[\xa1-\xdf])', 
+                 '|(?:[\x81-\x9f][\x40-\x7e])', 
+                 '|(?:[\xe0-\xef][\x80-\xfc]))'].join
+    utf8_pat  = ['(?:(?:[\x00-\x7f])', 
+                 '|(?:[\xc0-\xdf][\x80-\xbf])', 
+                 '|(?:[\xe0-\xef][\x80-\xbf][\x80-\xbf])', 
+                 '|(?:[\xf0-\xf7][\x80-\xbf][\x80-\xbf][\x80-\xbf]))'].join
+
+    ascii_match_length = string.scan(/#{ascii_pat}/on).join.length
+    jis_escseq_count   = string.scan(/#{jis_pat}/on).size
+    eucjp_match_length = string.scan(/#{eucjp_pat}/no).join.length
+    sjis_match_length  = string.scan(/#{sjis_pat}/no).join.length
+    utf8_match_length  = string.scan(/#{utf8_pat}/no).join.length
+
+    case
+    when 0 < jis_escseq_count                 # JIS escape sequense found
+      guessed_encoding = 'JIS'
+    when ascii_match_length == string.length  # every char is ASCII (but not JIS)
+      guessed_encoding = 'US-ASCII'
+    else
+      case
+      when eucjp_match_length < (string.length / 2) && 
+           sjis_match_length  < (string.length / 2) && 
+           utf8_match_length  < (string.length / 2)
+        guessed_encoding = 'UNKNOWN'  # either encoding did not match long enough
+      when (eucjp_match_length < utf8_match_length) && 
+           (sjis_match_length < utf8_match_length)
+        guessed_encoding = 'UTF-8'
+      when (eucjp_match_length < sjis_match_length) && 
+           (utf8_match_length < sjis_match_length)
+        guessed_encoding = 'Shift_JIS'
+      when (sjis_match_length < eucjp_match_length) && 
+           (utf8_match_length < eucjp_match_length)
+        guessed_encoding = 'EUC-JP'
+      else
+        guessed_encoding = 'UNKNOWN'  # cannot guess at all
+      end
+    end
+    return guessed_encoding
+  end
+
+  def CharString.guess_encoding_using_iconv(string)
+    valid_as_utf8   = CharString.valid_as("utf-8", string)
+    valid_as_sjis   = CharString.valid_as("cp932", string) # not sjis, but cp932
+    valid_as_jis    = CharString.valid_as("iso-2022-jp", string)
+    valid_as_eucjp  = CharString.valid_as("eucjp", string)
+    valid_as_ascii  = CharString.valid_as("ascii", string)
+    invalid_as_utf8   = CharString.invalid_as("utf-8", string)
+    invalid_as_sjis   = CharString.invalid_as("cp932", string) # not sjis, but cp932
+    invalid_as_jis    = CharString.invalid_as("iso-2022-jp", string)
+    invalid_as_eucjp  = CharString.invalid_as("eucjp", string)
+    invalid_as_ascii  = CharString.invalid_as("ascii", string)
+    case
+    when string == nil
+      nil
+    when valid_as_ascii
+      "US-ASCII"
+    when valid_as_jis  # Iconv sometimes recognizes JIS for ASCII, ignoring JIS escape sequence.
+      "JIS"
+    when valid_as_eucjp
+      "EUC-JP"
+    when valid_as_sjis && invalid_as_utf8 && invalid_as_eucjp && invalid_as_jis
+      "Shift_JIS"
+    when valid_as_utf8 && invalid_as_sjis && invalid_as_eucjp && invalid_as_jis
+      "UTF-8"
+    else
+      "UNKNOWN"
+    end
+  end
+
+  def CharString.valid_as(encoding_name, string)
+    begin
+      Iconv.iconv(encoding_name, encoding_name, string)
+    rescue Iconv::IllegalSequence, Iconv::InvalidCharacter, Iconv::OutOfRange
+      return false
+    else
+      return true
+    end
+  end
+
+  def CharString.invalid_as(encoding_name, string)
+    if CharString.valid_as(encoding_name, string)
+      false
+    else
+      true
+    end
+  end
+
+  def split_to_byte()
+    scan(/./nm)
+  end
+
+  def split_to_char()
+    raise "Encodings[encoding] is #{Encodings[encoding].inspect}: encoding not specified or auto-detection failed." unless Encodings[encoding]
+    # raise "EOLChars[eol] is #{EOLChars[eol].inspect}: eol not specified or auto-detection failed." unless EOLChars[eol]
+    if eol_char  # sometimes string has no end-of-line char
+      scan(Regexp.new("(?:#{eol_char})|(?:.)", 
+                      Regexp::MULTILINE, 
+                      encoding.sub(/ASCII/i, 'none'))
+      )
+    else                  # it seems that no EOL module was extended...
+      scan(Regexp.new("(?:.)", 
+                      Regexp::MULTILINE, 
+                      encoding.sub(/ASCII/i, 'none'))
+      )
+    end
+  end
+
+  def count_latin_graph_char()
+    raise "Encodings[encoding] is #{Encodings[encoding].inspect}: encoding not specified or auto-detection failed." unless Encodings[encoding]
+    # raise "EOLChars[eol] is #{EOLChars[eol].inspect}: eol not specified or auto-detection failed." unless EOLChars[eol]
+    scan(Regexp.new("[#{Encodings[encoding]::GRAPH}]", 
+                    Regexp::MULTILINE, 
+                    encoding.sub(/ASCII/i, 'none'))
+    ).size
+  end
+
+  def count_ja_graph_char()
+    raise "Encodings[encoding] is #{Encodings[encoding].inspect}: encoding not specified or auto-detection failed." unless Encodings[encoding]
+    # raise "EOLChars[eol] is #{EOLChars[eol].inspect}: eol not specified or auto-detection failed." unless EOLChars[eol]
+    scan(Regexp.new("[#{Encodings[encoding]::JA_GRAPH}]", 
+                    Regexp::MULTILINE, 
+                    encoding.sub(/ASCII/i, 'none'))
+    ).size
+  end
+
+  def count_latin_blank_char()
+    scan(Regexp.new("[#{Encodings[encoding]::BLANK}]", 
+                    Regexp::MULTILINE, 
+                    encoding.sub(/ASCII/i, 'none'))
+    ).size
+  end
+
+  def count_ja_blank_char()
+    scan(Regexp.new("[#{Encodings[encoding]::JA_BLANK}]", 
+                    Regexp::MULTILINE, 
+                    encoding.sub(/ASCII/i, 'none'))
+    ).size
+  end
+
+  def split_to_word()
+    raise "Encodings[encoding] is #{Encodings[encoding].inspect}: encoding not specified or auto-detection failed." unless Encodings[encoding]
+    # raise "EOLChars[eol] is #{EOLChars[eol].inspect}: eol not specified or auto-detection failed." unless EOLChars[eol]
+    scan(Regexp.new(Encodings[encoding]::WORD_REGEXP_SRC, 
+                    Regexp::MULTILINE, 
+                    encoding.sub(/ASCII/i, 'none'))
+    )
+  end
+
+  def count_latin_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings[encoding]::PRINT}]", 
+                         Regexp::MULTILINE, 
+                         encoding.sub(/ASCII/i, 'none')).match word
+    }.compact.size
+  end
+
+  def count_ja_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings[encoding]::JA_PRINT}]", 
+                         Regexp::MULTILINE, 
+                         encoding.sub(/ASCII/i, 'none')).match word
+    }.compact.size
+  end
+
+  def count_latin_valid_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings[encoding]::ALNUM}]", 
+                         Regexp::MULTILINE, 
+                         encoding.sub(/ASCII/i, 'none')).match word
+    }.compact.size
+  end
+
+  def count_ja_valid_word()
+    split_to_word.collect{|word|
+      word if Regexp.new("[#{Encodings[encoding]::JA_GRAPH}]", 
+                         Regexp::MULTILINE, 
+                         encoding.sub(/ASCII/i, 'none')).match word
+    }.compact.size
+  end
+
+  def split_to_line()
+#     scan(Regexp.new(".*?#{eol_char}|.+", 
+#                     Regexp::MULTILINE, 
+#                     encoding.sub(/ASCII/i, 'none'))
+#     )
+    raise "Encodings[encoding] is #{Encodings[encoding].inspect}: encoding not specified or auto-detection failed." unless Encodings[encoding]
+    raise "EOLChars[eol] is #{EOLChars[eol].inspect}: eol not specified or auto-detection failed." unless EOLChars[eol]
+    if defined? eol_char
+      scan(Regexp.new(".*?#{eol_char}|.+", 
+                      Regexp::MULTILINE, 
+                      encoding.sub(/ASCII/i, 'none'))
+      )
+    else
+      scan(Regexp.new(".+", 
+                      Regexp::MULTILINE, 
+                      encoding.sub(/ASCII/i, 'none'))
+      )
+    end
+  end
+
+  def count_graph_line()
+    split_to_line.collect{|line|
+      line if Regexp.new("[#{Encodings[encoding]::GRAPH}" + 
+                         "#{Encodings[encoding]::JA_GRAPH}]", 
+                         Regexp::MULTILINE, 
+                         encoding.sub(/ASCII/, 'none')).match line
+    }.compact.size
+  end
+
+  def count_blank_line()
+    split_to_line.collect{|line|
+      line if Regexp.new("^[#{Encodings[encoding]::BLANK}" + 
+                         "#{Encodings[encoding]::JA_BLANK}]+(?:#{eol_char})?", 
+                         Regexp::MULTILINE, 
+                         encoding.sub(/ASCII/, 'none')).match line
+    }.compact.size
+  end
+
+  # load encoding modules
+  require 'docdiff/encoding/en_ascii'
+  require 'docdiff/encoding/ja_eucjp'
+  require 'docdiff/encoding/ja_sjis'
+  require 'docdiff/encoding/ja_utf8'
+end # end ruby_m17n?
+  alias to_bytes split_to_byte
+  alias to_chars split_to_char
+  alias to_words split_to_word
+  alias to_lines split_to_line
+
+  module CR
+    EOL = 'CR'
+
+    def eol_char()
+      "\r"
+    end
+
+    CharString.register_eol(self)
+  end
+
+  module LF
+    EOL = 'LF'
+
+    def eol_char()
+      "\n"
+    end
+
+    CharString.register_eol(self)
+  end
+
+  module CRLF
+    EOL = 'CRLF'
+
+    def eol_char()
+      "\r\n"
+    end
+
+    CharString.register_eol(self)
+  end
+
+  module NoEOL
+    EOL = 'NONE'
+    def eol_char()
+      nil
+    end
+
+    CharString.register_eol(self)
+  end
+
+end  # module CharString
+
+# class String
+#   include CharString
+# end