docdiff 0.5.0

data/lib/docdiff/difference.rb

@@ -0,0 +1,92 @@
+# Difference class for DocDiff
+# 2003-03-24 .. 
+# Hisashi MORITA
+
+require 'docdiff/diff'
+
+class Difference < Array
+
+#  @resolution = nil # char, word, phrase, sentence, line, paragraph..
+#  @codeset = ''
+#  @eol_char = "\n"
+#  @source = 'source'
+#  @target = 'target'
+#  attr_accessor :resolution, :codeset, :eol_char, :source, :target
+
+  def initialize(array1 = nil, array2 = nil)
+    if (array1 == nil) && (array2 == nil)
+      return []
+    end
+    diff = Diff.new(array1, array2)
+    @raw_list = []
+    diff.ses.each{|block|  # Diff::EditScript does not have each_with_index()
+      @raw_list << block
+    }
+    combine_del_add_to_change!()
+  end
+
+  def combine_del_add_to_change!()
+
+    @raw_list.each_with_index{|block, i|
+      case block.first
+      when :common_elt_elt
+        if i == 0                       # first block
+          self << block
+        else                            # in-between or the last block
+          if @raw_list[i - 1].first == :del_elt  # previous block was del
+            self << @raw_list[i - 1]
+            self << block
+          else                                   # previous block was add
+            self << block
+          end
+        end
+      when :del_elt
+        if i == (@raw_list.size - 1)    # last block
+          self << block
+        else                            # first block or in-between
+          # do nothing, let the next block to decide what to do
+        end
+      when :add_elt
+        if i == 0                       # first block
+          self << block
+        else                            # in-between or the last block
+          if @raw_list[i - 1].first == :del_elt  # previous block was del
+            deleted = @raw_list[i - 1][1]
+            added   = @raw_list[i][2]
+            self << [:change_elt, deleted, added]
+          else                                   # previous block was common
+            self << block
+          end
+        end
+      else
+        raise "the first element of the block #{i} is invalid: (#{block.first})\n"
+      end
+    }
+  end
+  attr_accessor :raw_list
+
+  def former_only()
+    elms = self.dup.delete_if{|e| e[0] == :add_elt}
+    elms.collect!{|e|
+      if e[0] == :change_elt
+        [e[0], e[1], nil]
+      else
+        e
+      end
+    }
+    return elms
+  end
+
+  def latter_only()
+    elms = self.dup.delete_if{|e| e[0] == :del_elt}
+    elms.collect!{|e|
+      if e[0] == :change_elt
+        [e[0], nil, e[2]]
+      else
+        e
+      end
+    }
+    return elms
+  end
+
+end  # class Difference

data/lib/docdiff/document.rb

@@ -0,0 +1,127 @@
+# Document class, a part of DocDiff
+# 2004-01-14.. Hisashi MORITA
+
+require 'docdiff/charstring'
+
+class EncodingDetectionFailure < Exception
+end
+class EOLDetectionFailure < Exception
+end
+
+class Document
+
+  def initialize(str, enc = nil, e = nil)
+    @body = str
+    @body.extend CharString
+    if enc
+      @body.encoding = enc
+    elsif !@body.encoding
+      guessed_encoding = CharString.guess_encoding(str)
+      if guessed_encoding == "UNKNOWN"
+        raise EncodingDetectionFailure, "encoding not specified, and auto detection failed."
+        # @body.encoding = 'ASCII' # default to ASCII <= BAD!
+      else
+	@body.encoding = guessed_encoding
+      end
+    end
+    if e
+      @body.eol = e
+    else
+      guessed_eol = CharString.guess_eol(str)
+      if guessed_eol == "UNKNOWN"
+        raise EOLDetectionFailure, "eol not specified, and auto detection failed."
+        # @body.eol = 'LF' # default to LF
+      else
+        @body.eol = guessed_eol
+      end
+    end
+  end
+  def encoding()
+    @body.encoding
+  end
+  def encoding=(cs)
+    @body.encoding = cs
+  end
+  def eol()
+    @body.eol
+  end
+  def eol=(eolstr)
+    @body.eol = eolstr
+  end
+
+  def split_to_line()
+    @body.split_to_line
+  end
+  def split_to_word()
+    @body.split_to_word
+  end
+  def split_to_char()
+    @body.split_to_char
+  end
+  def split_to_byte()
+    @body.split_to_byte
+  end
+
+  def count_line()
+    @body.count_line
+  end
+  def count_blank_line()
+    @body.count_blank_line
+  end
+  def count_empty_line()
+    @body.count_empty_line
+  end
+  def count_graph_line()
+    @body.count_graph_line
+  end
+
+  def count_word()
+    @body.count_word
+  end
+  def count_latin_word()
+    @body.count_latin_word
+  end
+  def count_ja_word()
+    @body.count_ja_word
+  end
+  def count_valid_word()
+    @body.count_valid_word
+  end
+  def count_latin_valid_word()
+    @body.count_latin_valid_word
+  end
+  def count_ja_valid_word()
+    @body.count_ja_valid_word
+  end
+
+  def count_char()
+    @body.count_char
+  end
+  def count_blank_char()
+    @body.count_blank_char
+  end
+  def count_graph_char()
+    @body.count_graph_char
+  end
+  def count_latin_blank_char()
+    @body.count_latin_blank_char
+  end
+  def count_latin_graph_char()
+    @body.count_latin_graph_char
+  end
+  def count_ja_blank_char()
+    @body.count_ja_blank_char
+  end
+  def count_ja_graph_char()
+    @body.count_ja_graph_char
+  end
+
+  def count_byte()
+    @body.count_byte
+  end
+
+  def eol_char()
+    @body.eol_char
+  end
+
+end  # class Document

data/lib/docdiff/encoding/en_ascii.rb

@@ -0,0 +1,97 @@
+# English ASCII encoding module for CharString
+# 2003- Hisashi MORITA
+
+module CharString
+  module ASCII
+
+    Encoding = "US-ASCII"
+
+    CNTRL =     "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" \
+                "\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13" \
+                "\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d" \
+                "\x1e\x1f\x7f"
+    SPACE =     "\x09\x0a\x0b\x0c\x0d\x20"
+    BLANK =     "\x09\x20"
+    DIGIT =     "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39"
+    ALPHA =     "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a" \
+                "\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" \
+                "\x55\x56\x57\x58\x59\x5a\x61\x62\x63\x64" \
+                "\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e" \
+                "\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78" \
+                "\x79\x7a"
+    ALNUM =     "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" \
+                "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a" \
+                "\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" \
+                "\x55\x56\x57\x58\x59\x5a\x61\x62\x63\x64" \
+                "\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e" \
+                "\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78" \
+                "\x79\x7a"
+    PUNCT =     "\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a" \
+                "\x2b\x2c\x2d\x2e\x2f\x3a\x3b\x3c\x3d\x3e" \
+                "\x3f\x40\x5b\x5c\x5d\x5e\x5f\x60\x7b\x7c" \
+                "\x7d\x7e"
+    LOWER =     "\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a" \
+                "\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74" \
+                "\x75\x76\x77\x78\x79\x7a"
+    UPPER =     "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a" \
+                "\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" \
+                "\x55\x56\x57\x58\x59\x5a"
+    PRINT =     "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29" \
+                "\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33" \
+                "\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d" \
+                "\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47" \
+                "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51" \
+                "\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b" \
+                "\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" \
+                "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" \
+                "\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79" \
+                "\x7a\x7b\x7c\x7d\x7e"
+    GRAPH =     "\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a" \
+                "\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34" \
+                "\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e" \
+                "\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48" \
+                "\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52" \
+                "\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c" \
+                "\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65\x66" \
+                "\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" \
+                "\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a" \
+                "\x7b\x7c\x7d\x7e"
+    XDIGIT =    "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" \
+                "\x41\x42\x43\x44\x45\x46\x61\x62\x63\x64" \
+                "\x65\x66"
+
+    JA_BLANK =  "" # kludge...
+    JA_GRAPH =  "" # kludge...
+
+    PUNCT.replace(Regexp.quote(PUNCT)) # kludge to avoid warning "character class has `[' without escape"
+    PRINT.replace(Regexp.quote(PRINT)) # kludge to avoid warning "character class has `[' without escape"
+    GRAPH.replace(Regexp.quote(GRAPH)) # kludge to avoid warning "character class has `[' without escape"
+
+    WORD_REGEXP_SRC = ["(?:[#{GRAPH}]+[#{BLANK}]?)", 
+                       "|(?:[#{SPACE}]+)", 
+                       "|(?:.+?)"].join
+
+    # override default method, as ASCII has no Japanese in it
+    def count_ja_graph_char()
+      0
+    end
+
+    # override default method, as ASCII has no Japanese in it
+    def count_ja_blank_char()
+      0
+    end
+
+    # override default method, as ASCII has no Japanese in it
+    def count_ja_word()
+      0
+    end
+
+    # override default method, as ASCII has no Japanese in it
+    def count_ja_valid_word()
+      0
+    end
+
+    CharString.register_encoding(self)
+
+  end  # module ASCII
+end

data/lib/docdiff/encoding/ja_eucjp.rb

@@ -0,0 +1,269 @@
+# Japanese EUC-JP encoding module for CharString
+# 2003- Hisashi MORITA
+
+module CharString
+  module EUC_JP
+
+    Encoding = "EUC-JP"
+
+    # character table based on: 
+    # ftp://ftp.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/JIS0208.TXT
+
+    CNTRL =     "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" \
+                "\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13" \
+                "\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d" \
+                "\x1e\x1f\x7f"
+    SPACE =     "\x09\x0a\x0b\x0c\x0d\x20"
+    BLANK =     "\x09\x20"
+    DIGIT =     "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39"
+    ALPHA =     "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a" \
+                "\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" \
+                "\x55\x56\x57\x58\x59\x5a\x61\x62\x63\x64" \
+                "\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e" \
+                "\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78" \
+                "\x79\x7a"
+    ALNUM =     "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" \
+                "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a" \
+                "\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" \
+                "\x55\x56\x57\x58\x59\x5a\x61\x62\x63\x64" \
+                "\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e" \
+                "\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78" \
+                "\x79\x7a"
+    PUNCT =     "\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a" \
+                "\x2b\x2c\x2d\x2e\x2f\x3a\x3b\x3c\x3d\x3e" \
+                "\x3f\x40\x5b\x5c\x5d\x5e\x5f\x60\x7b\x7c" \
+                "\x7d\x7e"
+    LOWER =     "\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a" \
+                "\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74" \
+                "\x75\x76\x77\x78\x79\x7a"
+    UPPER =     "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a" \
+                "\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" \
+                "\x55\x56\x57\x58\x59\x5a"
+    PRINT =     "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29" \
+                "\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33" \
+                "\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d" \
+                "\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47" \
+                "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51" \
+                "\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b" \
+                "\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" \
+                "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" \
+                "\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79" \
+                "\x7a\x7b\x7c\x7d\x7e"
+    GRAPH =     "\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a" \
+                "\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34" \
+                "\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e" \
+                "\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48" \
+                "\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52" \
+                "\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c" \
+                "\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65\x66" \
+                "\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70" \
+                "\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a" \
+                "\x7b\x7c\x7d\x7e"
+    XDIGIT =    "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" \
+                "\x41\x42\x43\x44\x45\x46\x61\x62\x63\x64" \
+                "\x65\x66"
+    JA_SPACE =  "\xa1\xa1"
+    JA_BLANK =  "\xa1\xa1"
+    JA_ALNUM =  "\xa3\xb0\xa3\xb1\xa3\xb2\xa3\xb3\xa3\xb4" \
+                "\xa3\xb5\xa3\xb6\xa3\xb7\xa3\xb8\xa3\xb9" \
+                "\xa3\xc1\xa3\xc2\xa3\xc3\xa3\xc4\xa3\xc5" \
+                "\xa3\xc6\xa3\xc7\xa3\xc8\xa3\xc9\xa3\xca" \
+                "\xa3\xcb\xa3\xcc\xa3\xcd\xa3\xce\xa3\xcf" \
+                "\xa3\xd0\xa3\xd1\xa3\xd2\xa3\xd3\xa3\xd4" \
+                "\xa3\xd5\xa3\xd6\xa3\xd7\xa3\xd8\xa3\xd9" \
+                "\xa3\xda\xa3\xe1\xa3\xe2\xa3\xe3\xa3\xe4" \
+                "\xa3\xe5\xa3\xe6\xa3\xe7\xa3\xe8\xa3\xe9" \
+                "\xa3\xea\xa3\xeb\xa3\xec\xa3\xed\xa3\xee" \
+                "\xa3\xef\xa3\xf0\xa3\xf1\xa3\xf2\xa3\xf3" \
+                "\xa3\xf4\xa3\xf5\xa3\xf6\xa3\xf7\xa3\xf8" \
+                "\xa3\xf9\xa3\xfa"
+    JA_PUNCT =  "\xa1\xa2\xa1\xa3\xa1\xa4\xa1\xa5\xa1\xa6" \
+                "\xa1\xa7\xa1\xa8\xa1\xa9\xa1\xaa\xa1\xab" \
+                "\xa1\xac\xa1\xad\xa1\xae\xa1\xaf\xa1\xb0" \
+                "\xa1\xb1\xa1\xb2\xa1\xb3\xa1\xb4\xa1\xb5" \
+                "\xa1\xb6\xa1\xb7\xa1\xb8\xa1\xb9\xa1\xba" \
+                "\xa1\xbb\xa1\xbc\xa1\xbd\xa1\xbe\xa1\xbf" \
+                "\xa1\xc0\xa1\xc1\xa1\xc2\xa1\xc3\xa1\xc4" \
+                "\xa1\xc5\xa1\xc6\xa1\xc7\xa1\xc8\xa1\xc9" \
+                "\xa1\xca\xa1\xcb\xa1\xcc\xa1\xcd\xa1\xce" \
+                "\xa1\xcf\xa1\xd0\xa1\xd1\xa1\xd2\xa1\xd3" \
+                "\xa1\xd4\xa1\xd5\xa1\xd6\xa1\xd7\xa1\xd8" \
+                "\xa1\xd9\xa1\xda\xa1\xdb\xa1\xdc\xa1\xdd" \
+                "\xa1\xde\xa1\xdf\xa1\xe0\xa1\xe1\xa1\xe2" \
+                "\xa1\xe3\xa1\xe4\xa1\xe5\xa1\xe6\xa1\xe7" \
+                "\xa1\xe8\xa1\xe9\xa1\xea\xa1\xeb\xa1\xec" \
+                "\xa1\xed\xa1\xee\xa1\xef\xa1\xf0\xa1\xf1" \
+                "\xa1\xf2\xa1\xf3\xa1\xf4\xa1\xf5\xa1\xf6" \
+                "\xa1\xf7\xa1\xf8\xa1\xf9\xa1\xfa\xa1\xfb" \
+                "\xa1\xfc\xa1\xfd\xa1\xfe\xa2\xa1\xa2\xa2" \
+                "\xa2\xa3\xa2\xa4\xa2\xa5\xa2\xa6\xa2\xa7" \
+                "\xa2\xa8\xa2\xa9\xa2\xaa\xa2\xab\xa2\xac" \
+                "\xa2\xad\xa2\xae\xa2\xba\xa2\xbb\xa2\xbc" \
+                "\xa2\xbd\xa2\xbe\xa2\xbf\xa2\xc0\xa2\xc1" \
+                "\xa2\xca\xa2\xcb\xa2\xcc\xa2\xcd\xa2\xce" \
+                "\xa2\xcf\xa2\xd0\xa2\xdc\xa2\xdd\xa2\xde" \
+                "\xa2\xdf\xa2\xe0\xa2\xe1\xa2\xe2\xa2\xe3" \
+                "\xa2\xe4\xa2\xe5\xa2\xe6\xa2\xe7\xa2\xe8" \
+                "\xa2\xe9\xa2\xea\xa2\xf2\xa2\xf3\xa2\xf4" \
+                "\xa2\xf5\xa2\xf6\xa2\xf7\xa2\xf8\xa2\xf9" \
+                "\xa2\xfe\xa6\xa1\xa6\xa2\xa6\xa3\xa6\xa4" \
+                "\xa6\xa5\xa6\xa6\xa6\xa7\xa6\xa8\xa6\xa9" \
+                "\xa6\xaa\xa6\xab\xa6\xac\xa6\xad\xa6\xae" \
+                "\xa6\xaf\xa6\xb0\xa6\xb1\xa6\xb2\xa6\xb3" \
+                "\xa6\xb4\xa6\xb5\xa6\xb6\xa6\xb7\xa6\xb8" \
+                "\xa6\xc1\xa6\xc2\xa6\xc3\xa6\xc4\xa6\xc5" \
+                "\xa6\xc6\xa6\xc7\xa6\xc8\xa6\xc9\xa6\xca" \
+                "\xa6\xcb\xa6\xcc\xa6\xcd\xa6\xce\xa6\xcf" \
+                "\xa6\xd0\xa6\xd1\xa6\xd2\xa6\xd3\xa6\xd4" \
+                "\xa6\xd5\xa6\xd6\xa6\xd7\xa6\xd8\xa7\xa1" \
+                "\xa7\xa2\xa7\xa3\xa7\xa4\xa7\xa5\xa7\xa6" \
+                "\xa7\xa7\xa7\xa8\xa7\xa9\xa7\xaa\xa7\xab" \
+                "\xa7\xac\xa7\xad\xa7\xae\xa7\xaf\xa7\xb0" \
+                "\xa7\xb1\xa7\xb2\xa7\xb3\xa7\xb4\xa7\xb5" \
+                "\xa7\xb6\xa7\xb7\xa7\xb8\xa7\xb9\xa7\xba" \
+                "\xa7\xbb\xa7\xbc\xa7\xbd\xa7\xbe\xa7\xbf" \
+                "\xa7\xc0\xa7\xc1\xa7\xd1\xa7\xd2\xa7\xd3" \
+                "\xa7\xd4\xa7\xd5\xa7\xd6\xa7\xd7\xa7\xd8" \
+                "\xa7\xd9\xa7\xda\xa7\xdb\xa7\xdc\xa7\xdd" \
+                "\xa7\xde\xa7\xdf\xa7\xe0\xa7\xe1\xa7\xe2" \
+                "\xa7\xe3\xa7\xe4\xa7\xe5\xa7\xe6\xa7\xe7" \
+                "\xa7\xe8\xa7\xe9\xa7\xea\xa7\xeb\xa7\xec" \
+                "\xa7\xed\xa7\xee\xa7\xef\xa7\xf0\xa7\xf1" \
+                "\xa8\xa1\xa8\xa2\xa8\xa3\xa8\xa4\xa8\xa5" \
+                "\xa8\xa6\xa8\xa7\xa8\xa8\xa8\xa9\xa8\xaa" \
+                "\xa8\xab\xa8\xac\xa8\xad\xa8\xae\xa8\xaf" \
+                "\xa8\xb0\xa8\xb1\xa8\xb2\xa8\xb3\xa8\xb4" \
+                "\xa8\xb5\xa8\xb6\xa8\xb7\xa8\xb8\xa8\xb9" \
+                "\xa8\xba\xa8\xbb\xa8\xbc\xa8\xbd\xa8\xbe" \
+                "\xa8\xbf\xa8\xc0"
+    HIRA =      "\xa4\xa1\xa4\xa2\xa4\xa3\xa4\xa4\xa4\xa5" \
+                "\xa4\xa6\xa4\xa7\xa4\xa8\xa4\xa9\xa4\xaa" \
+                "\xa4\xab\xa4\xac\xa4\xad\xa4\xae\xa4\xaf" \
+                "\xa4\xb0\xa4\xb1\xa4\xb2\xa4\xb3\xa4\xb4" \
+                "\xa4\xb5\xa4\xb6\xa4\xb7\xa4\xb8\xa4\xb9" \
+                "\xa4\xba\xa4\xbb\xa4\xbc\xa4\xbd\xa4\xbe" \
+                "\xa4\xbf\xa4\xc0\xa4\xc1\xa4\xc2\xa4\xc3" \
+                "\xa4\xc4\xa4\xc5\xa4\xc6\xa4\xc7\xa4\xc8" \
+                "\xa4\xc9\xa4\xca\xa4\xcb\xa4\xcc\xa4\xcd" \
+                "\xa4\xce\xa4\xcf\xa4\xd0\xa4\xd1\xa4\xd2" \
+                "\xa4\xd3\xa4\xd4\xa4\xd5\xa4\xd6\xa4\xd7" \
+                "\xa4\xd8\xa4\xd9\xa4\xda\xa4\xdb\xa4\xdc" \
+                "\xa4\xdd\xa4\xde\xa4\xdf\xa4\xe0\xa4\xe1" \
+                "\xa4\xe2\xa4\xe3\xa4\xe4\xa4\xe5\xa4\xe6" \
+                "\xa4\xe7\xa4\xe8\xa4\xe9\xa4\xea\xa4\xeb" \
+                "\xa4\xec\xa4\xed\xa4\xee\xa4\xef\xa4\xf0" \
+                "\xa4\xf1\xa4\xf2\xa4\xf3"  # [\xa4\xa1-\xa4\xf3]
+    HIRA_EX =   HIRA +
+                "\xa1\xbc" + 
+                "\xa1\xb3\xa1\xb4" # add onbiki and kanagaeshi(hira)
+    KATA =      "\xa5\xa1\xa5\xa2\xa5\xa3\xa5\xa4\xa5\xa5" \
+                "\xa5\xa6\xa5\xa7\xa5\xa8\xa5\xa9\xa5\xaa" \
+                "\xa5\xab\xa5\xac\xa5\xad\xa5\xae\xa5\xaf" \
+                "\xa5\xb0\xa5\xb1\xa5\xb2\xa5\xb3\xa5\xb4" \
+                "\xa5\xb5\xa5\xb6\xa5\xb7\xa5\xb8\xa5\xb9" \
+                "\xa5\xba\xa5\xbb\xa5\xbc\xa5\xbd\xa5\xbe" \
+                "\xa5\xbf\xa5\xc0\xa5\xc1\xa5\xc2\xa5\xc3" \
+                "\xa5\xc4\xa5\xc5\xa5\xc6\xa5\xc7\xa5\xc8" \
+                "\xa5\xc9\xa5\xca\xa5\xcb\xa5\xcc\xa5\xcd" \
+                "\xa5\xce\xa5\xcf\xa5\xd0\xa5\xd1\xa5\xd2" \
+                "\xa5\xd3\xa5\xd4\xa5\xd5\xa5\xd6\xa5\xd7" \
+                "\xa5\xd8\xa5\xd9\xa5\xda\xa5\xdb\xa5\xdc" \
+                "\xa5\xdd\xa5\xde\xa5\xdf\xa5\xe0\xa5\xe1" \
+                "\xa5\xe2\xa5\xe3\xa5\xe4\xa5\xe5\xa5\xe6" \
+                "\xa5\xe7\xa5\xe8\xa5\xe9\xa5\xea\xa5\xeb" \
+                "\xa5\xec\xa5\xed\xa5\xee\xa5\xef\xa5\xf0" \
+                "\xa5\xf1\xa5\xf2\xa5\xf3\xa5\xf4\xa5\xf5" \
+                "\xa5\xf6"  # [\xa5\xa1-\xa5\xf6]
+    KATA_EX =   KATA + 
+                "\xa1\xbc" + 
+                "\xa1\xb5\xa1\xb6" # add onbiki and kanagaeshi(kata)
+    KANJI =     "\xb0\xa1-\xb0\xfe" \
+                "\xb1\xa1-\xb1\xfe" \
+                "\xb2\xa1-\xb2\xfe" \
+                "\xb3\xa1-\xb3\xfe" \
+                "\xb4\xa1-\xb4\xfe" \
+                "\xb5\xa1-\xb5\xfe" \
+                "\xb6\xa1-\xb6\xfe" \
+                "\xb7\xa1-\xb7\xfe" \
+                "\xb8\xa1-\xb8\xfe" \
+                "\xb9\xa1-\xb9\xfe" \
+                "\xba\xa1-\xba\xfe" \
+                "\xbb\xa1-\xbb\xfe" \
+                "\xbc\xa1-\xbc\xfe" \
+                "\xbd\xa1-\xbd\xfe" \
+                "\xbe\xa1-\xbe\xfe" \
+                "\xbf\xa1-\xbf\xfe" \
+                "\xc0\xa1-\xc0\xfe" \
+                "\xc1\xa1-\xc1\xfe" \
+                "\xc2\xa1-\xc2\xfe" \
+                "\xc3\xa1-\xc3\xfe" \
+                "\xc4\xa1-\xc4\xfe" \
+                "\xc5\xa1-\xc5\xfe" \
+                "\xc6\xa1-\xc6\xfe" \
+                "\xc7\xa1-\xc7\xfe" \
+                "\xc8\xa1-\xc8\xfe" \
+                "\xc9\xa1-\xc9\xfe" \
+                "\xca\xa1-\xca\xfe" \
+                "\xcb\xa1-\xcb\xfe" \
+                "\xcc\xa1-\xcc\xfe" \
+                "\xcd\xa1-\xcd\xfe" \
+                "\xce\xa1-\xce\xfe" \
+                "\xcf\xa1-\xcf\xd3" \
+                "\xd0\xa1-\xd0\xfe" \
+                "\xd1\xa1-\xd1\xfe" \
+                "\xd2\xa1-\xd2\xfe" \
+                "\xd3\xa1-\xd3\xfe" \
+                "\xd4\xa1-\xd4\xfe" \
+                "\xd5\xa1-\xd5\xfe" \
+                "\xd6\xa1-\xd6\xfe" \
+                "\xd7\xa1-\xd7\xfe" \
+                "\xd8\xa1-\xd8\xfe" \
+                "\xd9\xa1-\xd9\xfe" \
+                "\xda\xa1-\xda\xfe" \
+                "\xdb\xa1-\xdb\xfe" \
+                "\xdc\xa1-\xdc\xfe" \
+                "\xdd\xa1-\xdd\xfe" \
+                "\xde\xa1-\xde\xfe" \
+                "\xdf\xa1-\xdf\xfe" \
+                "\xe0\xa1-\xe0\xfe" \
+                "\xe1\xa1-\xe1\xfe" \
+                "\xe2\xa1-\xe2\xfe" \
+                "\xe3\xa1-\xe3\xfe" \
+                "\xe4\xa1-\xe4\xfe" \
+                "\xe5\xa1-\xe5\xfe" \
+                "\xe6\xa1-\xe6\xfe" \
+                "\xe7\xa1-\xe7\xfe" \
+                "\xe8\xa1-\xe8\xfe" \
+                "\xe9\xa1-\xe9\xfe" \
+                "\xea\xa1-\xea\xfe" \
+                "\xeb\xa1-\xeb\xfe" \
+                "\xec\xa1-\xec\xfe" \
+                "\xed\xa1-\xed\xfe" \
+                "\xee\xa1-\xee\xfe" \
+                "\xef\xa1-\xef\xfe" \
+                "\xf0\xa1-\xf0\xfe" \
+                "\xf1\xa1-\xf1\xfe" \
+                "\xf2\xa1-\xf2\xfe" \
+                "\xf3\xa1-\xf3\xfe" \
+                "\xf4\xa1-\xf4\xa6"
+    KANJI_EX =  KANJI + "\xa1\xb9" # + noma
+    JA_GRAPH =  JA_ALNUM + JA_PUNCT + HIRA + KATA + KANJI
+    JA_PRINT =  JA_GRAPH + JA_BLANK
+
+    PUNCT.replace(Regexp.quote(PUNCT)) # kludge to avoid warning "character class has `[' without escape"
+    PRINT.replace(Regexp.quote(PRINT)) # kludge to avoid warning "character class has `[' without escape"
+    GRAPH.replace(Regexp.quote(GRAPH)) # kludge to avoid warning "character class has `[' without escape"
+
+    WORD_REGEXP_SRC = ["(?:[#{GRAPH}]+[#{BLANK}]?)", 
+                       "|(?:[#{SPACE}]+)", 
+                       "|(?:[#{KANJI_EX}]+[#{HIRA}]+)", 
+                       "|(?:[#{KATA_EX}]+[#{HIRA}]+)", 
+                       "|(?:[#{KANJI_EX}]+)", 
+                       "|(?:[#{KATA_EX}]+)", 
+                       "|(?:[#{HIRA_EX}]+)", 
+                       "|(?:.+?)"].join
+
+    CharString.register_encoding(self)
+
+  end  # module EUCJP
+end